1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return MacroAssembler::max_trampoline_stub_size();
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_CompareAndSwapB:
1512 case Op_CompareAndSwapS:
1513 case Op_GetAndSetI:
1514 case Op_GetAndSetL:
1515 case Op_GetAndSetP:
1516 case Op_GetAndSetN:
1517 case Op_GetAndAddI:
1518 case Op_GetAndAddL:
1519 return true;
1520 case Op_CompareAndExchangeI:
1521 case Op_CompareAndExchangeN:
1522 case Op_CompareAndExchangeB:
1523 case Op_CompareAndExchangeS:
1524 case Op_CompareAndExchangeL:
1525 case Op_CompareAndExchangeP:
1526 case Op_WeakCompareAndSwapB:
1527 case Op_WeakCompareAndSwapS:
1528 case Op_WeakCompareAndSwapI:
1529 case Op_WeakCompareAndSwapL:
1530 case Op_WeakCompareAndSwapP:
1531 case Op_WeakCompareAndSwapN:
1532 return maybe_volatile;
1533 default:
1534 return false;
1535 }
1536 }
1537
1538 // helper to determine the maximum number of Phi nodes we may need to
1539 // traverse when searching from a card mark membar for the merge mem
1540 // feeding a trailing membar or vice versa
1541
1542 // predicates controlling emit of ldr<x>/ldar<x>
1543
1544 bool unnecessary_acquire(const Node *barrier)
1545 {
1546 assert(barrier->is_MemBar(), "expecting a membar");
1547
1548 MemBarNode* mb = barrier->as_MemBar();
1549
1550 if (mb->trailing_load()) {
1551 return true;
1552 }
1553
1554 if (mb->trailing_load_store()) {
1555 Node* load_store = mb->in(MemBarNode::Precedent);
1556 assert(load_store->is_LoadStore(), "unexpected graph shape");
1557 return is_CAS(load_store->Opcode(), true);
1558 }
1559
1560 return false;
1561 }
1562
1563 bool needs_acquiring_load(const Node *n)
1564 {
1565 assert(n->is_Load(), "expecting a load");
1566 LoadNode *ld = n->as_Load();
1567 return ld->is_acquire();
1568 }
1569
1570 bool unnecessary_release(const Node *n)
1571 {
1572 assert((n->is_MemBar() &&
1573 n->Opcode() == Op_MemBarRelease),
1574 "expecting a release membar");
1575
1576 MemBarNode *barrier = n->as_MemBar();
1577 if (!barrier->leading()) {
1578 return false;
1579 } else {
1580 Node* trailing = barrier->trailing_membar();
1581 MemBarNode* trailing_mb = trailing->as_MemBar();
1582 assert(trailing_mb->trailing(), "Not a trailing membar?");
1583 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1584
1585 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1586 if (mem->is_Store()) {
1587 assert(mem->as_Store()->is_release(), "");
1588 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1589 return true;
1590 } else {
1591 assert(mem->is_LoadStore(), "");
1592 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1593 return is_CAS(mem->Opcode(), true);
1594 }
1595 }
1596 return false;
1597 }
1598
1599 bool unnecessary_volatile(const Node *n)
1600 {
1601 // assert n->is_MemBar();
1602 MemBarNode *mbvol = n->as_MemBar();
1603
1604 bool release = mbvol->trailing_store();
1605 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1606 #ifdef ASSERT
1607 if (release) {
1608 Node* leading = mbvol->leading_membar();
1609 assert(leading->Opcode() == Op_MemBarRelease, "");
1610 assert(leading->as_MemBar()->leading_store(), "");
1611 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1612 }
1613 #endif
1614
1615 return release;
1616 }
1617
1618 // predicates controlling emit of str<x>/stlr<x>
1619
1620 bool needs_releasing_store(const Node *n)
1621 {
1622 // assert n->is_Store();
1623 StoreNode *st = n->as_Store();
1624 return st->trailing_membar() != nullptr;
1625 }
1626
1627 // predicate controlling translation of CAS
1628 //
1629 // returns true if CAS needs to use an acquiring load otherwise false
1630
1631 bool needs_acquiring_load_exclusive(const Node *n)
1632 {
1633 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1634 LoadStoreNode* ldst = n->as_LoadStore();
1635 if (is_CAS(n->Opcode(), false)) {
1636 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1637 } else {
1638 return ldst->trailing_membar() != nullptr;
1639 }
1640
1641 // so we can just return true here
1642 return true;
1643 }
1644
1645 #define __ masm->
1646
1647 // advance declarations for helper functions to convert register
1648 // indices to register objects
1649
1650 // the ad file has to provide implementations of certain methods
1651 // expected by the generic code
1652 //
1653 // REQUIRED FUNCTIONALITY
1654
1655 //=============================================================================
1656
1657 // !!!!! Special hack to get all types of calls to specify the byte offset
1658 // from the start of the call to the point where the return address
1659 // will point.
1660
1661 int MachCallStaticJavaNode::ret_addr_offset()
1662 {
1663 // call should be a simple bl
1664 int off = 4;
1665 return off;
1666 }
1667
1668 int MachCallDynamicJavaNode::ret_addr_offset()
1669 {
1670 return 16; // movz, movk, movk, bl
1671 }
1672
1673 int MachCallRuntimeNode::ret_addr_offset() {
1674 // for generated stubs the call will be
1675 // bl(addr)
1676 // or with far branches
1677 // bl(trampoline_stub)
1678 // for real runtime callouts it will be six instructions
1679 // see aarch64_enc_java_to_runtime
1680 // adr(rscratch2, retaddr)
1681 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1682 // lea(rscratch1, RuntimeAddress(addr)
1683 // blr(rscratch1)
1684 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1685 if (cb) {
1686 return 1 * NativeInstruction::instruction_size;
1687 } else {
1688 return 6 * NativeInstruction::instruction_size;
1689 }
1690 }
1691
1692 //=============================================================================
1693
1694 #ifndef PRODUCT
1695 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1696 st->print("BREAKPOINT");
1697 }
1698 #endif
1699
1700 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1701 __ brk(0);
1702 }
1703
1704 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1705 return MachNode::size(ra_);
1706 }
1707
1708 //=============================================================================
1709
1710 #ifndef PRODUCT
1711 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1712 st->print("nop \t# %d bytes pad for loops and calls", _count);
1713 }
1714 #endif
1715
1716 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1717 for (int i = 0; i < _count; i++) {
1718 __ nop();
1719 }
1720 }
1721
1722 uint MachNopNode::size(PhaseRegAlloc*) const {
1723 return _count * NativeInstruction::instruction_size;
1724 }
1725
1726 //=============================================================================
1727 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1728
1729 int ConstantTable::calculate_table_base_offset() const {
1730 return 0; // absolute addressing, no offset
1731 }
1732
1733 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1734 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1735 ShouldNotReachHere();
1736 }
1737
1738 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1739 // Empty encoding
1740 }
1741
1742 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1743 return 0;
1744 }
1745
1746 #ifndef PRODUCT
1747 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1748 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1749 }
1750 #endif
1751
1752 #ifndef PRODUCT
1753 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1754 Compile* C = ra_->C;
1755
1756 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1757
1758 if (C->output()->need_stack_bang(framesize))
1759 st->print("# stack bang size=%d\n\t", framesize);
1760
1761 if (VM_Version::use_rop_protection()) {
1762 st->print("ldr zr, [lr]\n\t");
1763 st->print("paciaz\n\t");
1764 }
1765 if (framesize < ((1 << 9) + 2 * wordSize)) {
1766 st->print("sub sp, sp, #%d\n\t", framesize);
1767 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1768 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1769 } else {
1770 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1771 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1772 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1773 st->print("sub sp, sp, rscratch1");
1774 }
1775 if (C->stub_function() == nullptr) {
1776 st->print("\n\t");
1777 st->print("ldr rscratch1, [guard]\n\t");
1778 st->print("dmb ishld\n\t");
1779 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1780 st->print("cmp rscratch1, rscratch2\n\t");
1781 st->print("b.eq skip");
1782 st->print("\n\t");
1783 st->print("blr #nmethod_entry_barrier_stub\n\t");
1784 st->print("b skip\n\t");
1785 st->print("guard: int\n\t");
1786 st->print("\n\t");
1787 st->print("skip:\n\t");
1788 }
1789 }
1790 #endif
1791
1792 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1793 Compile* C = ra_->C;
1794
1795 // n.b. frame size includes space for return pc and rfp
1796 const int framesize = C->output()->frame_size_in_bytes();
1797
1798 if (C->clinit_barrier_on_entry()) {
1799 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1800
1801 Label L_skip_barrier;
1802
1803 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1804 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1805 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1806 __ bind(L_skip_barrier);
1807 }
1808
1809 if (C->max_vector_size() > 0) {
1810 __ reinitialize_ptrue();
1811 }
1812
1813 int bangsize = C->output()->bang_size_in_bytes();
1814 if (C->output()->need_stack_bang(bangsize))
1815 __ generate_stack_overflow_check(bangsize);
1816
1817 __ build_frame(framesize);
1818
1819 if (C->stub_function() == nullptr) {
1820 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1821 // Dummy labels for just measuring the code size
1822 Label dummy_slow_path;
1823 Label dummy_continuation;
1824 Label dummy_guard;
1825 Label* slow_path = &dummy_slow_path;
1826 Label* continuation = &dummy_continuation;
1827 Label* guard = &dummy_guard;
1828 if (!Compile::current()->output()->in_scratch_emit_size()) {
1829 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1830 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1831 Compile::current()->output()->add_stub(stub);
1832 slow_path = &stub->entry();
1833 continuation = &stub->continuation();
1834 guard = &stub->guard();
1835 }
1836 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1837 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1838 }
1839
1840 if (VerifyStackAtCalls) {
1841 Unimplemented();
1842 }
1843
1844 C->output()->set_frame_complete(__ offset());
1845
1846 if (C->has_mach_constant_base_node()) {
1847 // NOTE: We set the table base offset here because users might be
1848 // emitted before MachConstantBaseNode.
1849 ConstantTable& constant_table = C->output()->constant_table();
1850 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1851 }
1852 }
1853
1854 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1855 {
1856 return MachNode::size(ra_); // too many variables; just compute it
1857 // the hard way
1858 }
1859
1860 int MachPrologNode::reloc() const
1861 {
1862 return 0;
1863 }
1864
1865 //=============================================================================
1866
1867 #ifndef PRODUCT
1868 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1869 Compile* C = ra_->C;
1870 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1871
1872 st->print("# pop frame %d\n\t",framesize);
1873
1874 if (framesize == 0) {
1875 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1876 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1877 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1878 st->print("add sp, sp, #%d\n\t", framesize);
1879 } else {
1880 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1881 st->print("add sp, sp, rscratch1\n\t");
1882 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1883 }
1884 if (VM_Version::use_rop_protection()) {
1885 st->print("autiaz\n\t");
1886 st->print("ldr zr, [lr]\n\t");
1887 }
1888
1889 if (do_polling() && C->is_method_compilation()) {
1890 st->print("# test polling word\n\t");
1891 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1892 st->print("cmp sp, rscratch1\n\t");
1893 st->print("bhi #slow_path");
1894 }
1895 }
1896 #endif
1897
1898 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1899 Compile* C = ra_->C;
1900 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1901
1902 __ remove_frame(framesize);
1903
1904 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1905 __ reserved_stack_check();
1906 }
1907
1908 if (do_polling() && C->is_method_compilation()) {
1909 Label dummy_label;
1910 Label* code_stub = &dummy_label;
1911 if (!C->output()->in_scratch_emit_size()) {
1912 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1913 C->output()->add_stub(stub);
1914 code_stub = &stub->entry();
1915 }
1916 __ relocate(relocInfo::poll_return_type);
1917 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1918 }
1919 }
1920
1921 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1922 // Variable size. Determine dynamically.
1923 return MachNode::size(ra_);
1924 }
1925
1926 int MachEpilogNode::reloc() const {
1927 // Return number of relocatable values contained in this instruction.
1928 return 1; // 1 for polling page.
1929 }
1930
1931 const Pipeline * MachEpilogNode::pipeline() const {
1932 return MachNode::pipeline_class();
1933 }
1934
1935 //=============================================================================
1936
1937 static enum RC rc_class(OptoReg::Name reg) {
1938
1939 if (reg == OptoReg::Bad) {
1940 return rc_bad;
1941 }
1942
1943 // we have 32 int registers * 2 halves
1944 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1945
1946 if (reg < slots_of_int_registers) {
1947 return rc_int;
1948 }
1949
1950 // we have 32 float register * 8 halves
1951 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1952 if (reg < slots_of_int_registers + slots_of_float_registers) {
1953 return rc_float;
1954 }
1955
1956 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1957 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1958 return rc_predicate;
1959 }
1960
1961 // Between predicate regs & stack is the flags.
1962 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1963
1964 return rc_stack;
1965 }
1966
1967 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1968 Compile* C = ra_->C;
1969
1970 // Get registers to move.
1971 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1972 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1973 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1974 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1975
1976 enum RC src_hi_rc = rc_class(src_hi);
1977 enum RC src_lo_rc = rc_class(src_lo);
1978 enum RC dst_hi_rc = rc_class(dst_hi);
1979 enum RC dst_lo_rc = rc_class(dst_lo);
1980
1981 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1982
1983 if (src_hi != OptoReg::Bad && !bottom_type()->isa_pvectmask()) {
1984 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1985 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1986 "expected aligned-adjacent pairs");
1987 }
1988
1989 if (src_lo == dst_lo && src_hi == dst_hi) {
1990 return 0; // Self copy, no move.
1991 }
1992
1993 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1994 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1995 int src_offset = ra_->reg2offset(src_lo);
1996 int dst_offset = ra_->reg2offset(dst_lo);
1997
1998 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
1999 uint ireg = ideal_reg();
2000 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
2001 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
2002 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
2003 if (ireg == Op_VecA && masm) {
2004 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2005 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2006 // stack->stack
2007 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2008 sve_vector_reg_size_in_bytes);
2009 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2010 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2011 sve_vector_reg_size_in_bytes);
2012 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2013 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2016 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 as_FloatRegister(Matcher::_regEncode[src_lo]),
2018 as_FloatRegister(Matcher::_regEncode[src_lo]));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 } else if (masm) {
2023 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2024 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2025 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2026 // stack->stack
2027 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2028 if (ireg == Op_VecD) {
2029 __ unspill(rscratch1, true, src_offset);
2030 __ spill(rscratch1, true, dst_offset);
2031 } else {
2032 __ spill_copy128(src_offset, dst_offset);
2033 }
2034 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2035 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2036 ireg == Op_VecD ? __ T8B : __ T16B,
2037 as_FloatRegister(Matcher::_regEncode[src_lo]));
2038 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2039 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2040 ireg == Op_VecD ? __ D : __ Q,
2041 ra_->reg2offset(dst_lo));
2042 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2043 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2044 ireg == Op_VecD ? __ D : __ Q,
2045 ra_->reg2offset(src_lo));
2046 } else {
2047 ShouldNotReachHere();
2048 }
2049 }
2050 } else if (masm) {
2051 switch (src_lo_rc) {
2052 case rc_int:
2053 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2054 if (is64) {
2055 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2056 as_Register(Matcher::_regEncode[src_lo]));
2057 } else {
2058 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2059 as_Register(Matcher::_regEncode[src_lo]));
2060 }
2061 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2062 if (is64) {
2063 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2064 as_Register(Matcher::_regEncode[src_lo]));
2065 } else {
2066 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2067 as_Register(Matcher::_regEncode[src_lo]));
2068 }
2069 } else { // gpr --> stack spill
2070 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2071 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2072 }
2073 break;
2074 case rc_float:
2075 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2076 if (is64) {
2077 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2078 as_FloatRegister(Matcher::_regEncode[src_lo]));
2079 } else {
2080 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2081 as_FloatRegister(Matcher::_regEncode[src_lo]));
2082 }
2083 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2084 if (is64) {
2085 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2086 as_FloatRegister(Matcher::_regEncode[src_lo]));
2087 } else {
2088 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2089 as_FloatRegister(Matcher::_regEncode[src_lo]));
2090 }
2091 } else { // fpr --> stack spill
2092 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2093 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2094 is64 ? __ D : __ S, dst_offset);
2095 }
2096 break;
2097 case rc_stack:
2098 if (dst_lo_rc == rc_int) { // stack --> gpr load
2099 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2100 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2101 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2102 is64 ? __ D : __ S, src_offset);
2103 } else if (dst_lo_rc == rc_predicate) {
2104 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2105 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2106 } else { // stack --> stack copy
2107 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2108 if (ideal_reg() == Op_RegVectMask) {
2109 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2110 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2111 } else {
2112 __ unspill(rscratch1, is64, src_offset);
2113 __ spill(rscratch1, is64, dst_offset);
2114 }
2115 }
2116 break;
2117 case rc_predicate:
2118 if (dst_lo_rc == rc_predicate) {
2119 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2120 } else if (dst_lo_rc == rc_stack) {
2121 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2122 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2123 } else {
2124 assert(false, "bad src and dst rc_class combination.");
2125 ShouldNotReachHere();
2126 }
2127 break;
2128 default:
2129 assert(false, "bad rc_class for spill");
2130 ShouldNotReachHere();
2131 }
2132 }
2133
2134 if (st) {
2135 st->print("spill ");
2136 if (src_lo_rc == rc_stack) {
2137 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2138 } else {
2139 st->print("%s -> ", Matcher::regName[src_lo]);
2140 }
2141 if (dst_lo_rc == rc_stack) {
2142 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2143 } else {
2144 st->print("%s", Matcher::regName[dst_lo]);
2145 }
2146 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
2147 int vsize = 0;
2148 switch (ideal_reg()) {
2149 case Op_VecD:
2150 vsize = 64;
2151 break;
2152 case Op_VecX:
2153 vsize = 128;
2154 break;
2155 case Op_VecA:
2156 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2157 break;
2158 default:
2159 assert(false, "bad register type for spill");
2160 ShouldNotReachHere();
2161 }
2162 st->print("\t# vector spill size = %d", vsize);
2163 } else if (ideal_reg() == Op_RegVectMask) {
2164 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2165 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2166 st->print("\t# predicate spill size = %d", vsize);
2167 } else {
2168 st->print("\t# spill size = %d", is64 ? 64 : 32);
2169 }
2170 }
2171
2172 return 0;
2173
2174 }
2175
2176 #ifndef PRODUCT
2177 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2178 if (!ra_)
2179 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2180 else
2181 implementation(nullptr, ra_, false, st);
2182 }
2183 #endif
2184
2185 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2186 implementation(masm, ra_, false, nullptr);
2187 }
2188
2189 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2190 return MachNode::size(ra_);
2191 }
2192
2193 //=============================================================================
2194
2195 #ifndef PRODUCT
2196 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2197 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2198 int reg = ra_->get_reg_first(this);
2199 st->print("add %s, rsp, #%d]\t# box lock",
2200 Matcher::regName[reg], offset);
2201 }
2202 #endif
2203
2204 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2205 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2206 int reg = ra_->get_encode(this);
2207
2208 // This add will handle any 24-bit signed offset. 24 bits allows an
2209 // 8 megabyte stack frame.
2210 __ add(as_Register(reg), sp, offset);
2211 }
2212
2213 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2214 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2215 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2216
2217 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2218 return NativeInstruction::instruction_size;
2219 } else {
2220 return 2 * NativeInstruction::instruction_size;
2221 }
2222 }
2223
2224 //=============================================================================
2225
2226 #ifndef PRODUCT
2227 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2228 {
2229 st->print_cr("# MachUEPNode");
2230 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2231 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2232 st->print_cr("\tcmpw rscratch1, r10");
2233 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2234 }
2235 #endif
2236
2237 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2238 {
2239 __ ic_check(InteriorEntryAlignment);
2240 }
2241
2242 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2243 {
2244 return MachNode::size(ra_);
2245 }
2246
2247 // REQUIRED EMIT CODE
2248
2249 //=============================================================================
2250
2251 // Emit deopt handler code.
2252 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2253 {
2254 // Note that the code buffer's insts_mark is always relative to insts.
2255 // That's why we must use the macroassembler to generate a handler.
2256 address base = __ start_a_stub(size_deopt_handler());
2257 if (base == nullptr) {
2258 ciEnv::current()->record_failure("CodeCache is full");
2259 return 0; // CodeBuffer::expand failed
2260 }
2261
2262 int offset = __ offset();
2263 Label start;
2264 __ bind(start);
2265 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2266
2267 int entry_offset = __ offset();
2268 __ b(start);
2269
2270 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2271 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2272 "out of bounds read in post-call NOP check");
2273 __ end_a_stub();
2274 return entry_offset;
2275 }
2276
2277 // REQUIRED MATCHER CODE
2278
2279 //=============================================================================
2280
2281 bool Matcher::match_rule_supported(int opcode) {
2282 if (!has_match_rule(opcode))
2283 return false;
2284
2285 switch (opcode) {
2286 case Op_OnSpinWait:
2287 return VM_Version::supports_on_spin_wait();
2288 case Op_CacheWB:
2289 case Op_CacheWBPreSync:
2290 case Op_CacheWBPostSync:
2291 if (!VM_Version::supports_data_cache_line_flush()) {
2292 return false;
2293 }
2294 break;
2295 case Op_ExpandBits:
2296 case Op_CompressBits:
2297 if (!VM_Version::supports_svebitperm()) {
2298 return false;
2299 }
2300 break;
2301 case Op_FmaF:
2302 case Op_FmaD:
2303 case Op_FmaVF:
2304 case Op_FmaVD:
2305 if (!UseFMA) {
2306 return false;
2307 }
2308 break;
2309 case Op_FmaHF:
2310 // UseFMA flag also needs to be checked along with FEAT_FP16
2311 if (!UseFMA || !is_feat_fp16_supported()) {
2312 return false;
2313 }
2314 break;
2315 case Op_AddHF:
2316 case Op_SubHF:
2317 case Op_MulHF:
2318 case Op_DivHF:
2319 case Op_MinHF:
2320 case Op_MaxHF:
2321 case Op_SqrtHF:
2322 // Half-precision floating point scalar operations require FEAT_FP16
2323 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2324 // features are supported.
2325 if (!is_feat_fp16_supported()) {
2326 return false;
2327 }
2328 break;
2329 }
2330
2331 return true; // Per default match rules are supported.
2332 }
2333
2334 const RegMask* Matcher::predicate_reg_mask(void) {
2335 return &_PR_REG_mask;
2336 }
2337
2338 bool Matcher::supports_vector_calling_convention(void) {
2339 return EnableVectorSupport;
2340 }
2341
2342 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2343 assert(EnableVectorSupport, "sanity");
2344 int lo = V0_num;
2345 int hi = V0_H_num;
2346 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2347 hi = V0_K_num;
2348 }
2349 return OptoRegPair(hi, lo);
2350 }
2351
2352 // Is this branch offset short enough that a short branch can be used?
2353 //
2354 // NOTE: If the platform does not provide any short branch variants, then
2355 // this method should return false for offset 0.
2356 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2357 // The passed offset is relative to address of the branch.
2358
2359 return (-32768 <= offset && offset < 32768);
2360 }
2361
2362 // Vector width in bytes.
2363 int Matcher::vector_width_in_bytes(BasicType bt) {
2364 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2365 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2366 // Minimum 2 values in vector
2367 if (size < 2*type2aelembytes(bt)) size = 0;
2368 // But never < 4
2369 if (size < 4) size = 0;
2370 return size;
2371 }
2372
2373 // Limits on vector size (number of elements) loaded into vector.
2374 int Matcher::max_vector_size(const BasicType bt) {
2375 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2376 }
2377
2378 int Matcher::min_vector_size(const BasicType bt) {
2379 // Usually, the shortest vector length supported by AArch64 ISA and
2380 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2381 // vectors in a few special cases.
2382 int size;
2383 switch(bt) {
2384 case T_BOOLEAN:
2385 // Load/store a vector mask with only 2 elements for vector types
2386 // such as "2I/2F/2L/2D".
2387 size = 2;
2388 break;
2389 case T_BYTE:
2390 // Generate a "4B" vector, to support vector cast between "8B/16B"
2391 // and "4S/4I/4L/4F/4D".
2392 size = 4;
2393 break;
2394 case T_SHORT:
2395 // Generate a "2S" vector, to support vector cast between "4S/8S"
2396 // and "2I/2L/2F/2D".
2397 size = 2;
2398 break;
2399 default:
2400 // Limit the min vector length to 64-bit.
2401 size = 8 / type2aelembytes(bt);
2402 // The number of elements in a vector should be at least 2.
2403 size = MAX2(size, 2);
2404 }
2405
2406 int max_size = max_vector_size(bt);
2407 return MIN2(size, max_size);
2408 }
2409
2410 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2411 return Matcher::max_vector_size(bt);
2412 }
2413
2414 // Actual max scalable vector register length.
2415 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2416 return Matcher::max_vector_size(bt);
2417 }
2418
2419 // Vector ideal reg.
2420 uint Matcher::vector_ideal_reg(int len) {
2421 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2422 return Op_VecA;
2423 }
2424 switch(len) {
2425 // For 16-bit/32-bit mask vector, reuse VecD.
2426 case 2:
2427 case 4:
2428 case 8: return Op_VecD;
2429 case 16: return Op_VecX;
2430 }
2431 ShouldNotReachHere();
2432 return 0;
2433 }
2434
2435 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2436 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2437 switch (ideal_reg) {
2438 case Op_VecA: return new vecAOper();
2439 case Op_VecD: return new vecDOper();
2440 case Op_VecX: return new vecXOper();
2441 }
2442 ShouldNotReachHere();
2443 return nullptr;
2444 }
2445
2446 bool Matcher::is_reg2reg_move(MachNode* m) {
2447 return false;
2448 }
2449
2450 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2451 return false;
2452 }
2453
2454 bool Matcher::is_generic_vector(MachOper* opnd) {
2455 return opnd->opcode() == VREG;
2456 }
2457
2458 #ifdef ASSERT
2459 // Return whether or not this register is ever used as an argument.
2460 bool Matcher::can_be_java_arg(int reg)
2461 {
2462 return
2463 reg == R0_num || reg == R0_H_num ||
2464 reg == R1_num || reg == R1_H_num ||
2465 reg == R2_num || reg == R2_H_num ||
2466 reg == R3_num || reg == R3_H_num ||
2467 reg == R4_num || reg == R4_H_num ||
2468 reg == R5_num || reg == R5_H_num ||
2469 reg == R6_num || reg == R6_H_num ||
2470 reg == R7_num || reg == R7_H_num ||
2471 reg == V0_num || reg == V0_H_num ||
2472 reg == V1_num || reg == V1_H_num ||
2473 reg == V2_num || reg == V2_H_num ||
2474 reg == V3_num || reg == V3_H_num ||
2475 reg == V4_num || reg == V4_H_num ||
2476 reg == V5_num || reg == V5_H_num ||
2477 reg == V6_num || reg == V6_H_num ||
2478 reg == V7_num || reg == V7_H_num;
2479 }
2480 #endif
2481
2482 uint Matcher::int_pressure_limit()
2483 {
2484 // JDK-8183543: When taking the number of available registers as int
2485 // register pressure threshold, the jtreg test:
2486 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2487 // failed due to C2 compilation failure with
2488 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2489 //
2490 // A derived pointer is live at CallNode and then is flagged by RA
2491 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2492 // derived pointers and lastly fail to spill after reaching maximum
2493 // number of iterations. Lowering the default pressure threshold to
2494 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2495 // a high register pressure area of the code so that split_DEF can
2496 // generate DefinitionSpillCopy for the derived pointer.
2497 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2498 if (!PreserveFramePointer) {
2499 // When PreserveFramePointer is off, frame pointer is allocatable,
2500 // but different from other SOC registers, it is excluded from
2501 // fatproj's mask because its save type is No-Save. Decrease 1 to
2502 // ensure high pressure at fatproj when PreserveFramePointer is off.
2503 // See check_pressure_at_fatproj().
2504 default_int_pressure_threshold--;
2505 }
2506 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2507 }
2508
2509 uint Matcher::float_pressure_limit()
2510 {
2511 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2512 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2513 }
2514
2515 const RegMask& Matcher::divI_proj_mask() {
2516 ShouldNotReachHere();
2517 return RegMask::EMPTY;
2518 }
2519
2520 // Register for MODI projection of divmodI.
2521 const RegMask& Matcher::modI_proj_mask() {
2522 ShouldNotReachHere();
2523 return RegMask::EMPTY;
2524 }
2525
2526 // Register for DIVL projection of divmodL.
2527 const RegMask& Matcher::divL_proj_mask() {
2528 ShouldNotReachHere();
2529 return RegMask::EMPTY;
2530 }
2531
2532 // Register for MODL projection of divmodL.
2533 const RegMask& Matcher::modL_proj_mask() {
2534 ShouldNotReachHere();
2535 return RegMask::EMPTY;
2536 }
2537
2538 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2539 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2540 Node* u = addp->fast_out(i);
2541 if (u->is_LoadStore()) {
2542 // On AArch64, LoadStoreNodes (i.e. compare and swap
2543 // instructions) only take register indirect as an operand, so
2544 // any attempt to use an AddPNode as an input to a LoadStoreNode
2545 // must fail.
2546 return false;
2547 }
2548 if (u->is_Mem()) {
2549 int opsize = u->as_Mem()->memory_size();
2550 assert(opsize > 0, "unexpected memory operand size");
2551 if (u->as_Mem()->memory_size() != (1<<shift)) {
2552 return false;
2553 }
2554 }
2555 }
2556 return true;
2557 }
2558
2559 // Convert BoolTest condition to Assembler condition.
2560 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2561 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2562 Assembler::Condition result;
2563 switch(cond) {
2564 case BoolTest::eq:
2565 result = Assembler::EQ; break;
2566 case BoolTest::ne:
2567 result = Assembler::NE; break;
2568 case BoolTest::le:
2569 result = Assembler::LE; break;
2570 case BoolTest::ge:
2571 result = Assembler::GE; break;
2572 case BoolTest::lt:
2573 result = Assembler::LT; break;
2574 case BoolTest::gt:
2575 result = Assembler::GT; break;
2576 case BoolTest::ule:
2577 result = Assembler::LS; break;
2578 case BoolTest::uge:
2579 result = Assembler::HS; break;
2580 case BoolTest::ult:
2581 result = Assembler::LO; break;
2582 case BoolTest::ugt:
2583 result = Assembler::HI; break;
2584 case BoolTest::overflow:
2585 result = Assembler::VS; break;
2586 case BoolTest::no_overflow:
2587 result = Assembler::VC; break;
2588 default:
2589 ShouldNotReachHere();
2590 return Assembler::Condition(-1);
2591 }
2592
2593 // Check conversion
2594 if (cond & BoolTest::unsigned_compare) {
2595 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2596 } else {
2597 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2598 }
2599
2600 return result;
2601 }
2602
2603 // Binary src (Replicate con)
2604 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2605 if (n == nullptr || m == nullptr) {
2606 return false;
2607 }
2608
2609 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2610 return false;
2611 }
2612
2613 Node* imm_node = m->in(1);
2614 if (!imm_node->is_Con()) {
2615 return false;
2616 }
2617
2618 const Type* t = imm_node->bottom_type();
2619 if (!(t->isa_int() || t->isa_long())) {
2620 return false;
2621 }
2622
2623 switch (n->Opcode()) {
2624 case Op_AndV:
2625 case Op_OrV:
2626 case Op_XorV: {
2627 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2628 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2629 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2630 }
2631 case Op_AddVB:
2632 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2633 case Op_AddVS:
2634 case Op_AddVI:
2635 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2636 case Op_AddVL:
2637 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2638 default:
2639 return false;
2640 }
2641 }
2642
2643 // (XorV src (Replicate m1))
2644 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2645 if (n != nullptr && m != nullptr) {
2646 return n->Opcode() == Op_XorV &&
2647 VectorNode::is_all_ones_vector(m);
2648 }
2649 return false;
2650 }
2651
2652 // Returns true if (n, m) matches "(XorVMask vm2 (MaskAll m1))" and that XorVMask
2653 // is used only by an AndVMask. In that case, cloning m (the MaskAll) lets the
2654 // matcher avoid sharing the MaskAll node and subsume the pattern into rule:
2655 // "(AndVMask vm1 (XorVMask vm2 (MaskAll m1)))".
2656 //
2657 // Limitation: the "andNot" rule still cannot be matched if "m" has other
2658 // uses outside this pattern.
2659 static bool is_vector_mask_not_operand_in_andnot_pattern(Node* n, Node* m) {
2660 if (n == nullptr || m == nullptr) {
2661 return false;
2662 }
2663
2664 if (VectorNode::is_all_ones_vector(m) &&
2665 n->Opcode() == Op_XorVMask &&
2666 n->outcnt() == 1 &&
2667 n->unique_out()->Opcode() == Op_AndVMask) {
2668 // If another input of the AndVMask is also a mask-not pattern that would
2669 // qualify for the `maskAll` cloning, do not clone the "maskAll" here,
2670 // because the match rule can only consume one such pattern.
2671 Node* use = n->unique_out();
2672 Node* other_input = use->in(1) == n ? use->in(2) : use->in(1);
2673 return !VectorNode::is_vectormask_bitwise_not_pattern(other_input);
2674 }
2675 return false;
2676 }
2677
2678 // Should the matcher clone input 'm' of node 'n'?
2679 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2680 if (is_vshift_con_pattern(n, m) ||
2681 is_vector_bitwise_not_pattern(n, m) ||
2682 is_vector_mask_not_operand_in_andnot_pattern(n, m) ||
2683 is_valid_sve_arith_imm_pattern(n, m) ||
2684 is_encode_and_store_pattern(n, m)) {
2685 mstack.push(m, Visit);
2686 return true;
2687 }
2688 return false;
2689 }
2690
2691 // Should the Matcher clone shifts on addressing modes, expecting them
2692 // to be subsumed into complex addressing expressions or compute them
2693 // into registers?
2694 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2695
2696 // Loads and stores with indirect memory input (e.g., volatile loads and
2697 // stores) do not subsume the input into complex addressing expressions. If
2698 // the addressing expression is input to at least one such load or store, do
2699 // not clone the addressing expression. Query needs_acquiring_load and
2700 // needs_releasing_store as a proxy for indirect memory input, as it is not
2701 // possible to directly query for indirect memory input at this stage.
2702 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2703 Node* n = m->fast_out(i);
2704 if (n->is_Load() && needs_acquiring_load(n)) {
2705 return false;
2706 }
2707 if (n->is_Store() && needs_releasing_store(n)) {
2708 return false;
2709 }
2710 }
2711
2712 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2713 return true;
2714 }
2715
2716 Node *off = m->in(AddPNode::Offset);
2717 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2718 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2719 // Are there other uses besides address expressions?
2720 !is_visited(off)) {
2721 address_visited.set(off->_idx); // Flag as address_visited
2722 mstack.push(off->in(2), Visit);
2723 Node *conv = off->in(1);
2724 if (conv->Opcode() == Op_ConvI2L &&
2725 // Are there other uses besides address expressions?
2726 !is_visited(conv)) {
2727 address_visited.set(conv->_idx); // Flag as address_visited
2728 mstack.push(conv->in(1), Pre_Visit);
2729 } else {
2730 mstack.push(conv, Pre_Visit);
2731 }
2732 address_visited.test_set(m->_idx); // Flag as address_visited
2733 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2734 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2735 return true;
2736 } else if (off->Opcode() == Op_ConvI2L &&
2737 // Are there other uses besides address expressions?
2738 !is_visited(off)) {
2739 address_visited.test_set(m->_idx); // Flag as address_visited
2740 address_visited.set(off->_idx); // Flag as address_visited
2741 mstack.push(off->in(1), Pre_Visit);
2742 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2743 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2744 return true;
2745 }
2746 return false;
2747 }
2748
2749 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2750 { \
2751 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2752 guarantee(DISP == 0, "mode not permitted for volatile"); \
2753 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2754 __ INSN(REG, as_Register(BASE)); \
2755 }
2756
2757
2758 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2759 {
2760 Address::extend scale;
2761
2762 // Hooboy, this is fugly. We need a way to communicate to the
2763 // encoder that the index needs to be sign extended, so we have to
2764 // enumerate all the cases.
2765 switch (opcode) {
2766 case INDINDEXSCALEDI2L:
2767 case INDINDEXSCALEDI2LN:
2768 case INDINDEXI2L:
2769 case INDINDEXI2LN:
2770 scale = Address::sxtw(size);
2771 break;
2772 default:
2773 scale = Address::lsl(size);
2774 }
2775
2776 if (index == -1) {
2777 return Address(base, disp);
2778 } else {
2779 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2780 return Address(base, as_Register(index), scale);
2781 }
2782 }
2783
2784
2785 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2786 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2787 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2788 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2789 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2790
2791 // Used for all non-volatile memory accesses. The use of
2792 // $mem->opcode() to discover whether this pattern uses sign-extended
2793 // offsets is something of a kludge.
2794 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2795 Register reg, int opcode,
2796 Register base, int index, int scale, int disp,
2797 int size_in_memory)
2798 {
2799 Address addr = mem2address(opcode, base, index, scale, disp);
2800 if (addr.getMode() == Address::base_plus_offset) {
2801 /* Fix up any out-of-range offsets. */
2802 assert_different_registers(rscratch1, base);
2803 assert_different_registers(rscratch1, reg);
2804 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2805 }
2806 (masm->*insn)(reg, addr);
2807 }
2808
2809 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2810 FloatRegister reg, int opcode,
2811 Register base, int index, int size, int disp,
2812 int size_in_memory)
2813 {
2814 Address::extend scale;
2815
2816 switch (opcode) {
2817 case INDINDEXSCALEDI2L:
2818 case INDINDEXSCALEDI2LN:
2819 scale = Address::sxtw(size);
2820 break;
2821 default:
2822 scale = Address::lsl(size);
2823 }
2824
2825 if (index == -1) {
2826 // Fix up any out-of-range offsets.
2827 assert_different_registers(rscratch1, base);
2828 Address addr = Address(base, disp);
2829 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2830 (masm->*insn)(reg, addr);
2831 } else {
2832 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2833 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2834 }
2835 }
2836
2837 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2838 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2839 int opcode, Register base, int index, int size, int disp)
2840 {
2841 if (index == -1) {
2842 (masm->*insn)(reg, T, Address(base, disp));
2843 } else {
2844 assert(disp == 0, "unsupported address mode");
2845 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2846 }
2847 }
2848
2849 %}
2850
2851
2852
2853 //----------ENCODING BLOCK-----------------------------------------------------
2854 // This block specifies the encoding classes used by the compiler to
2855 // output byte streams. Encoding classes are parameterized macros
2856 // used by Machine Instruction Nodes in order to generate the bit
2857 // encoding of the instruction. Operands specify their base encoding
2858 // interface with the interface keyword. There are currently
2859 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2860 // COND_INTER. REG_INTER causes an operand to generate a function
2861 // which returns its register number when queried. CONST_INTER causes
2862 // an operand to generate a function which returns the value of the
2863 // constant when queried. MEMORY_INTER causes an operand to generate
2864 // four functions which return the Base Register, the Index Register,
2865 // the Scale Value, and the Offset Value of the operand when queried.
2866 // COND_INTER causes an operand to generate six functions which return
2867 // the encoding code (ie - encoding bits for the instruction)
2868 // associated with each basic boolean condition for a conditional
2869 // instruction.
2870 //
2871 // Instructions specify two basic values for encoding. Again, a
2872 // function is available to check if the constant displacement is an
2873 // oop. They use the ins_encode keyword to specify their encoding
2874 // classes (which must be a sequence of enc_class names, and their
2875 // parameters, specified in the encoding block), and they use the
2876 // opcode keyword to specify, in order, their primary, secondary, and
2877 // tertiary opcode. Only the opcode sections which a particular
2878 // instruction needs for encoding need to be specified.
2879 encode %{
2880 // Build emit functions for each basic byte or larger field in the
2881 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2882 // from C++ code in the enc_class source block. Emit functions will
2883 // live in the main source block for now. In future, we can
2884 // generalize this by adding a syntax that specifies the sizes of
2885 // fields in an order, so that the adlc can build the emit functions
2886 // automagically
2887
2888 // catch all for unimplemented encodings
2889 enc_class enc_unimplemented %{
2890 __ unimplemented("C2 catch all");
2891 %}
2892
2893 // BEGIN Non-volatile memory access
2894
2895 // This encoding class is generated automatically from ad_encode.m4.
2896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2897 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2898 Register dst_reg = as_Register($dst$$reg);
2899 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2900 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2901 %}
2902
2903 // This encoding class is generated automatically from ad_encode.m4.
2904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2905 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2906 Register dst_reg = as_Register($dst$$reg);
2907 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2908 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2909 %}
2910
2911 // This encoding class is generated automatically from ad_encode.m4.
2912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2913 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2914 Register dst_reg = as_Register($dst$$reg);
2915 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2916 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2917 %}
2918
2919 // This encoding class is generated automatically from ad_encode.m4.
2920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2921 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2922 Register dst_reg = as_Register($dst$$reg);
2923 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2924 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2925 %}
2926
2927 // This encoding class is generated automatically from ad_encode.m4.
2928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2929 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2930 Register dst_reg = as_Register($dst$$reg);
2931 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2932 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2933 %}
2934
2935 // This encoding class is generated automatically from ad_encode.m4.
2936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2937 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2938 Register dst_reg = as_Register($dst$$reg);
2939 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2940 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2941 %}
2942
2943 // This encoding class is generated automatically from ad_encode.m4.
2944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2945 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2946 Register dst_reg = as_Register($dst$$reg);
2947 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2948 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2949 %}
2950
2951 // This encoding class is generated automatically from ad_encode.m4.
2952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2953 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2954 Register dst_reg = as_Register($dst$$reg);
2955 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2956 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2957 %}
2958
2959 // This encoding class is generated automatically from ad_encode.m4.
2960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2961 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2962 Register dst_reg = as_Register($dst$$reg);
2963 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2964 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2965 %}
2966
2967 // This encoding class is generated automatically from ad_encode.m4.
2968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2969 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2970 Register dst_reg = as_Register($dst$$reg);
2971 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2972 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2973 %}
2974
2975 // This encoding class is generated automatically from ad_encode.m4.
2976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2977 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2978 Register dst_reg = as_Register($dst$$reg);
2979 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2980 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2981 %}
2982
2983 // This encoding class is generated automatically from ad_encode.m4.
2984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2985 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2986 Register dst_reg = as_Register($dst$$reg);
2987 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2988 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2989 %}
2990
2991 // This encoding class is generated automatically from ad_encode.m4.
2992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2993 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2994 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2995 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2996 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2997 %}
2998
2999 // This encoding class is generated automatically from ad_encode.m4.
3000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3001 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
3002 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3003 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
3004 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3005 %}
3006
3007 // This encoding class is generated automatically from ad_encode.m4.
3008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3009 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3010 Register src_reg = as_Register($src$$reg);
3011 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3012 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3013 %}
3014
3015 // This encoding class is generated automatically from ad_encode.m4.
3016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3017 enc_class aarch64_enc_strb0(memory1 mem) %{
3018 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3019 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3020 %}
3021
3022 // This encoding class is generated automatically from ad_encode.m4.
3023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3024 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3025 Register src_reg = as_Register($src$$reg);
3026 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3027 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3028 %}
3029
3030 // This encoding class is generated automatically from ad_encode.m4.
3031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3032 enc_class aarch64_enc_strh0(memory2 mem) %{
3033 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3034 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3035 %}
3036
3037 // This encoding class is generated automatically from ad_encode.m4.
3038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3039 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3040 Register src_reg = as_Register($src$$reg);
3041 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3042 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3043 %}
3044
3045 // This encoding class is generated automatically from ad_encode.m4.
3046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3047 enc_class aarch64_enc_strw0(memory4 mem) %{
3048 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3049 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3050 %}
3051
3052 // This encoding class is generated automatically from ad_encode.m4.
3053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3054 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3055 Register src_reg = as_Register($src$$reg);
3056 // we sometimes get asked to store the stack pointer into the
3057 // current thread -- we cannot do that directly on AArch64
3058 if (src_reg == r31_sp) {
3059 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3060 __ mov(rscratch2, sp);
3061 src_reg = rscratch2;
3062 }
3063 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3064 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3065 %}
3066
3067 // This encoding class is generated automatically from ad_encode.m4.
3068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3069 enc_class aarch64_enc_str0(memory8 mem) %{
3070 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3071 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3072 %}
3073
3074 // This encoding class is generated automatically from ad_encode.m4.
3075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3076 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3077 FloatRegister src_reg = as_FloatRegister($src$$reg);
3078 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3079 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3080 %}
3081
3082 // This encoding class is generated automatically from ad_encode.m4.
3083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3084 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3085 FloatRegister src_reg = as_FloatRegister($src$$reg);
3086 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3087 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3088 %}
3089
3090 // This encoding class is generated automatically from ad_encode.m4.
3091 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3092 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3093 __ membar(Assembler::StoreStore);
3094 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3095 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3096 %}
3097
3098 // END Non-volatile memory access
3099
3100 // Vector loads and stores
3101 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3102 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3103 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3104 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3105 %}
3106
3107 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3108 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3109 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3110 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3111 %}
3112
3113 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3114 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3115 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3116 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3117 %}
3118
3119 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3120 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3121 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3122 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3123 %}
3124
3125 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3126 FloatRegister src_reg = as_FloatRegister($src$$reg);
3127 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3128 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3129 %}
3130
3131 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3132 FloatRegister src_reg = as_FloatRegister($src$$reg);
3133 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3134 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3135 %}
3136
3137 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3138 FloatRegister src_reg = as_FloatRegister($src$$reg);
3139 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3140 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3141 %}
3142
3143 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3144 FloatRegister src_reg = as_FloatRegister($src$$reg);
3145 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3146 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3147 %}
3148
3149 // volatile loads and stores
3150
3151 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3152 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, stlrb);
3154 %}
3155
3156 enc_class aarch64_enc_stlrb0(memory mem) %{
3157 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3158 rscratch1, stlrb);
3159 %}
3160
3161 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3162 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3163 rscratch1, stlrh);
3164 %}
3165
3166 enc_class aarch64_enc_stlrh0(memory mem) %{
3167 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3168 rscratch1, stlrh);
3169 %}
3170
3171 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3172 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3173 rscratch1, stlrw);
3174 %}
3175
3176 enc_class aarch64_enc_stlrw0(memory mem) %{
3177 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3178 rscratch1, stlrw);
3179 %}
3180
3181 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3182 Register dst_reg = as_Register($dst$$reg);
3183 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3184 rscratch1, ldarb);
3185 __ sxtbw(dst_reg, dst_reg);
3186 %}
3187
3188 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3189 Register dst_reg = as_Register($dst$$reg);
3190 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3191 rscratch1, ldarb);
3192 __ sxtb(dst_reg, dst_reg);
3193 %}
3194
3195 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3196 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3197 rscratch1, ldarb);
3198 %}
3199
3200 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3201 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3202 rscratch1, ldarb);
3203 %}
3204
3205 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3206 Register dst_reg = as_Register($dst$$reg);
3207 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3208 rscratch1, ldarh);
3209 __ sxthw(dst_reg, dst_reg);
3210 %}
3211
3212 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3213 Register dst_reg = as_Register($dst$$reg);
3214 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3215 rscratch1, ldarh);
3216 __ sxth(dst_reg, dst_reg);
3217 %}
3218
3219 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3220 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3221 rscratch1, ldarh);
3222 %}
3223
3224 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3225 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, ldarh);
3227 %}
3228
3229 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3230 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3231 rscratch1, ldarw);
3232 %}
3233
3234 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3235 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3236 rscratch1, ldarw);
3237 %}
3238
3239 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3240 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3241 rscratch1, ldar);
3242 %}
3243
3244 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3245 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3246 rscratch1, ldarw);
3247 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3248 %}
3249
3250 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3251 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3252 rscratch1, ldar);
3253 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3254 %}
3255
3256 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3257 Register src_reg = as_Register($src$$reg);
3258 // we sometimes get asked to store the stack pointer into the
3259 // current thread -- we cannot do that directly on AArch64
3260 if (src_reg == r31_sp) {
3261 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3262 __ mov(rscratch2, sp);
3263 src_reg = rscratch2;
3264 }
3265 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3266 rscratch1, stlr);
3267 %}
3268
3269 enc_class aarch64_enc_stlr0(memory mem) %{
3270 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3271 rscratch1, stlr);
3272 %}
3273
3274 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3275 {
3276 FloatRegister src_reg = as_FloatRegister($src$$reg);
3277 __ fmovs(rscratch2, src_reg);
3278 }
3279 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3280 rscratch1, stlrw);
3281 %}
3282
3283 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3284 {
3285 FloatRegister src_reg = as_FloatRegister($src$$reg);
3286 __ fmovd(rscratch2, src_reg);
3287 }
3288 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3289 rscratch1, stlr);
3290 %}
3291
3292 // synchronized read/update encodings
3293
3294 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3295 Register dst_reg = as_Register($dst$$reg);
3296 Register base = as_Register($mem$$base);
3297 int index = $mem$$index;
3298 int scale = $mem$$scale;
3299 int disp = $mem$$disp;
3300 if (index == -1) {
3301 if (disp != 0) {
3302 __ lea(rscratch1, Address(base, disp));
3303 __ ldaxr(dst_reg, rscratch1);
3304 } else {
3305 // TODO
3306 // should we ever get anything other than this case?
3307 __ ldaxr(dst_reg, base);
3308 }
3309 } else {
3310 Register index_reg = as_Register(index);
3311 if (disp == 0) {
3312 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3313 __ ldaxr(dst_reg, rscratch1);
3314 } else {
3315 __ lea(rscratch1, Address(base, disp));
3316 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3317 __ ldaxr(dst_reg, rscratch1);
3318 }
3319 }
3320 %}
3321
3322 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3323 Register src_reg = as_Register($src$$reg);
3324 Register base = as_Register($mem$$base);
3325 int index = $mem$$index;
3326 int scale = $mem$$scale;
3327 int disp = $mem$$disp;
3328 if (index == -1) {
3329 if (disp != 0) {
3330 __ lea(rscratch2, Address(base, disp));
3331 __ stlxr(rscratch1, src_reg, rscratch2);
3332 } else {
3333 // TODO
3334 // should we ever get anything other than this case?
3335 __ stlxr(rscratch1, src_reg, base);
3336 }
3337 } else {
3338 Register index_reg = as_Register(index);
3339 if (disp == 0) {
3340 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3341 __ stlxr(rscratch1, src_reg, rscratch2);
3342 } else {
3343 __ lea(rscratch2, Address(base, disp));
3344 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3345 __ stlxr(rscratch1, src_reg, rscratch2);
3346 }
3347 }
3348 __ cmpw(rscratch1, zr);
3349 %}
3350
3351 // prefetch encodings
3352
3353 enc_class aarch64_enc_prefetchw(memory mem) %{
3354 Register base = as_Register($mem$$base);
3355 int index = $mem$$index;
3356 int scale = $mem$$scale;
3357 int disp = $mem$$disp;
3358 if (index == -1) {
3359 // Fix up any out-of-range offsets.
3360 assert_different_registers(rscratch1, base);
3361 Address addr = Address(base, disp);
3362 addr = __ legitimize_address(addr, 8, rscratch1);
3363 __ prfm(addr, PSTL1KEEP);
3364 } else {
3365 Register index_reg = as_Register(index);
3366 if (disp == 0) {
3367 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3368 } else {
3369 __ lea(rscratch1, Address(base, disp));
3370 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3371 }
3372 }
3373 %}
3374
3375 // mov encodings
3376
3377 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3378 uint32_t con = (uint32_t)$src$$constant;
3379 Register dst_reg = as_Register($dst$$reg);
3380 if (con == 0) {
3381 __ movw(dst_reg, zr);
3382 } else {
3383 __ movw(dst_reg, con);
3384 }
3385 %}
3386
3387 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3388 Register dst_reg = as_Register($dst$$reg);
3389 uint64_t con = (uint64_t)$src$$constant;
3390 if (con == 0) {
3391 __ mov(dst_reg, zr);
3392 } else {
3393 __ mov(dst_reg, con);
3394 }
3395 %}
3396
3397 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3398 Register dst_reg = as_Register($dst$$reg);
3399 address con = (address)$src$$constant;
3400 if (con == nullptr || con == (address)1) {
3401 ShouldNotReachHere();
3402 } else {
3403 relocInfo::relocType rtype = $src->constant_reloc();
3404 if (rtype == relocInfo::oop_type) {
3405 __ movoop(dst_reg, (jobject)con);
3406 } else if (rtype == relocInfo::metadata_type) {
3407 __ mov_metadata(dst_reg, (Metadata*)con);
3408 } else {
3409 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3410 // load fake address constants using a normal move
3411 if (! __ is_valid_AArch64_address(con) ||
3412 con < (address)(uintptr_t)os::vm_page_size() ||
3413 rtype == relocInfo::none) {
3414 __ mov(dst_reg, con);
3415 } else {
3416 // use shorter adrp/add sequence for external_word relocation
3417 uint64_t offset;
3418 __ adrp(dst_reg, Address(con, rtype), offset);
3419 __ add(dst_reg, dst_reg, offset);
3420 }
3421 }
3422 }
3423 %}
3424
3425 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3426 Register dst_reg = as_Register($dst$$reg);
3427 __ mov(dst_reg, zr);
3428 %}
3429
3430 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3431 Register dst_reg = as_Register($dst$$reg);
3432 __ mov(dst_reg, (uint64_t)1);
3433 %}
3434
3435 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3436 Register dst_reg = as_Register($dst$$reg);
3437 address con = (address)$src$$constant;
3438 if (con == nullptr) {
3439 ShouldNotReachHere();
3440 } else {
3441 relocInfo::relocType rtype = $src->constant_reloc();
3442 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3443 __ set_narrow_oop(dst_reg, (jobject)con);
3444 }
3445 %}
3446
3447 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3448 Register dst_reg = as_Register($dst$$reg);
3449 __ mov(dst_reg, zr);
3450 %}
3451
3452 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3453 Register dst_reg = as_Register($dst$$reg);
3454 address con = (address)$src$$constant;
3455 if (con == nullptr) {
3456 ShouldNotReachHere();
3457 } else {
3458 relocInfo::relocType rtype = $src->constant_reloc();
3459 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3460 __ set_narrow_klass(dst_reg, (Klass *)con);
3461 }
3462 %}
3463
3464 // arithmetic encodings
3465
3466 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3467 Register dst_reg = as_Register($dst$$reg);
3468 Register src_reg = as_Register($src1$$reg);
3469 int32_t con = (int32_t)$src2$$constant;
3470 // add has primary == 0, subtract has primary == 1
3471 if ($primary) { con = -con; }
3472 if (con < 0) {
3473 __ subw(dst_reg, src_reg, -con);
3474 } else {
3475 __ addw(dst_reg, src_reg, con);
3476 }
3477 %}
3478
3479 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3480 Register dst_reg = as_Register($dst$$reg);
3481 Register src_reg = as_Register($src1$$reg);
3482 int32_t con = (int32_t)$src2$$constant;
3483 // add has primary == 0, subtract has primary == 1
3484 if ($primary) { con = -con; }
3485 if (con < 0) {
3486 __ sub(dst_reg, src_reg, -con);
3487 } else {
3488 __ add(dst_reg, src_reg, con);
3489 }
3490 %}
3491
3492 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3493 Register dst_reg = as_Register($dst$$reg);
3494 Register src1_reg = as_Register($src1$$reg);
3495 Register src2_reg = as_Register($src2$$reg);
3496 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3497 %}
3498
3499 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3500 Register dst_reg = as_Register($dst$$reg);
3501 Register src1_reg = as_Register($src1$$reg);
3502 Register src2_reg = as_Register($src2$$reg);
3503 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3504 %}
3505
3506 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3507 Register dst_reg = as_Register($dst$$reg);
3508 Register src1_reg = as_Register($src1$$reg);
3509 Register src2_reg = as_Register($src2$$reg);
3510 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3511 %}
3512
3513 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3514 Register dst_reg = as_Register($dst$$reg);
3515 Register src1_reg = as_Register($src1$$reg);
3516 Register src2_reg = as_Register($src2$$reg);
3517 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3518 %}
3519
3520 // compare instruction encodings
3521
3522 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3523 Register reg1 = as_Register($src1$$reg);
3524 Register reg2 = as_Register($src2$$reg);
3525 __ cmpw(reg1, reg2);
3526 %}
3527
3528 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3529 Register reg = as_Register($src1$$reg);
3530 int32_t val = $src2$$constant;
3531 if (val >= 0) {
3532 __ subsw(zr, reg, val);
3533 } else {
3534 __ addsw(zr, reg, -val);
3535 }
3536 %}
3537
3538 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3539 Register reg1 = as_Register($src1$$reg);
3540 uint32_t val = (uint32_t)$src2$$constant;
3541 __ movw(rscratch1, val);
3542 __ cmpw(reg1, rscratch1);
3543 %}
3544
3545 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3546 Register reg1 = as_Register($src1$$reg);
3547 Register reg2 = as_Register($src2$$reg);
3548 __ cmp(reg1, reg2);
3549 %}
3550
3551 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3552 Register reg = as_Register($src1$$reg);
3553 int64_t val = $src2$$constant;
3554 if (val >= 0) {
3555 __ subs(zr, reg, val);
3556 } else if (val != -val) {
3557 __ adds(zr, reg, -val);
3558 } else {
3559 // aargh, Long.MIN_VALUE is a special case
3560 __ orr(rscratch1, zr, (uint64_t)val);
3561 __ subs(zr, reg, rscratch1);
3562 }
3563 %}
3564
3565 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3566 Register reg1 = as_Register($src1$$reg);
3567 uint64_t val = (uint64_t)$src2$$constant;
3568 __ mov(rscratch1, val);
3569 __ cmp(reg1, rscratch1);
3570 %}
3571
3572 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3573 Register reg1 = as_Register($src1$$reg);
3574 Register reg2 = as_Register($src2$$reg);
3575 __ cmp(reg1, reg2);
3576 %}
3577
3578 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3579 Register reg1 = as_Register($src1$$reg);
3580 Register reg2 = as_Register($src2$$reg);
3581 __ cmpw(reg1, reg2);
3582 %}
3583
3584 enc_class aarch64_enc_testp(iRegP src) %{
3585 Register reg = as_Register($src$$reg);
3586 __ cmp(reg, zr);
3587 %}
3588
3589 enc_class aarch64_enc_testn(iRegN src) %{
3590 Register reg = as_Register($src$$reg);
3591 __ cmpw(reg, zr);
3592 %}
3593
3594 enc_class aarch64_enc_b(label lbl) %{
3595 Label *L = $lbl$$label;
3596 __ b(*L);
3597 %}
3598
3599 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3600 Label *L = $lbl$$label;
3601 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3602 %}
3603
3604 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3605 Label *L = $lbl$$label;
3606 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3607 %}
3608
3609 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3610 %{
3611 Register sub_reg = as_Register($sub$$reg);
3612 Register super_reg = as_Register($super$$reg);
3613 Register temp_reg = as_Register($temp$$reg);
3614 Register result_reg = as_Register($result$$reg);
3615
3616 Label miss;
3617 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3618 nullptr, &miss,
3619 /*set_cond_codes:*/ true);
3620 if ($primary) {
3621 __ mov(result_reg, zr);
3622 }
3623 __ bind(miss);
3624 %}
3625
3626 enc_class aarch64_enc_java_static_call(method meth) %{
3627 address addr = (address)$meth$$method;
3628 address call;
3629 if (!_method) {
3630 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3631 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3632 if (call == nullptr) {
3633 ciEnv::current()->record_failure("CodeCache is full");
3634 return;
3635 }
3636 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3637 // The NOP here is purely to ensure that eliding a call to
3638 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3639 __ nop();
3640 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3641 } else {
3642 int method_index = resolved_method_index(masm);
3643 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3644 : static_call_Relocation::spec(method_index);
3645 call = __ trampoline_call(Address(addr, rspec));
3646 if (call == nullptr) {
3647 ciEnv::current()->record_failure("CodeCache is full");
3648 return;
3649 }
3650 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3651 // Calls of the same statically bound method can share
3652 // a stub to the interpreter.
3653 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3654 } else {
3655 // Emit stub for static call
3656 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3657 if (stub == nullptr) {
3658 ciEnv::current()->record_failure("CodeCache is full");
3659 return;
3660 }
3661 }
3662 }
3663
3664 __ post_call_nop();
3665
3666 // Only non uncommon_trap calls need to reinitialize ptrue.
3667 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3668 __ reinitialize_ptrue();
3669 }
3670 %}
3671
3672 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3673 int method_index = resolved_method_index(masm);
3674 address call = __ ic_call((address)$meth$$method, method_index);
3675 if (call == nullptr) {
3676 ciEnv::current()->record_failure("CodeCache is full");
3677 return;
3678 }
3679 __ post_call_nop();
3680 if (Compile::current()->max_vector_size() > 0) {
3681 __ reinitialize_ptrue();
3682 }
3683 %}
3684
3685 enc_class aarch64_enc_call_epilog() %{
3686 if (VerifyStackAtCalls) {
3687 // Check that stack depth is unchanged: find majik cookie on stack
3688 __ call_Unimplemented();
3689 }
3690 %}
3691
3692 enc_class aarch64_enc_java_to_runtime(method meth) %{
3693 // some calls to generated routines (arraycopy code) are scheduled
3694 // by C2 as runtime calls. if so we can call them using a br (they
3695 // will be in a reachable segment) otherwise we have to use a blr
3696 // which loads the absolute address into a register.
3697 address entry = (address)$meth$$method;
3698 CodeBlob *cb = CodeCache::find_blob(entry);
3699 if (cb) {
3700 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3701 if (call == nullptr) {
3702 ciEnv::current()->record_failure("CodeCache is full");
3703 return;
3704 }
3705 __ post_call_nop();
3706 } else {
3707 Label retaddr;
3708 // Make the anchor frame walkable
3709 __ adr(rscratch2, retaddr);
3710 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3711 __ lea(rscratch1, RuntimeAddress(entry));
3712 __ blr(rscratch1);
3713 __ bind(retaddr);
3714 __ post_call_nop();
3715 }
3716 if (Compile::current()->max_vector_size() > 0) {
3717 __ reinitialize_ptrue();
3718 }
3719 %}
3720
3721 enc_class aarch64_enc_rethrow() %{
3722 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3723 %}
3724
3725 enc_class aarch64_enc_ret() %{
3726 #ifdef ASSERT
3727 if (Compile::current()->max_vector_size() > 0) {
3728 __ verify_ptrue();
3729 }
3730 #endif
3731 __ ret(lr);
3732 %}
3733
3734 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3735 Register target_reg = as_Register($jump_target$$reg);
3736 __ br(target_reg);
3737 %}
3738
3739 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3740 Register target_reg = as_Register($jump_target$$reg);
3741 // exception oop should be in r0
3742 // ret addr has been popped into lr
3743 // callee expects it in r3
3744 __ mov(r3, lr);
3745 __ br(target_reg);
3746 %}
3747
3748 %}
3749
3750 //----------FRAME--------------------------------------------------------------
3751 // Definition of frame structure and management information.
3752 //
3753 // S T A C K L A Y O U T Allocators stack-slot number
3754 // | (to get allocators register number
3755 // G Owned by | | v add OptoReg::stack0())
3756 // r CALLER | |
3757 // o | +--------+ pad to even-align allocators stack-slot
3758 // w V | pad0 | numbers; owned by CALLER
3759 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3760 // h ^ | in | 5
3761 // | | args | 4 Holes in incoming args owned by SELF
3762 // | | | | 3
3763 // | | +--------+
3764 // V | | old out| Empty on Intel, window on Sparc
3765 // | old |preserve| Must be even aligned.
3766 // | SP-+--------+----> Matcher::_old_SP, even aligned
3767 // | | in | 3 area for Intel ret address
3768 // Owned by |preserve| Empty on Sparc.
3769 // SELF +--------+
3770 // | | pad2 | 2 pad to align old SP
3771 // | +--------+ 1
3772 // | | locks | 0
3773 // | +--------+----> OptoReg::stack0(), even aligned
3774 // | | pad1 | 11 pad to align new SP
3775 // | +--------+
3776 // | | | 10
3777 // | | spills | 9 spills
3778 // V | | 8 (pad0 slot for callee)
3779 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3780 // ^ | out | 7
3781 // | | args | 6 Holes in outgoing args owned by CALLEE
3782 // Owned by +--------+
3783 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3784 // | new |preserve| Must be even-aligned.
3785 // | SP-+--------+----> Matcher::_new_SP, even aligned
3786 // | | |
3787 //
3788 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3789 // known from SELF's arguments and the Java calling convention.
3790 // Region 6-7 is determined per call site.
3791 // Note 2: If the calling convention leaves holes in the incoming argument
3792 // area, those holes are owned by SELF. Holes in the outgoing area
3793 // are owned by the CALLEE. Holes should not be necessary in the
3794 // incoming area, as the Java calling convention is completely under
3795 // the control of the AD file. Doubles can be sorted and packed to
3796 // avoid holes. Holes in the outgoing arguments may be necessary for
3797 // varargs C calling conventions.
3798 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3799 // even aligned with pad0 as needed.
3800 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3801 // (the latter is true on Intel but is it false on AArch64?)
3802 // region 6-11 is even aligned; it may be padded out more so that
3803 // the region from SP to FP meets the minimum stack alignment.
3804 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3805 // alignment. Region 11, pad1, may be dynamically extended so that
3806 // SP meets the minimum alignment.
3807
3808 frame %{
3809 // These three registers define part of the calling convention
3810 // between compiled code and the interpreter.
3811
3812 // Inline Cache Register or Method for I2C.
3813 inline_cache_reg(R12);
3814
3815 // Number of stack slots consumed by locking an object
3816 sync_stack_slots(2);
3817
3818 // Compiled code's Frame Pointer
3819 frame_pointer(R31);
3820
3821 // Stack alignment requirement
3822 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3823
3824 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3825 // for calls to C. Supports the var-args backing area for register parms.
3826 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3827
3828 // The after-PROLOG location of the return address. Location of
3829 // return address specifies a type (REG or STACK) and a number
3830 // representing the register number (i.e. - use a register name) or
3831 // stack slot.
3832 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3833 // Otherwise, it is above the locks and verification slot and alignment word
3834 // TODO this may well be correct but need to check why that - 2 is there
3835 // ppc port uses 0 but we definitely need to allow for fixed_slots
3836 // which folds in the space used for monitors
3837 return_addr(STACK - 2 +
3838 align_up((Compile::current()->in_preserve_stack_slots() +
3839 Compile::current()->fixed_slots()),
3840 stack_alignment_in_slots()));
3841
3842 // Location of compiled Java return values. Same as C for now.
3843 return_value
3844 %{
3845 // TODO do we allow ideal_reg == Op_RegN???
3846 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3847 "only return normal values");
3848
3849 static const int lo[Op_RegL + 1] = { // enum name
3850 0, // Op_Node
3851 0, // Op_Set
3852 R0_num, // Op_RegN
3853 R0_num, // Op_RegI
3854 R0_num, // Op_RegP
3855 V0_num, // Op_RegF
3856 V0_num, // Op_RegD
3857 R0_num // Op_RegL
3858 };
3859
3860 static const int hi[Op_RegL + 1] = { // enum name
3861 0, // Op_Node
3862 0, // Op_Set
3863 OptoReg::Bad, // Op_RegN
3864 OptoReg::Bad, // Op_RegI
3865 R0_H_num, // Op_RegP
3866 OptoReg::Bad, // Op_RegF
3867 V0_H_num, // Op_RegD
3868 R0_H_num // Op_RegL
3869 };
3870
3871 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3872 %}
3873 %}
3874
3875 //----------ATTRIBUTES---------------------------------------------------------
3876 //----------Operand Attributes-------------------------------------------------
3877 op_attrib op_cost(1); // Required cost attribute
3878
3879 //----------Instruction Attributes---------------------------------------------
3880 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3881 ins_attrib ins_size(32); // Required size attribute (in bits)
3882 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3883 // a non-matching short branch variant
3884 // of some long branch?
3885 ins_attrib ins_alignment(4); // Required alignment attribute (must
3886 // be a power of 2) specifies the
3887 // alignment that some part of the
3888 // instruction (not necessarily the
3889 // start) requires. If > 1, a
3890 // compute_padding() function must be
3891 // provided for the instruction
3892
3893 // Whether this node is expanded during code emission into a sequence of
3894 // instructions and the first instruction can perform an implicit null check.
3895 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3896
3897 //----------OPERANDS-----------------------------------------------------------
3898 // Operand definitions must precede instruction definitions for correct parsing
3899 // in the ADLC because operands constitute user defined types which are used in
3900 // instruction definitions.
3901
3902 //----------Simple Operands----------------------------------------------------
3903
3904 // Integer operands 32 bit
3905 // 32 bit immediate
3906 operand immI()
3907 %{
3908 match(ConI);
3909
3910 op_cost(0);
3911 format %{ %}
3912 interface(CONST_INTER);
3913 %}
3914
3915 // 32 bit zero
3916 operand immI0()
3917 %{
3918 predicate(n->get_int() == 0);
3919 match(ConI);
3920
3921 op_cost(0);
3922 format %{ %}
3923 interface(CONST_INTER);
3924 %}
3925
3926 // 32 bit unit increment
3927 operand immI_1()
3928 %{
3929 predicate(n->get_int() == 1);
3930 match(ConI);
3931
3932 op_cost(0);
3933 format %{ %}
3934 interface(CONST_INTER);
3935 %}
3936
3937 // 32 bit unit decrement
3938 operand immI_M1()
3939 %{
3940 predicate(n->get_int() == -1);
3941 match(ConI);
3942
3943 op_cost(0);
3944 format %{ %}
3945 interface(CONST_INTER);
3946 %}
3947
3948 // Shift values for add/sub extension shift
3949 operand immIExt()
3950 %{
3951 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3952 match(ConI);
3953
3954 op_cost(0);
3955 format %{ %}
3956 interface(CONST_INTER);
3957 %}
3958
3959 operand immI_gt_1()
3960 %{
3961 predicate(n->get_int() > 1);
3962 match(ConI);
3963
3964 op_cost(0);
3965 format %{ %}
3966 interface(CONST_INTER);
3967 %}
3968
3969 operand immI_le_4()
3970 %{
3971 predicate(n->get_int() <= 4);
3972 match(ConI);
3973
3974 op_cost(0);
3975 format %{ %}
3976 interface(CONST_INTER);
3977 %}
3978
3979 operand immI_16()
3980 %{
3981 predicate(n->get_int() == 16);
3982 match(ConI);
3983
3984 op_cost(0);
3985 format %{ %}
3986 interface(CONST_INTER);
3987 %}
3988
3989 operand immI_24()
3990 %{
3991 predicate(n->get_int() == 24);
3992 match(ConI);
3993
3994 op_cost(0);
3995 format %{ %}
3996 interface(CONST_INTER);
3997 %}
3998
3999 operand immI_32()
4000 %{
4001 predicate(n->get_int() == 32);
4002 match(ConI);
4003
4004 op_cost(0);
4005 format %{ %}
4006 interface(CONST_INTER);
4007 %}
4008
4009 operand immI_48()
4010 %{
4011 predicate(n->get_int() == 48);
4012 match(ConI);
4013
4014 op_cost(0);
4015 format %{ %}
4016 interface(CONST_INTER);
4017 %}
4018
4019 operand immI_56()
4020 %{
4021 predicate(n->get_int() == 56);
4022 match(ConI);
4023
4024 op_cost(0);
4025 format %{ %}
4026 interface(CONST_INTER);
4027 %}
4028
4029 operand immI_255()
4030 %{
4031 predicate(n->get_int() == 255);
4032 match(ConI);
4033
4034 op_cost(0);
4035 format %{ %}
4036 interface(CONST_INTER);
4037 %}
4038
4039 operand immI_65535()
4040 %{
4041 predicate(n->get_int() == 65535);
4042 match(ConI);
4043
4044 op_cost(0);
4045 format %{ %}
4046 interface(CONST_INTER);
4047 %}
4048
4049 operand immI_positive()
4050 %{
4051 predicate(n->get_int() > 0);
4052 match(ConI);
4053
4054 op_cost(0);
4055 format %{ %}
4056 interface(CONST_INTER);
4057 %}
4058
4059 // BoolTest condition for signed compare
4060 operand immI_cmp_cond()
4061 %{
4062 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4063 match(ConI);
4064
4065 op_cost(0);
4066 format %{ %}
4067 interface(CONST_INTER);
4068 %}
4069
4070 // BoolTest condition for unsigned compare
4071 operand immI_cmpU_cond()
4072 %{
4073 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4074 match(ConI);
4075
4076 op_cost(0);
4077 format %{ %}
4078 interface(CONST_INTER);
4079 %}
4080
4081 operand immL_255()
4082 %{
4083 predicate(n->get_long() == 255L);
4084 match(ConL);
4085
4086 op_cost(0);
4087 format %{ %}
4088 interface(CONST_INTER);
4089 %}
4090
4091 operand immL_65535()
4092 %{
4093 predicate(n->get_long() == 65535L);
4094 match(ConL);
4095
4096 op_cost(0);
4097 format %{ %}
4098 interface(CONST_INTER);
4099 %}
4100
4101 operand immL_4294967295()
4102 %{
4103 predicate(n->get_long() == 4294967295L);
4104 match(ConL);
4105
4106 op_cost(0);
4107 format %{ %}
4108 interface(CONST_INTER);
4109 %}
4110
4111 operand immL_bitmask()
4112 %{
4113 predicate((n->get_long() != 0)
4114 && ((n->get_long() & 0xc000000000000000l) == 0)
4115 && is_power_of_2(n->get_long() + 1));
4116 match(ConL);
4117
4118 op_cost(0);
4119 format %{ %}
4120 interface(CONST_INTER);
4121 %}
4122
4123 operand immI_bitmask()
4124 %{
4125 predicate((n->get_int() != 0)
4126 && ((n->get_int() & 0xc0000000) == 0)
4127 && is_power_of_2(n->get_int() + 1));
4128 match(ConI);
4129
4130 op_cost(0);
4131 format %{ %}
4132 interface(CONST_INTER);
4133 %}
4134
4135 operand immL_positive_bitmaskI()
4136 %{
4137 predicate((n->get_long() != 0)
4138 && ((julong)n->get_long() < 0x80000000ULL)
4139 && is_power_of_2(n->get_long() + 1));
4140 match(ConL);
4141
4142 op_cost(0);
4143 format %{ %}
4144 interface(CONST_INTER);
4145 %}
4146
4147 // Scale values for scaled offset addressing modes (up to long but not quad)
4148 operand immIScale()
4149 %{
4150 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4151 match(ConI);
4152
4153 op_cost(0);
4154 format %{ %}
4155 interface(CONST_INTER);
4156 %}
4157
4158 // 5 bit signed integer
4159 operand immI5()
4160 %{
4161 predicate(Assembler::is_simm(n->get_int(), 5));
4162 match(ConI);
4163
4164 op_cost(0);
4165 format %{ %}
4166 interface(CONST_INTER);
4167 %}
4168
4169 // 7 bit unsigned integer
4170 operand immIU7()
4171 %{
4172 predicate(Assembler::is_uimm(n->get_int(), 7));
4173 match(ConI);
4174
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 // Offset for scaled or unscaled immediate loads and stores
4181 operand immIOffset()
4182 %{
4183 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4184 match(ConI);
4185
4186 op_cost(0);
4187 format %{ %}
4188 interface(CONST_INTER);
4189 %}
4190
4191 operand immIOffset1()
4192 %{
4193 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4194 match(ConI);
4195
4196 op_cost(0);
4197 format %{ %}
4198 interface(CONST_INTER);
4199 %}
4200
4201 operand immIOffset2()
4202 %{
4203 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4204 match(ConI);
4205
4206 op_cost(0);
4207 format %{ %}
4208 interface(CONST_INTER);
4209 %}
4210
4211 operand immIOffset4()
4212 %{
4213 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4214 match(ConI);
4215
4216 op_cost(0);
4217 format %{ %}
4218 interface(CONST_INTER);
4219 %}
4220
4221 operand immIOffset8()
4222 %{
4223 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4224 match(ConI);
4225
4226 op_cost(0);
4227 format %{ %}
4228 interface(CONST_INTER);
4229 %}
4230
4231 operand immIOffset16()
4232 %{
4233 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4234 match(ConI);
4235
4236 op_cost(0);
4237 format %{ %}
4238 interface(CONST_INTER);
4239 %}
4240
4241 operand immLOffset()
4242 %{
4243 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4244 match(ConL);
4245
4246 op_cost(0);
4247 format %{ %}
4248 interface(CONST_INTER);
4249 %}
4250
4251 operand immLoffset1()
4252 %{
4253 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4254 match(ConL);
4255
4256 op_cost(0);
4257 format %{ %}
4258 interface(CONST_INTER);
4259 %}
4260
4261 operand immLoffset2()
4262 %{
4263 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4264 match(ConL);
4265
4266 op_cost(0);
4267 format %{ %}
4268 interface(CONST_INTER);
4269 %}
4270
4271 operand immLoffset4()
4272 %{
4273 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4274 match(ConL);
4275
4276 op_cost(0);
4277 format %{ %}
4278 interface(CONST_INTER);
4279 %}
4280
4281 operand immLoffset8()
4282 %{
4283 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4284 match(ConL);
4285
4286 op_cost(0);
4287 format %{ %}
4288 interface(CONST_INTER);
4289 %}
4290
4291 operand immLoffset16()
4292 %{
4293 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4294 match(ConL);
4295
4296 op_cost(0);
4297 format %{ %}
4298 interface(CONST_INTER);
4299 %}
4300
4301 // 5 bit signed long integer
4302 operand immL5()
4303 %{
4304 predicate(Assembler::is_simm(n->get_long(), 5));
4305 match(ConL);
4306
4307 op_cost(0);
4308 format %{ %}
4309 interface(CONST_INTER);
4310 %}
4311
4312 // 7 bit unsigned long integer
4313 operand immLU7()
4314 %{
4315 predicate(Assembler::is_uimm(n->get_long(), 7));
4316 match(ConL);
4317
4318 op_cost(0);
4319 format %{ %}
4320 interface(CONST_INTER);
4321 %}
4322
4323 // 8 bit signed value.
4324 operand immI8()
4325 %{
4326 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4327 match(ConI);
4328
4329 op_cost(0);
4330 format %{ %}
4331 interface(CONST_INTER);
4332 %}
4333
4334 // 8 bit signed value (simm8), or #simm8 LSL 8.
4335 operand immIDupV()
4336 %{
4337 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4338 match(ConI);
4339
4340 op_cost(0);
4341 format %{ %}
4342 interface(CONST_INTER);
4343 %}
4344
4345 // 8 bit signed value (simm8), or #simm8 LSL 8.
4346 operand immLDupV()
4347 %{
4348 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4349 match(ConL);
4350
4351 op_cost(0);
4352 format %{ %}
4353 interface(CONST_INTER);
4354 %}
4355
4356 // 8 bit signed value (simm8), or #simm8 LSL 8.
4357 operand immHDupV()
4358 %{
4359 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4360 match(ConH);
4361
4362 op_cost(0);
4363 format %{ %}
4364 interface(CONST_INTER);
4365 %}
4366
4367 // 8 bit integer valid for vector add sub immediate
4368 operand immBAddSubV()
4369 %{
4370 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4371 match(ConI);
4372
4373 op_cost(0);
4374 format %{ %}
4375 interface(CONST_INTER);
4376 %}
4377
4378 // 32 bit integer valid for add sub immediate
4379 operand immIAddSub()
4380 %{
4381 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4382 match(ConI);
4383 op_cost(0);
4384 format %{ %}
4385 interface(CONST_INTER);
4386 %}
4387
4388 // 32 bit integer valid for vector add sub immediate
4389 operand immIAddSubV()
4390 %{
4391 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4392 match(ConI);
4393
4394 op_cost(0);
4395 format %{ %}
4396 interface(CONST_INTER);
4397 %}
4398
4399 // 32 bit unsigned integer valid for logical immediate
4400
4401 operand immBLog()
4402 %{
4403 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4404 match(ConI);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 operand immSLog()
4412 %{
4413 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4414 match(ConI);
4415
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 operand immILog()
4422 %{
4423 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4424 match(ConI);
4425
4426 op_cost(0);
4427 format %{ %}
4428 interface(CONST_INTER);
4429 %}
4430
4431 // Integer operands 64 bit
4432 // 64 bit immediate
4433 operand immL()
4434 %{
4435 match(ConL);
4436
4437 op_cost(0);
4438 format %{ %}
4439 interface(CONST_INTER);
4440 %}
4441
4442 // 64 bit zero
4443 operand immL0()
4444 %{
4445 predicate(n->get_long() == 0);
4446 match(ConL);
4447
4448 op_cost(0);
4449 format %{ %}
4450 interface(CONST_INTER);
4451 %}
4452
4453 // 64 bit unit decrement
4454 operand immL_M1()
4455 %{
4456 predicate(n->get_long() == -1);
4457 match(ConL);
4458
4459 op_cost(0);
4460 format %{ %}
4461 interface(CONST_INTER);
4462 %}
4463
4464 // 64 bit integer valid for add sub immediate
4465 operand immLAddSub()
4466 %{
4467 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4468 match(ConL);
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 // 64 bit integer valid for addv subv immediate
4475 operand immLAddSubV()
4476 %{
4477 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4478 match(ConL);
4479
4480 op_cost(0);
4481 format %{ %}
4482 interface(CONST_INTER);
4483 %}
4484
4485 // 64 bit integer valid for logical immediate
4486 operand immLLog()
4487 %{
4488 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4489 match(ConL);
4490 op_cost(0);
4491 format %{ %}
4492 interface(CONST_INTER);
4493 %}
4494
4495 // Long Immediate: low 32-bit mask
4496 operand immL_32bits()
4497 %{
4498 predicate(n->get_long() == 0xFFFFFFFFL);
4499 match(ConL);
4500 op_cost(0);
4501 format %{ %}
4502 interface(CONST_INTER);
4503 %}
4504
4505 // Pointer operands
4506 // Pointer Immediate
4507 operand immP()
4508 %{
4509 match(ConP);
4510
4511 op_cost(0);
4512 format %{ %}
4513 interface(CONST_INTER);
4514 %}
4515
4516 // nullptr Pointer Immediate
4517 operand immP0()
4518 %{
4519 predicate(n->get_ptr() == 0);
4520 match(ConP);
4521
4522 op_cost(0);
4523 format %{ %}
4524 interface(CONST_INTER);
4525 %}
4526
4527 // Pointer Immediate One
4528 // this is used in object initialization (initial object header)
4529 operand immP_1()
4530 %{
4531 predicate(n->get_ptr() == 1);
4532 match(ConP);
4533
4534 op_cost(0);
4535 format %{ %}
4536 interface(CONST_INTER);
4537 %}
4538
4539 // AOT Runtime Constants Address
4540 operand immAOTRuntimeConstantsAddress()
4541 %{
4542 // Check if the address is in the range of AOT Runtime Constants
4543 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4544 match(ConP);
4545
4546 op_cost(0);
4547 format %{ %}
4548 interface(CONST_INTER);
4549 %}
4550
4551 // Float and Double operands
4552 // Double Immediate
4553 operand immD()
4554 %{
4555 match(ConD);
4556 op_cost(0);
4557 format %{ %}
4558 interface(CONST_INTER);
4559 %}
4560
4561 // Double Immediate: +0.0d
4562 operand immD0()
4563 %{
4564 predicate(jlong_cast(n->getd()) == 0);
4565 match(ConD);
4566
4567 op_cost(0);
4568 format %{ %}
4569 interface(CONST_INTER);
4570 %}
4571
4572 // constant 'double +0.0'.
4573 operand immDPacked()
4574 %{
4575 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4576 match(ConD);
4577 op_cost(0);
4578 format %{ %}
4579 interface(CONST_INTER);
4580 %}
4581
4582 // Float Immediate
4583 operand immF()
4584 %{
4585 match(ConF);
4586 op_cost(0);
4587 format %{ %}
4588 interface(CONST_INTER);
4589 %}
4590
4591 // Float Immediate: +0.0f.
4592 operand immF0()
4593 %{
4594 predicate(jint_cast(n->getf()) == 0);
4595 match(ConF);
4596
4597 op_cost(0);
4598 format %{ %}
4599 interface(CONST_INTER);
4600 %}
4601
4602 // Half Float (FP16) Immediate
4603 operand immH()
4604 %{
4605 match(ConH);
4606 op_cost(0);
4607 format %{ %}
4608 interface(CONST_INTER);
4609 %}
4610
4611 //
4612 operand immFPacked()
4613 %{
4614 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4615 match(ConF);
4616 op_cost(0);
4617 format %{ %}
4618 interface(CONST_INTER);
4619 %}
4620
4621 // Narrow pointer operands
4622 // Narrow Pointer Immediate
4623 operand immN()
4624 %{
4625 match(ConN);
4626
4627 op_cost(0);
4628 format %{ %}
4629 interface(CONST_INTER);
4630 %}
4631
4632 // Narrow nullptr Pointer Immediate
4633 operand immN0()
4634 %{
4635 predicate(n->get_narrowcon() == 0);
4636 match(ConN);
4637
4638 op_cost(0);
4639 format %{ %}
4640 interface(CONST_INTER);
4641 %}
4642
4643 operand immNKlass()
4644 %{
4645 match(ConNKlass);
4646
4647 op_cost(0);
4648 format %{ %}
4649 interface(CONST_INTER);
4650 %}
4651
4652 // Integer 32 bit Register Operands
4653 // Integer 32 bitRegister (excludes SP)
4654 operand iRegI()
4655 %{
4656 constraint(ALLOC_IN_RC(any_reg32));
4657 match(RegI);
4658 match(iRegINoSp);
4659 op_cost(0);
4660 format %{ %}
4661 interface(REG_INTER);
4662 %}
4663
4664 // Integer 32 bit Register not Special
4665 operand iRegINoSp()
4666 %{
4667 constraint(ALLOC_IN_RC(no_special_reg32));
4668 match(RegI);
4669 op_cost(0);
4670 format %{ %}
4671 interface(REG_INTER);
4672 %}
4673
4674 // Integer 64 bit Register Operands
4675 // Integer 64 bit Register (includes SP)
4676 operand iRegL()
4677 %{
4678 constraint(ALLOC_IN_RC(any_reg));
4679 match(RegL);
4680 match(iRegLNoSp);
4681 op_cost(0);
4682 format %{ %}
4683 interface(REG_INTER);
4684 %}
4685
4686 // Integer 64 bit Register not Special
4687 operand iRegLNoSp()
4688 %{
4689 constraint(ALLOC_IN_RC(no_special_reg));
4690 match(RegL);
4691 match(iRegL_R0);
4692 format %{ %}
4693 interface(REG_INTER);
4694 %}
4695
4696 // Pointer Register Operands
4697 // Pointer Register
4698 operand iRegP()
4699 %{
4700 constraint(ALLOC_IN_RC(ptr_reg));
4701 match(RegP);
4702 match(iRegPNoSp);
4703 match(iRegP_R0);
4704 //match(iRegP_R2);
4705 //match(iRegP_R4);
4706 match(iRegP_R5);
4707 match(thread_RegP);
4708 op_cost(0);
4709 format %{ %}
4710 interface(REG_INTER);
4711 %}
4712
4713 // Pointer 64 bit Register not Special
4714 operand iRegPNoSp()
4715 %{
4716 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4717 match(RegP);
4718 // match(iRegP);
4719 // match(iRegP_R0);
4720 // match(iRegP_R2);
4721 // match(iRegP_R4);
4722 // match(iRegP_R5);
4723 // match(thread_RegP);
4724 op_cost(0);
4725 format %{ %}
4726 interface(REG_INTER);
4727 %}
4728
4729 // This operand is not allowed to use rfp even if
4730 // rfp is not used to hold the frame pointer.
4731 operand iRegPNoSpNoRfp()
4732 %{
4733 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4734 match(RegP);
4735 match(iRegPNoSp);
4736 op_cost(0);
4737 format %{ %}
4738 interface(REG_INTER);
4739 %}
4740
4741 // Pointer 64 bit Register R0 only
4742 operand iRegP_R0()
4743 %{
4744 constraint(ALLOC_IN_RC(r0_reg));
4745 match(RegP);
4746 // match(iRegP);
4747 match(iRegPNoSp);
4748 op_cost(0);
4749 format %{ %}
4750 interface(REG_INTER);
4751 %}
4752
4753 // Pointer 64 bit Register R1 only
4754 operand iRegP_R1()
4755 %{
4756 constraint(ALLOC_IN_RC(r1_reg));
4757 match(RegP);
4758 // match(iRegP);
4759 match(iRegPNoSp);
4760 op_cost(0);
4761 format %{ %}
4762 interface(REG_INTER);
4763 %}
4764
4765 // Pointer 64 bit Register R2 only
4766 operand iRegP_R2()
4767 %{
4768 constraint(ALLOC_IN_RC(r2_reg));
4769 match(RegP);
4770 // match(iRegP);
4771 match(iRegPNoSp);
4772 op_cost(0);
4773 format %{ %}
4774 interface(REG_INTER);
4775 %}
4776
4777 // Pointer 64 bit Register R3 only
4778 operand iRegP_R3()
4779 %{
4780 constraint(ALLOC_IN_RC(r3_reg));
4781 match(RegP);
4782 // match(iRegP);
4783 match(iRegPNoSp);
4784 op_cost(0);
4785 format %{ %}
4786 interface(REG_INTER);
4787 %}
4788
4789 // Pointer 64 bit Register R4 only
4790 operand iRegP_R4()
4791 %{
4792 constraint(ALLOC_IN_RC(r4_reg));
4793 match(RegP);
4794 // match(iRegP);
4795 match(iRegPNoSp);
4796 op_cost(0);
4797 format %{ %}
4798 interface(REG_INTER);
4799 %}
4800
4801 // Pointer 64 bit Register R5 only
4802 operand iRegP_R5()
4803 %{
4804 constraint(ALLOC_IN_RC(r5_reg));
4805 match(RegP);
4806 // match(iRegP);
4807 match(iRegPNoSp);
4808 op_cost(0);
4809 format %{ %}
4810 interface(REG_INTER);
4811 %}
4812
4813 // Pointer 64 bit Register R10 only
4814 operand iRegP_R10()
4815 %{
4816 constraint(ALLOC_IN_RC(r10_reg));
4817 match(RegP);
4818 // match(iRegP);
4819 match(iRegPNoSp);
4820 op_cost(0);
4821 format %{ %}
4822 interface(REG_INTER);
4823 %}
4824
4825 // Long 64 bit Register R0 only
4826 operand iRegL_R0()
4827 %{
4828 constraint(ALLOC_IN_RC(r0_reg));
4829 match(RegL);
4830 match(iRegLNoSp);
4831 op_cost(0);
4832 format %{ %}
4833 interface(REG_INTER);
4834 %}
4835
4836 // Long 64 bit Register R11 only
4837 operand iRegL_R11()
4838 %{
4839 constraint(ALLOC_IN_RC(r11_reg));
4840 match(RegL);
4841 match(iRegLNoSp);
4842 op_cost(0);
4843 format %{ %}
4844 interface(REG_INTER);
4845 %}
4846
4847 // Register R0 only
4848 operand iRegI_R0()
4849 %{
4850 constraint(ALLOC_IN_RC(int_r0_reg));
4851 match(RegI);
4852 match(iRegINoSp);
4853 op_cost(0);
4854 format %{ %}
4855 interface(REG_INTER);
4856 %}
4857
4858 // Register R2 only
4859 operand iRegI_R2()
4860 %{
4861 constraint(ALLOC_IN_RC(int_r2_reg));
4862 match(RegI);
4863 match(iRegINoSp);
4864 op_cost(0);
4865 format %{ %}
4866 interface(REG_INTER);
4867 %}
4868
4869 // Register R3 only
4870 operand iRegI_R3()
4871 %{
4872 constraint(ALLOC_IN_RC(int_r3_reg));
4873 match(RegI);
4874 match(iRegINoSp);
4875 op_cost(0);
4876 format %{ %}
4877 interface(REG_INTER);
4878 %}
4879
4880
4881 // Register R4 only
4882 operand iRegI_R4()
4883 %{
4884 constraint(ALLOC_IN_RC(int_r4_reg));
4885 match(RegI);
4886 match(iRegINoSp);
4887 op_cost(0);
4888 format %{ %}
4889 interface(REG_INTER);
4890 %}
4891
4892
4893 // Pointer Register Operands
4894 // Narrow Pointer Register
4895 operand iRegN()
4896 %{
4897 constraint(ALLOC_IN_RC(any_reg32));
4898 match(RegN);
4899 match(iRegNNoSp);
4900 op_cost(0);
4901 format %{ %}
4902 interface(REG_INTER);
4903 %}
4904
4905 // Integer 64 bit Register not Special
4906 operand iRegNNoSp()
4907 %{
4908 constraint(ALLOC_IN_RC(no_special_reg32));
4909 match(RegN);
4910 op_cost(0);
4911 format %{ %}
4912 interface(REG_INTER);
4913 %}
4914
4915 // Float Register
4916 // Float register operands
4917 operand vRegF()
4918 %{
4919 constraint(ALLOC_IN_RC(float_reg));
4920 match(RegF);
4921
4922 op_cost(0);
4923 format %{ %}
4924 interface(REG_INTER);
4925 %}
4926
4927 // Double Register
4928 // Double register operands
4929 operand vRegD()
4930 %{
4931 constraint(ALLOC_IN_RC(double_reg));
4932 match(RegD);
4933
4934 op_cost(0);
4935 format %{ %}
4936 interface(REG_INTER);
4937 %}
4938
4939 // Generic vector class. This will be used for
4940 // all vector operands, including NEON and SVE.
4941 operand vReg()
4942 %{
4943 constraint(ALLOC_IN_RC(dynamic));
4944 match(VecA);
4945 match(VecD);
4946 match(VecX);
4947
4948 op_cost(0);
4949 format %{ %}
4950 interface(REG_INTER);
4951 %}
4952
4953 operand vReg_V10()
4954 %{
4955 constraint(ALLOC_IN_RC(v10_veca_reg));
4956 match(vReg);
4957
4958 op_cost(0);
4959 format %{ %}
4960 interface(REG_INTER);
4961 %}
4962
4963 operand vReg_V11()
4964 %{
4965 constraint(ALLOC_IN_RC(v11_veca_reg));
4966 match(vReg);
4967
4968 op_cost(0);
4969 format %{ %}
4970 interface(REG_INTER);
4971 %}
4972
4973 operand vReg_V12()
4974 %{
4975 constraint(ALLOC_IN_RC(v12_veca_reg));
4976 match(vReg);
4977
4978 op_cost(0);
4979 format %{ %}
4980 interface(REG_INTER);
4981 %}
4982
4983 operand vReg_V13()
4984 %{
4985 constraint(ALLOC_IN_RC(v13_veca_reg));
4986 match(vReg);
4987
4988 op_cost(0);
4989 format %{ %}
4990 interface(REG_INTER);
4991 %}
4992
4993 operand vReg_V17()
4994 %{
4995 constraint(ALLOC_IN_RC(v17_veca_reg));
4996 match(vReg);
4997
4998 op_cost(0);
4999 format %{ %}
5000 interface(REG_INTER);
5001 %}
5002
5003 operand vReg_V18()
5004 %{
5005 constraint(ALLOC_IN_RC(v18_veca_reg));
5006 match(vReg);
5007
5008 op_cost(0);
5009 format %{ %}
5010 interface(REG_INTER);
5011 %}
5012
5013 operand vReg_V23()
5014 %{
5015 constraint(ALLOC_IN_RC(v23_veca_reg));
5016 match(vReg);
5017
5018 op_cost(0);
5019 format %{ %}
5020 interface(REG_INTER);
5021 %}
5022
5023 operand vReg_V24()
5024 %{
5025 constraint(ALLOC_IN_RC(v24_veca_reg));
5026 match(vReg);
5027
5028 op_cost(0);
5029 format %{ %}
5030 interface(REG_INTER);
5031 %}
5032
5033 operand vecA()
5034 %{
5035 constraint(ALLOC_IN_RC(vectora_reg));
5036 match(VecA);
5037
5038 op_cost(0);
5039 format %{ %}
5040 interface(REG_INTER);
5041 %}
5042
5043 operand vecD()
5044 %{
5045 constraint(ALLOC_IN_RC(vectord_reg));
5046 match(VecD);
5047
5048 op_cost(0);
5049 format %{ %}
5050 interface(REG_INTER);
5051 %}
5052
5053 operand vecX()
5054 %{
5055 constraint(ALLOC_IN_RC(vectorx_reg));
5056 match(VecX);
5057
5058 op_cost(0);
5059 format %{ %}
5060 interface(REG_INTER);
5061 %}
5062
5063 operand vRegD_V0()
5064 %{
5065 constraint(ALLOC_IN_RC(v0_reg));
5066 match(RegD);
5067 op_cost(0);
5068 format %{ %}
5069 interface(REG_INTER);
5070 %}
5071
5072 operand vRegD_V1()
5073 %{
5074 constraint(ALLOC_IN_RC(v1_reg));
5075 match(RegD);
5076 op_cost(0);
5077 format %{ %}
5078 interface(REG_INTER);
5079 %}
5080
5081 operand vRegD_V2()
5082 %{
5083 constraint(ALLOC_IN_RC(v2_reg));
5084 match(RegD);
5085 op_cost(0);
5086 format %{ %}
5087 interface(REG_INTER);
5088 %}
5089
5090 operand vRegD_V3()
5091 %{
5092 constraint(ALLOC_IN_RC(v3_reg));
5093 match(RegD);
5094 op_cost(0);
5095 format %{ %}
5096 interface(REG_INTER);
5097 %}
5098
5099 operand vRegD_V4()
5100 %{
5101 constraint(ALLOC_IN_RC(v4_reg));
5102 match(RegD);
5103 op_cost(0);
5104 format %{ %}
5105 interface(REG_INTER);
5106 %}
5107
5108 operand vRegD_V5()
5109 %{
5110 constraint(ALLOC_IN_RC(v5_reg));
5111 match(RegD);
5112 op_cost(0);
5113 format %{ %}
5114 interface(REG_INTER);
5115 %}
5116
5117 operand vRegD_V6()
5118 %{
5119 constraint(ALLOC_IN_RC(v6_reg));
5120 match(RegD);
5121 op_cost(0);
5122 format %{ %}
5123 interface(REG_INTER);
5124 %}
5125
5126 operand vRegD_V7()
5127 %{
5128 constraint(ALLOC_IN_RC(v7_reg));
5129 match(RegD);
5130 op_cost(0);
5131 format %{ %}
5132 interface(REG_INTER);
5133 %}
5134
5135 operand vRegD_V12()
5136 %{
5137 constraint(ALLOC_IN_RC(v12_reg));
5138 match(RegD);
5139 op_cost(0);
5140 format %{ %}
5141 interface(REG_INTER);
5142 %}
5143
5144 operand vRegD_V13()
5145 %{
5146 constraint(ALLOC_IN_RC(v13_reg));
5147 match(RegD);
5148 op_cost(0);
5149 format %{ %}
5150 interface(REG_INTER);
5151 %}
5152
5153 operand pReg()
5154 %{
5155 constraint(ALLOC_IN_RC(pr_reg));
5156 match(RegVectMask);
5157 match(pRegGov);
5158 op_cost(0);
5159 format %{ %}
5160 interface(REG_INTER);
5161 %}
5162
5163 operand pRegGov()
5164 %{
5165 constraint(ALLOC_IN_RC(gov_pr));
5166 match(RegVectMask);
5167 match(pReg);
5168 op_cost(0);
5169 format %{ %}
5170 interface(REG_INTER);
5171 %}
5172
5173 operand pRegGov_P0()
5174 %{
5175 constraint(ALLOC_IN_RC(p0_reg));
5176 match(RegVectMask);
5177 op_cost(0);
5178 format %{ %}
5179 interface(REG_INTER);
5180 %}
5181
5182 operand pRegGov_P1()
5183 %{
5184 constraint(ALLOC_IN_RC(p1_reg));
5185 match(RegVectMask);
5186 op_cost(0);
5187 format %{ %}
5188 interface(REG_INTER);
5189 %}
5190
5191 // Flags register, used as output of signed compare instructions
5192
5193 // note that on AArch64 we also use this register as the output for
5194 // for floating point compare instructions (CmpF CmpD). this ensures
5195 // that ordered inequality tests use GT, GE, LT or LE none of which
5196 // pass through cases where the result is unordered i.e. one or both
5197 // inputs to the compare is a NaN. this means that the ideal code can
5198 // replace e.g. a GT with an LE and not end up capturing the NaN case
5199 // (where the comparison should always fail). EQ and NE tests are
5200 // always generated in ideal code so that unordered folds into the NE
5201 // case, matching the behaviour of AArch64 NE.
5202 //
5203 // This differs from x86 where the outputs of FP compares use a
5204 // special FP flags registers and where compares based on this
5205 // register are distinguished into ordered inequalities (cmpOpUCF) and
5206 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5207 // to explicitly handle the unordered case in branches. x86 also has
5208 // to include extra CMoveX rules to accept a cmpOpUCF input.
5209
5210 operand rFlagsReg()
5211 %{
5212 constraint(ALLOC_IN_RC(int_flags));
5213 match(RegFlags);
5214
5215 op_cost(0);
5216 format %{ "RFLAGS" %}
5217 interface(REG_INTER);
5218 %}
5219
5220 // Flags register, used as output of unsigned compare instructions
5221 operand rFlagsRegU()
5222 %{
5223 constraint(ALLOC_IN_RC(int_flags));
5224 match(RegFlags);
5225
5226 op_cost(0);
5227 format %{ "RFLAGSU" %}
5228 interface(REG_INTER);
5229 %}
5230
5231 // Special Registers
5232
5233 // Method Register
5234 operand inline_cache_RegP(iRegP reg)
5235 %{
5236 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5237 match(reg);
5238 match(iRegPNoSp);
5239 op_cost(0);
5240 format %{ %}
5241 interface(REG_INTER);
5242 %}
5243
5244 // Thread Register
5245 operand thread_RegP(iRegP reg)
5246 %{
5247 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5248 match(reg);
5249 op_cost(0);
5250 format %{ %}
5251 interface(REG_INTER);
5252 %}
5253
5254 //----------Memory Operands----------------------------------------------------
5255
5256 operand indirect(iRegP reg)
5257 %{
5258 constraint(ALLOC_IN_RC(ptr_reg));
5259 match(reg);
5260 op_cost(0);
5261 format %{ "[$reg]" %}
5262 interface(MEMORY_INTER) %{
5263 base($reg);
5264 index(0xffffffff);
5265 scale(0x0);
5266 disp(0x0);
5267 %}
5268 %}
5269
5270 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5271 %{
5272 constraint(ALLOC_IN_RC(ptr_reg));
5273 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5274 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5275 op_cost(0);
5276 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5277 interface(MEMORY_INTER) %{
5278 base($reg);
5279 index($ireg);
5280 scale($scale);
5281 disp(0x0);
5282 %}
5283 %}
5284
5285 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5286 %{
5287 constraint(ALLOC_IN_RC(ptr_reg));
5288 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5289 match(AddP reg (LShiftL lreg scale));
5290 op_cost(0);
5291 format %{ "$reg, $lreg lsl($scale)" %}
5292 interface(MEMORY_INTER) %{
5293 base($reg);
5294 index($lreg);
5295 scale($scale);
5296 disp(0x0);
5297 %}
5298 %}
5299
5300 operand indIndexI2L(iRegP reg, iRegI ireg)
5301 %{
5302 constraint(ALLOC_IN_RC(ptr_reg));
5303 match(AddP reg (ConvI2L ireg));
5304 op_cost(0);
5305 format %{ "$reg, $ireg, 0, I2L" %}
5306 interface(MEMORY_INTER) %{
5307 base($reg);
5308 index($ireg);
5309 scale(0x0);
5310 disp(0x0);
5311 %}
5312 %}
5313
5314 operand indIndex(iRegP reg, iRegL lreg)
5315 %{
5316 constraint(ALLOC_IN_RC(ptr_reg));
5317 match(AddP reg lreg);
5318 op_cost(0);
5319 format %{ "$reg, $lreg" %}
5320 interface(MEMORY_INTER) %{
5321 base($reg);
5322 index($lreg);
5323 scale(0x0);
5324 disp(0x0);
5325 %}
5326 %}
5327
5328 operand indOffI1(iRegP reg, immIOffset1 off)
5329 %{
5330 constraint(ALLOC_IN_RC(ptr_reg));
5331 match(AddP reg off);
5332 op_cost(0);
5333 format %{ "[$reg, $off]" %}
5334 interface(MEMORY_INTER) %{
5335 base($reg);
5336 index(0xffffffff);
5337 scale(0x0);
5338 disp($off);
5339 %}
5340 %}
5341
5342 operand indOffI2(iRegP reg, immIOffset2 off)
5343 %{
5344 constraint(ALLOC_IN_RC(ptr_reg));
5345 match(AddP reg off);
5346 op_cost(0);
5347 format %{ "[$reg, $off]" %}
5348 interface(MEMORY_INTER) %{
5349 base($reg);
5350 index(0xffffffff);
5351 scale(0x0);
5352 disp($off);
5353 %}
5354 %}
5355
5356 operand indOffI4(iRegP reg, immIOffset4 off)
5357 %{
5358 constraint(ALLOC_IN_RC(ptr_reg));
5359 match(AddP reg off);
5360 op_cost(0);
5361 format %{ "[$reg, $off]" %}
5362 interface(MEMORY_INTER) %{
5363 base($reg);
5364 index(0xffffffff);
5365 scale(0x0);
5366 disp($off);
5367 %}
5368 %}
5369
5370 operand indOffI8(iRegP reg, immIOffset8 off)
5371 %{
5372 constraint(ALLOC_IN_RC(ptr_reg));
5373 match(AddP reg off);
5374 op_cost(0);
5375 format %{ "[$reg, $off]" %}
5376 interface(MEMORY_INTER) %{
5377 base($reg);
5378 index(0xffffffff);
5379 scale(0x0);
5380 disp($off);
5381 %}
5382 %}
5383
5384 operand indOffI16(iRegP reg, immIOffset16 off)
5385 %{
5386 constraint(ALLOC_IN_RC(ptr_reg));
5387 match(AddP reg off);
5388 op_cost(0);
5389 format %{ "[$reg, $off]" %}
5390 interface(MEMORY_INTER) %{
5391 base($reg);
5392 index(0xffffffff);
5393 scale(0x0);
5394 disp($off);
5395 %}
5396 %}
5397
5398 operand indOffL1(iRegP reg, immLoffset1 off)
5399 %{
5400 constraint(ALLOC_IN_RC(ptr_reg));
5401 match(AddP reg off);
5402 op_cost(0);
5403 format %{ "[$reg, $off]" %}
5404 interface(MEMORY_INTER) %{
5405 base($reg);
5406 index(0xffffffff);
5407 scale(0x0);
5408 disp($off);
5409 %}
5410 %}
5411
5412 operand indOffL2(iRegP reg, immLoffset2 off)
5413 %{
5414 constraint(ALLOC_IN_RC(ptr_reg));
5415 match(AddP reg off);
5416 op_cost(0);
5417 format %{ "[$reg, $off]" %}
5418 interface(MEMORY_INTER) %{
5419 base($reg);
5420 index(0xffffffff);
5421 scale(0x0);
5422 disp($off);
5423 %}
5424 %}
5425
5426 operand indOffL4(iRegP reg, immLoffset4 off)
5427 %{
5428 constraint(ALLOC_IN_RC(ptr_reg));
5429 match(AddP reg off);
5430 op_cost(0);
5431 format %{ "[$reg, $off]" %}
5432 interface(MEMORY_INTER) %{
5433 base($reg);
5434 index(0xffffffff);
5435 scale(0x0);
5436 disp($off);
5437 %}
5438 %}
5439
5440 operand indOffL8(iRegP reg, immLoffset8 off)
5441 %{
5442 constraint(ALLOC_IN_RC(ptr_reg));
5443 match(AddP reg off);
5444 op_cost(0);
5445 format %{ "[$reg, $off]" %}
5446 interface(MEMORY_INTER) %{
5447 base($reg);
5448 index(0xffffffff);
5449 scale(0x0);
5450 disp($off);
5451 %}
5452 %}
5453
5454 operand indOffL16(iRegP reg, immLoffset16 off)
5455 %{
5456 constraint(ALLOC_IN_RC(ptr_reg));
5457 match(AddP reg off);
5458 op_cost(0);
5459 format %{ "[$reg, $off]" %}
5460 interface(MEMORY_INTER) %{
5461 base($reg);
5462 index(0xffffffff);
5463 scale(0x0);
5464 disp($off);
5465 %}
5466 %}
5467
5468 operand indirectX2P(iRegL reg)
5469 %{
5470 constraint(ALLOC_IN_RC(ptr_reg));
5471 match(CastX2P reg);
5472 op_cost(0);
5473 format %{ "[$reg]\t# long -> ptr" %}
5474 interface(MEMORY_INTER) %{
5475 base($reg);
5476 index(0xffffffff);
5477 scale(0x0);
5478 disp(0x0);
5479 %}
5480 %}
5481
5482 operand indOffX2P(iRegL reg, immLOffset off)
5483 %{
5484 constraint(ALLOC_IN_RC(ptr_reg));
5485 match(AddP (CastX2P reg) off);
5486 op_cost(0);
5487 format %{ "[$reg, $off]\t# long -> ptr" %}
5488 interface(MEMORY_INTER) %{
5489 base($reg);
5490 index(0xffffffff);
5491 scale(0x0);
5492 disp($off);
5493 %}
5494 %}
5495
5496 operand indirectN(iRegN reg)
5497 %{
5498 predicate(CompressedOops::shift() == 0);
5499 constraint(ALLOC_IN_RC(ptr_reg));
5500 match(DecodeN reg);
5501 op_cost(0);
5502 format %{ "[$reg]\t# narrow" %}
5503 interface(MEMORY_INTER) %{
5504 base($reg);
5505 index(0xffffffff);
5506 scale(0x0);
5507 disp(0x0);
5508 %}
5509 %}
5510
5511 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5512 %{
5513 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5514 constraint(ALLOC_IN_RC(ptr_reg));
5515 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5516 op_cost(0);
5517 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5518 interface(MEMORY_INTER) %{
5519 base($reg);
5520 index($ireg);
5521 scale($scale);
5522 disp(0x0);
5523 %}
5524 %}
5525
5526 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5527 %{
5528 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5529 constraint(ALLOC_IN_RC(ptr_reg));
5530 match(AddP (DecodeN reg) (LShiftL lreg scale));
5531 op_cost(0);
5532 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5533 interface(MEMORY_INTER) %{
5534 base($reg);
5535 index($lreg);
5536 scale($scale);
5537 disp(0x0);
5538 %}
5539 %}
5540
5541 operand indIndexI2LN(iRegN reg, iRegI ireg)
5542 %{
5543 predicate(CompressedOops::shift() == 0);
5544 constraint(ALLOC_IN_RC(ptr_reg));
5545 match(AddP (DecodeN reg) (ConvI2L ireg));
5546 op_cost(0);
5547 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5548 interface(MEMORY_INTER) %{
5549 base($reg);
5550 index($ireg);
5551 scale(0x0);
5552 disp(0x0);
5553 %}
5554 %}
5555
5556 operand indIndexN(iRegN reg, iRegL lreg)
5557 %{
5558 predicate(CompressedOops::shift() == 0);
5559 constraint(ALLOC_IN_RC(ptr_reg));
5560 match(AddP (DecodeN reg) lreg);
5561 op_cost(0);
5562 format %{ "$reg, $lreg\t# narrow" %}
5563 interface(MEMORY_INTER) %{
5564 base($reg);
5565 index($lreg);
5566 scale(0x0);
5567 disp(0x0);
5568 %}
5569 %}
5570
5571 operand indOffIN(iRegN reg, immIOffset off)
5572 %{
5573 predicate(CompressedOops::shift() == 0);
5574 constraint(ALLOC_IN_RC(ptr_reg));
5575 match(AddP (DecodeN reg) off);
5576 op_cost(0);
5577 format %{ "[$reg, $off]\t# narrow" %}
5578 interface(MEMORY_INTER) %{
5579 base($reg);
5580 index(0xffffffff);
5581 scale(0x0);
5582 disp($off);
5583 %}
5584 %}
5585
5586 operand indOffLN(iRegN reg, immLOffset off)
5587 %{
5588 predicate(CompressedOops::shift() == 0);
5589 constraint(ALLOC_IN_RC(ptr_reg));
5590 match(AddP (DecodeN reg) off);
5591 op_cost(0);
5592 format %{ "[$reg, $off]\t# narrow" %}
5593 interface(MEMORY_INTER) %{
5594 base($reg);
5595 index(0xffffffff);
5596 scale(0x0);
5597 disp($off);
5598 %}
5599 %}
5600
5601
5602 //----------Special Memory Operands--------------------------------------------
5603 // Stack Slot Operand - This operand is used for loading and storing temporary
5604 // values on the stack where a match requires a value to
5605 // flow through memory.
5606 operand stackSlotP(sRegP reg)
5607 %{
5608 constraint(ALLOC_IN_RC(stack_slots));
5609 op_cost(100);
5610 // No match rule because this operand is only generated in matching
5611 // match(RegP);
5612 format %{ "[$reg]" %}
5613 interface(MEMORY_INTER) %{
5614 base(0x1e); // RSP
5615 index(0x0); // No Index
5616 scale(0x0); // No Scale
5617 disp($reg); // Stack Offset
5618 %}
5619 %}
5620
5621 operand stackSlotI(sRegI reg)
5622 %{
5623 constraint(ALLOC_IN_RC(stack_slots));
5624 // No match rule because this operand is only generated in matching
5625 // match(RegI);
5626 format %{ "[$reg]" %}
5627 interface(MEMORY_INTER) %{
5628 base(0x1e); // RSP
5629 index(0x0); // No Index
5630 scale(0x0); // No Scale
5631 disp($reg); // Stack Offset
5632 %}
5633 %}
5634
5635 operand stackSlotF(sRegF reg)
5636 %{
5637 constraint(ALLOC_IN_RC(stack_slots));
5638 // No match rule because this operand is only generated in matching
5639 // match(RegF);
5640 format %{ "[$reg]" %}
5641 interface(MEMORY_INTER) %{
5642 base(0x1e); // RSP
5643 index(0x0); // No Index
5644 scale(0x0); // No Scale
5645 disp($reg); // Stack Offset
5646 %}
5647 %}
5648
5649 operand stackSlotD(sRegD reg)
5650 %{
5651 constraint(ALLOC_IN_RC(stack_slots));
5652 // No match rule because this operand is only generated in matching
5653 // match(RegD);
5654 format %{ "[$reg]" %}
5655 interface(MEMORY_INTER) %{
5656 base(0x1e); // RSP
5657 index(0x0); // No Index
5658 scale(0x0); // No Scale
5659 disp($reg); // Stack Offset
5660 %}
5661 %}
5662
5663 operand stackSlotL(sRegL reg)
5664 %{
5665 constraint(ALLOC_IN_RC(stack_slots));
5666 // No match rule because this operand is only generated in matching
5667 // match(RegL);
5668 format %{ "[$reg]" %}
5669 interface(MEMORY_INTER) %{
5670 base(0x1e); // RSP
5671 index(0x0); // No Index
5672 scale(0x0); // No Scale
5673 disp($reg); // Stack Offset
5674 %}
5675 %}
5676
5677 // Operands for expressing Control Flow
5678 // NOTE: Label is a predefined operand which should not be redefined in
5679 // the AD file. It is generically handled within the ADLC.
5680
5681 //----------Conditional Branch Operands----------------------------------------
5682 // Comparison Op - This is the operation of the comparison, and is limited to
5683 // the following set of codes:
5684 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5685 //
5686 // Other attributes of the comparison, such as unsignedness, are specified
5687 // by the comparison instruction that sets a condition code flags register.
5688 // That result is represented by a flags operand whose subtype is appropriate
5689 // to the unsignedness (etc.) of the comparison.
5690 //
5691 // Later, the instruction which matches both the Comparison Op (a Bool) and
5692 // the flags (produced by the Cmp) specifies the coding of the comparison op
5693 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5694
5695 // used for signed integral comparisons and fp comparisons
5696
5697 operand cmpOp()
5698 %{
5699 match(Bool);
5700
5701 format %{ "" %}
5702 interface(COND_INTER) %{
5703 equal(0x0, "eq");
5704 not_equal(0x1, "ne");
5705 less(0xb, "lt");
5706 greater_equal(0xa, "ge");
5707 less_equal(0xd, "le");
5708 greater(0xc, "gt");
5709 overflow(0x6, "vs");
5710 no_overflow(0x7, "vc");
5711 %}
5712 %}
5713
5714 // used for unsigned integral comparisons
5715
5716 operand cmpOpU()
5717 %{
5718 match(Bool);
5719
5720 format %{ "" %}
5721 interface(COND_INTER) %{
5722 equal(0x0, "eq");
5723 not_equal(0x1, "ne");
5724 less(0x3, "lo");
5725 greater_equal(0x2, "hs");
5726 less_equal(0x9, "ls");
5727 greater(0x8, "hi");
5728 overflow(0x6, "vs");
5729 no_overflow(0x7, "vc");
5730 %}
5731 %}
5732
5733 // used for certain integral comparisons which can be
5734 // converted to cbxx or tbxx instructions
5735
5736 operand cmpOpEqNe()
5737 %{
5738 match(Bool);
5739 op_cost(0);
5740 predicate(n->as_Bool()->_test._test == BoolTest::ne
5741 || n->as_Bool()->_test._test == BoolTest::eq);
5742
5743 format %{ "" %}
5744 interface(COND_INTER) %{
5745 equal(0x0, "eq");
5746 not_equal(0x1, "ne");
5747 less(0xb, "lt");
5748 greater_equal(0xa, "ge");
5749 less_equal(0xd, "le");
5750 greater(0xc, "gt");
5751 overflow(0x6, "vs");
5752 no_overflow(0x7, "vc");
5753 %}
5754 %}
5755
5756 // used for certain integral comparisons which can be
5757 // converted to cbxx or tbxx instructions
5758
5759 operand cmpOpLtGe()
5760 %{
5761 match(Bool);
5762 op_cost(0);
5763
5764 predicate(n->as_Bool()->_test._test == BoolTest::lt
5765 || n->as_Bool()->_test._test == BoolTest::ge);
5766
5767 format %{ "" %}
5768 interface(COND_INTER) %{
5769 equal(0x0, "eq");
5770 not_equal(0x1, "ne");
5771 less(0xb, "lt");
5772 greater_equal(0xa, "ge");
5773 less_equal(0xd, "le");
5774 greater(0xc, "gt");
5775 overflow(0x6, "vs");
5776 no_overflow(0x7, "vc");
5777 %}
5778 %}
5779
5780 // used for certain unsigned integral comparisons which can be
5781 // converted to cbxx or tbxx instructions
5782
5783 operand cmpOpUEqNeLeGt()
5784 %{
5785 match(Bool);
5786 op_cost(0);
5787
5788 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5789 n->as_Bool()->_test._test == BoolTest::ne ||
5790 n->as_Bool()->_test._test == BoolTest::le ||
5791 n->as_Bool()->_test._test == BoolTest::gt);
5792
5793 format %{ "" %}
5794 interface(COND_INTER) %{
5795 equal(0x0, "eq");
5796 not_equal(0x1, "ne");
5797 less(0x3, "lo");
5798 greater_equal(0x2, "hs");
5799 less_equal(0x9, "ls");
5800 greater(0x8, "hi");
5801 overflow(0x6, "vs");
5802 no_overflow(0x7, "vc");
5803 %}
5804 %}
5805
5806 // Special operand allowing long args to int ops to be truncated for free
5807
5808 operand iRegL2I(iRegL reg) %{
5809
5810 op_cost(0);
5811
5812 match(ConvL2I reg);
5813
5814 format %{ "l2i($reg)" %}
5815
5816 interface(REG_INTER)
5817 %}
5818
5819 operand iRegL2P(iRegL reg) %{
5820
5821 op_cost(0);
5822
5823 match(CastX2P reg);
5824
5825 format %{ "l2p($reg)" %}
5826
5827 interface(REG_INTER)
5828 %}
5829
5830 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5831 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5832 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5833 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5834
5835 //----------OPERAND CLASSES----------------------------------------------------
5836 // Operand Classes are groups of operands that are used as to simplify
5837 // instruction definitions by not requiring the AD writer to specify
5838 // separate instructions for every form of operand when the
5839 // instruction accepts multiple operand types with the same basic
5840 // encoding and format. The classic case of this is memory operands.
5841
5842 // memory is used to define read/write location for load/store
5843 // instruction defs. we can turn a memory op into an Address
5844
5845 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5846 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5847
5848 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5849 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5850
5851 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5852 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5853
5854 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5855 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5856
5857 // All of the memory operands. For the pipeline description.
5858 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5859 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5860 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5861
5862 opclass memory_noindex(indirect,
5863 indOffI1, indOffL1,indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5864 indirectN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5865
5866 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5867 // operations. it allows the src to be either an iRegI or a (ConvL2I
5868 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5869 // can be elided because the 32-bit instruction will just employ the
5870 // lower 32 bits anyway.
5871 //
5872 // n.b. this does not elide all L2I conversions. if the truncated
5873 // value is consumed by more than one operation then the ConvL2I
5874 // cannot be bundled into the consuming nodes so an l2i gets planted
5875 // (actually a movw $dst $src) and the downstream instructions consume
5876 // the result of the l2i as an iRegI input. That's a shame since the
5877 // movw is actually redundant but its not too costly.
5878
5879 opclass iRegIorL2I(iRegI, iRegL2I);
5880 opclass iRegPorL2P(iRegP, iRegL2P);
5881
5882 //----------PIPELINE-----------------------------------------------------------
5883 // Rules which define the behavior of the target architectures pipeline.
5884
5885 // For specific pipelines, eg A53, define the stages of that pipeline
5886 //pipe_desc(ISS, EX1, EX2, WR);
5887 #define ISS S0
5888 #define EX1 S1
5889 #define EX2 S2
5890 #define WR S3
5891
5892 // Integer ALU reg operation
5893 pipeline %{
5894
5895 attributes %{
5896 // ARM instructions are of fixed length
5897 fixed_size_instructions; // Fixed size instructions TODO does
5898 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5899 // ARM instructions come in 32-bit word units
5900 instruction_unit_size = 4; // An instruction is 4 bytes long
5901 instruction_fetch_unit_size = 64; // The processor fetches one line
5902 instruction_fetch_units = 1; // of 64 bytes
5903 %}
5904
5905 // We don't use an actual pipeline model so don't care about resources
5906 // or description. we do use pipeline classes to introduce fixed
5907 // latencies
5908
5909 //----------RESOURCES----------------------------------------------------------
5910 // Resources are the functional units available to the machine
5911
5912 resources( INS0, INS1, INS01 = INS0 | INS1,
5913 ALU0, ALU1, ALU = ALU0 | ALU1,
5914 MAC,
5915 DIV,
5916 BRANCH,
5917 LDST,
5918 NEON_FP);
5919
5920 //----------PIPELINE DESCRIPTION-----------------------------------------------
5921 // Pipeline Description specifies the stages in the machine's pipeline
5922
5923 // Define the pipeline as a generic 6 stage pipeline
5924 pipe_desc(S0, S1, S2, S3, S4, S5);
5925
5926 //----------PIPELINE CLASSES---------------------------------------------------
5927 // Pipeline Classes describe the stages in which input and output are
5928 // referenced by the hardware pipeline.
5929
5930 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5931 %{
5932 single_instruction;
5933 src1 : S1(read);
5934 src2 : S2(read);
5935 dst : S5(write);
5936 INS01 : ISS;
5937 NEON_FP : S5;
5938 %}
5939
5940 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5941 %{
5942 single_instruction;
5943 src1 : S1(read);
5944 src2 : S2(read);
5945 dst : S5(write);
5946 INS01 : ISS;
5947 NEON_FP : S5;
5948 %}
5949
5950 pipe_class fp_uop_s(vRegF dst, vRegF src)
5951 %{
5952 single_instruction;
5953 src : S1(read);
5954 dst : S5(write);
5955 INS01 : ISS;
5956 NEON_FP : S5;
5957 %}
5958
5959 pipe_class fp_uop_d(vRegD dst, vRegD src)
5960 %{
5961 single_instruction;
5962 src : S1(read);
5963 dst : S5(write);
5964 INS01 : ISS;
5965 NEON_FP : S5;
5966 %}
5967
5968 pipe_class fp_d2f(vRegF dst, vRegD src)
5969 %{
5970 single_instruction;
5971 src : S1(read);
5972 dst : S5(write);
5973 INS01 : ISS;
5974 NEON_FP : S5;
5975 %}
5976
5977 pipe_class fp_f2d(vRegD dst, vRegF src)
5978 %{
5979 single_instruction;
5980 src : S1(read);
5981 dst : S5(write);
5982 INS01 : ISS;
5983 NEON_FP : S5;
5984 %}
5985
5986 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5987 %{
5988 single_instruction;
5989 src : S1(read);
5990 dst : S5(write);
5991 INS01 : ISS;
5992 NEON_FP : S5;
5993 %}
5994
5995 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5996 %{
5997 single_instruction;
5998 src : S1(read);
5999 dst : S5(write);
6000 INS01 : ISS;
6001 NEON_FP : S5;
6002 %}
6003
6004 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6005 %{
6006 single_instruction;
6007 src : S1(read);
6008 dst : S5(write);
6009 INS01 : ISS;
6010 NEON_FP : S5;
6011 %}
6012
6013 pipe_class fp_l2f(vRegF dst, iRegL src)
6014 %{
6015 single_instruction;
6016 src : S1(read);
6017 dst : S5(write);
6018 INS01 : ISS;
6019 NEON_FP : S5;
6020 %}
6021
6022 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6023 %{
6024 single_instruction;
6025 src : S1(read);
6026 dst : S5(write);
6027 INS01 : ISS;
6028 NEON_FP : S5;
6029 %}
6030
6031 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6032 %{
6033 single_instruction;
6034 src : S1(read);
6035 dst : S5(write);
6036 INS01 : ISS;
6037 NEON_FP : S5;
6038 %}
6039
6040 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6041 %{
6042 single_instruction;
6043 src : S1(read);
6044 dst : S5(write);
6045 INS01 : ISS;
6046 NEON_FP : S5;
6047 %}
6048
6049 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6050 %{
6051 single_instruction;
6052 src : S1(read);
6053 dst : S5(write);
6054 INS01 : ISS;
6055 NEON_FP : S5;
6056 %}
6057
6058 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6059 %{
6060 single_instruction;
6061 src1 : S1(read);
6062 src2 : S2(read);
6063 dst : S5(write);
6064 INS0 : ISS;
6065 NEON_FP : S5;
6066 %}
6067
6068 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6069 %{
6070 single_instruction;
6071 src1 : S1(read);
6072 src2 : S2(read);
6073 dst : S5(write);
6074 INS0 : ISS;
6075 NEON_FP : S5;
6076 %}
6077
6078 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6079 %{
6080 single_instruction;
6081 cr : S1(read);
6082 src1 : S1(read);
6083 src2 : S1(read);
6084 dst : S3(write);
6085 INS01 : ISS;
6086 NEON_FP : S3;
6087 %}
6088
6089 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6090 %{
6091 single_instruction;
6092 cr : S1(read);
6093 src1 : S1(read);
6094 src2 : S1(read);
6095 dst : S3(write);
6096 INS01 : ISS;
6097 NEON_FP : S3;
6098 %}
6099
6100 pipe_class fp_imm_s(vRegF dst)
6101 %{
6102 single_instruction;
6103 dst : S3(write);
6104 INS01 : ISS;
6105 NEON_FP : S3;
6106 %}
6107
6108 pipe_class fp_imm_d(vRegD dst)
6109 %{
6110 single_instruction;
6111 dst : S3(write);
6112 INS01 : ISS;
6113 NEON_FP : S3;
6114 %}
6115
6116 pipe_class fp_load_constant_s(vRegF dst)
6117 %{
6118 single_instruction;
6119 dst : S4(write);
6120 INS01 : ISS;
6121 NEON_FP : S4;
6122 %}
6123
6124 pipe_class fp_load_constant_d(vRegD dst)
6125 %{
6126 single_instruction;
6127 dst : S4(write);
6128 INS01 : ISS;
6129 NEON_FP : S4;
6130 %}
6131
6132 //------- Integer ALU operations --------------------------
6133
6134 // Integer ALU reg-reg operation
6135 // Operands needed in EX1, result generated in EX2
6136 // Eg. ADD x0, x1, x2
6137 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6138 %{
6139 single_instruction;
6140 dst : EX2(write);
6141 src1 : EX1(read);
6142 src2 : EX1(read);
6143 INS01 : ISS; // Dual issue as instruction 0 or 1
6144 ALU : EX2;
6145 %}
6146
6147 // Integer ALU reg-reg operation with constant shift
6148 // Shifted register must be available in LATE_ISS instead of EX1
6149 // Eg. ADD x0, x1, x2, LSL #2
6150 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6151 %{
6152 single_instruction;
6153 dst : EX2(write);
6154 src1 : EX1(read);
6155 src2 : ISS(read);
6156 INS01 : ISS;
6157 ALU : EX2;
6158 %}
6159
6160 // Integer ALU reg operation with constant shift
6161 // Eg. LSL x0, x1, #shift
6162 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6163 %{
6164 single_instruction;
6165 dst : EX2(write);
6166 src1 : ISS(read);
6167 INS01 : ISS;
6168 ALU : EX2;
6169 %}
6170
6171 // Integer ALU reg-reg operation with variable shift
6172 // Both operands must be available in LATE_ISS instead of EX1
6173 // Result is available in EX1 instead of EX2
6174 // Eg. LSLV x0, x1, x2
6175 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6176 %{
6177 single_instruction;
6178 dst : EX1(write);
6179 src1 : ISS(read);
6180 src2 : ISS(read);
6181 INS01 : ISS;
6182 ALU : EX1;
6183 %}
6184
6185 // Integer ALU reg-reg operation with extract
6186 // As for _vshift above, but result generated in EX2
6187 // Eg. EXTR x0, x1, x2, #N
6188 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6189 %{
6190 single_instruction;
6191 dst : EX2(write);
6192 src1 : ISS(read);
6193 src2 : ISS(read);
6194 INS1 : ISS; // Can only dual issue as Instruction 1
6195 ALU : EX1;
6196 %}
6197
6198 // Integer ALU reg operation
6199 // Eg. NEG x0, x1
6200 pipe_class ialu_reg(iRegI dst, iRegI src)
6201 %{
6202 single_instruction;
6203 dst : EX2(write);
6204 src : EX1(read);
6205 INS01 : ISS;
6206 ALU : EX2;
6207 %}
6208
6209 // Integer ALU reg mmediate operation
6210 // Eg. ADD x0, x1, #N
6211 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6212 %{
6213 single_instruction;
6214 dst : EX2(write);
6215 src1 : EX1(read);
6216 INS01 : ISS;
6217 ALU : EX2;
6218 %}
6219
6220 // Integer ALU immediate operation (no source operands)
6221 // Eg. MOV x0, #N
6222 pipe_class ialu_imm(iRegI dst)
6223 %{
6224 single_instruction;
6225 dst : EX1(write);
6226 INS01 : ISS;
6227 ALU : EX1;
6228 %}
6229
6230 //------- Compare operation -------------------------------
6231
6232 // Compare reg-reg
6233 // Eg. CMP x0, x1
6234 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6235 %{
6236 single_instruction;
6237 // fixed_latency(16);
6238 cr : EX2(write);
6239 op1 : EX1(read);
6240 op2 : EX1(read);
6241 INS01 : ISS;
6242 ALU : EX2;
6243 %}
6244
6245 // Compare reg-reg
6246 // Eg. CMP x0, #N
6247 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6248 %{
6249 single_instruction;
6250 // fixed_latency(16);
6251 cr : EX2(write);
6252 op1 : EX1(read);
6253 INS01 : ISS;
6254 ALU : EX2;
6255 %}
6256
6257 //------- Conditional instructions ------------------------
6258
6259 // Conditional no operands
6260 // Eg. CSINC x0, zr, zr, <cond>
6261 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6262 %{
6263 single_instruction;
6264 cr : EX1(read);
6265 dst : EX2(write);
6266 INS01 : ISS;
6267 ALU : EX2;
6268 %}
6269
6270 // Conditional 2 operand
6271 // EG. CSEL X0, X1, X2, <cond>
6272 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6273 %{
6274 single_instruction;
6275 cr : EX1(read);
6276 src1 : EX1(read);
6277 src2 : EX1(read);
6278 dst : EX2(write);
6279 INS01 : ISS;
6280 ALU : EX2;
6281 %}
6282
6283 // Conditional 2 operand
6284 // EG. CSEL X0, X1, X2, <cond>
6285 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6286 %{
6287 single_instruction;
6288 cr : EX1(read);
6289 src : EX1(read);
6290 dst : EX2(write);
6291 INS01 : ISS;
6292 ALU : EX2;
6293 %}
6294
6295 //------- Multiply pipeline operations --------------------
6296
6297 // Multiply reg-reg
6298 // Eg. MUL w0, w1, w2
6299 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6300 %{
6301 single_instruction;
6302 dst : WR(write);
6303 src1 : ISS(read);
6304 src2 : ISS(read);
6305 INS01 : ISS;
6306 MAC : WR;
6307 %}
6308
6309 // Multiply accumulate
6310 // Eg. MADD w0, w1, w2, w3
6311 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6312 %{
6313 single_instruction;
6314 dst : WR(write);
6315 src1 : ISS(read);
6316 src2 : ISS(read);
6317 src3 : ISS(read);
6318 INS01 : ISS;
6319 MAC : WR;
6320 %}
6321
6322 // Eg. MUL w0, w1, w2
6323 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6324 %{
6325 single_instruction;
6326 fixed_latency(3); // Maximum latency for 64 bit mul
6327 dst : WR(write);
6328 src1 : ISS(read);
6329 src2 : ISS(read);
6330 INS01 : ISS;
6331 MAC : WR;
6332 %}
6333
6334 // Multiply accumulate
6335 // Eg. MADD w0, w1, w2, w3
6336 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6337 %{
6338 single_instruction;
6339 fixed_latency(3); // Maximum latency for 64 bit mul
6340 dst : WR(write);
6341 src1 : ISS(read);
6342 src2 : ISS(read);
6343 src3 : ISS(read);
6344 INS01 : ISS;
6345 MAC : WR;
6346 %}
6347
6348 //------- Divide pipeline operations --------------------
6349
6350 // Eg. SDIV w0, w1, w2
6351 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6352 %{
6353 single_instruction;
6354 fixed_latency(8); // Maximum latency for 32 bit divide
6355 dst : WR(write);
6356 src1 : ISS(read);
6357 src2 : ISS(read);
6358 INS0 : ISS; // Can only dual issue as instruction 0
6359 DIV : WR;
6360 %}
6361
6362 // Eg. SDIV x0, x1, x2
6363 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6364 %{
6365 single_instruction;
6366 fixed_latency(16); // Maximum latency for 64 bit divide
6367 dst : WR(write);
6368 src1 : ISS(read);
6369 src2 : ISS(read);
6370 INS0 : ISS; // Can only dual issue as instruction 0
6371 DIV : WR;
6372 %}
6373
6374 //------- Load pipeline operations ------------------------
6375
6376 // Load - prefetch
6377 // Eg. PFRM <mem>
6378 pipe_class iload_prefetch(memory mem)
6379 %{
6380 single_instruction;
6381 mem : ISS(read);
6382 INS01 : ISS;
6383 LDST : WR;
6384 %}
6385
6386 // Load - reg, mem
6387 // Eg. LDR x0, <mem>
6388 pipe_class iload_reg_mem(iRegI dst, memory mem)
6389 %{
6390 single_instruction;
6391 dst : WR(write);
6392 mem : ISS(read);
6393 INS01 : ISS;
6394 LDST : WR;
6395 %}
6396
6397 // Load - reg, reg
6398 // Eg. LDR x0, [sp, x1]
6399 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6400 %{
6401 single_instruction;
6402 dst : WR(write);
6403 src : ISS(read);
6404 INS01 : ISS;
6405 LDST : WR;
6406 %}
6407
6408 //------- Store pipeline operations -----------------------
6409
6410 // Store - zr, mem
6411 // Eg. STR zr, <mem>
6412 pipe_class istore_mem(memory mem)
6413 %{
6414 single_instruction;
6415 mem : ISS(read);
6416 INS01 : ISS;
6417 LDST : WR;
6418 %}
6419
6420 // Store - reg, mem
6421 // Eg. STR x0, <mem>
6422 pipe_class istore_reg_mem(iRegI src, memory mem)
6423 %{
6424 single_instruction;
6425 mem : ISS(read);
6426 src : EX2(read);
6427 INS01 : ISS;
6428 LDST : WR;
6429 %}
6430
6431 // Store - reg, reg
6432 // Eg. STR x0, [sp, x1]
6433 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6434 %{
6435 single_instruction;
6436 dst : ISS(read);
6437 src : EX2(read);
6438 INS01 : ISS;
6439 LDST : WR;
6440 %}
6441
6442 //------- Store pipeline operations -----------------------
6443
6444 // Branch
6445 pipe_class pipe_branch()
6446 %{
6447 single_instruction;
6448 INS01 : ISS;
6449 BRANCH : EX1;
6450 %}
6451
6452 // Conditional branch
6453 pipe_class pipe_branch_cond(rFlagsReg cr)
6454 %{
6455 single_instruction;
6456 cr : EX1(read);
6457 INS01 : ISS;
6458 BRANCH : EX1;
6459 %}
6460
6461 // Compare & Branch
6462 // EG. CBZ/CBNZ
6463 pipe_class pipe_cmp_branch(iRegI op1)
6464 %{
6465 single_instruction;
6466 op1 : EX1(read);
6467 INS01 : ISS;
6468 BRANCH : EX1;
6469 %}
6470
6471 //------- Synchronisation operations ----------------------
6472
6473 // Any operation requiring serialization.
6474 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6475 pipe_class pipe_serial()
6476 %{
6477 single_instruction;
6478 force_serialization;
6479 fixed_latency(16);
6480 INS01 : ISS(2); // Cannot dual issue with any other instruction
6481 LDST : WR;
6482 %}
6483
6484 // Generic big/slow expanded idiom - also serialized
6485 pipe_class pipe_slow()
6486 %{
6487 instruction_count(10);
6488 multiple_bundles;
6489 force_serialization;
6490 fixed_latency(16);
6491 INS01 : ISS(2); // Cannot dual issue with any other instruction
6492 LDST : WR;
6493 %}
6494
6495 // Empty pipeline class
6496 pipe_class pipe_class_empty()
6497 %{
6498 single_instruction;
6499 fixed_latency(0);
6500 %}
6501
6502 // Default pipeline class.
6503 pipe_class pipe_class_default()
6504 %{
6505 single_instruction;
6506 fixed_latency(2);
6507 %}
6508
6509 // Pipeline class for compares.
6510 pipe_class pipe_class_compare()
6511 %{
6512 single_instruction;
6513 fixed_latency(16);
6514 %}
6515
6516 // Pipeline class for memory operations.
6517 pipe_class pipe_class_memory()
6518 %{
6519 single_instruction;
6520 fixed_latency(16);
6521 %}
6522
6523 // Pipeline class for call.
6524 pipe_class pipe_class_call()
6525 %{
6526 single_instruction;
6527 fixed_latency(100);
6528 %}
6529
6530 // Define the class for the Nop node.
6531 define %{
6532 MachNop = pipe_class_empty;
6533 %}
6534
6535 %}
6536 //----------INSTRUCTIONS-------------------------------------------------------
6537 //
6538 // match -- States which machine-independent subtree may be replaced
6539 // by this instruction.
6540 // ins_cost -- The estimated cost of this instruction is used by instruction
6541 // selection to identify a minimum cost tree of machine
6542 // instructions that matches a tree of machine-independent
6543 // instructions.
6544 // format -- A string providing the disassembly for this instruction.
6545 // The value of an instruction's operand may be inserted
6546 // by referring to it with a '$' prefix.
6547 // opcode -- Three instruction opcodes may be provided. These are referred
6548 // to within an encode class as $primary, $secondary, and $tertiary
6549 // rrspectively. The primary opcode is commonly used to
6550 // indicate the type of machine instruction, while secondary
6551 // and tertiary are often used for prefix options or addressing
6552 // modes.
6553 // ins_encode -- A list of encode classes with parameters. The encode class
6554 // name must have been defined in an 'enc_class' specification
6555 // in the encode section of the architecture description.
6556
6557 // ============================================================================
6558 // Memory (Load/Store) Instructions
6559
6560 // Load Instructions
6561
6562 // Load Byte (8 bit signed)
6563 instruct loadB(iRegINoSp dst, memory1 mem)
6564 %{
6565 match(Set dst (LoadB mem));
6566 predicate(!needs_acquiring_load(n));
6567
6568 ins_cost(4 * INSN_COST);
6569 format %{ "ldrsbw $dst, $mem\t# byte" %}
6570
6571 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6572
6573 ins_pipe(iload_reg_mem);
6574 %}
6575
6576 // Load Byte (8 bit signed) into long
6577 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6578 %{
6579 match(Set dst (ConvI2L (LoadB mem)));
6580 predicate(!needs_acquiring_load(n->in(1)));
6581
6582 ins_cost(4 * INSN_COST);
6583 format %{ "ldrsb $dst, $mem\t# byte" %}
6584
6585 ins_encode(aarch64_enc_ldrsb(dst, mem));
6586
6587 ins_pipe(iload_reg_mem);
6588 %}
6589
6590 // Load Byte (8 bit unsigned)
6591 instruct loadUB(iRegINoSp dst, memory1 mem)
6592 %{
6593 match(Set dst (LoadUB mem));
6594 predicate(!needs_acquiring_load(n));
6595
6596 ins_cost(4 * INSN_COST);
6597 format %{ "ldrbw $dst, $mem\t# byte" %}
6598
6599 ins_encode(aarch64_enc_ldrb(dst, mem));
6600
6601 ins_pipe(iload_reg_mem);
6602 %}
6603
6604 // Load Byte (8 bit unsigned) into long
6605 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6606 %{
6607 match(Set dst (ConvI2L (LoadUB mem)));
6608 predicate(!needs_acquiring_load(n->in(1)));
6609
6610 ins_cost(4 * INSN_COST);
6611 format %{ "ldrb $dst, $mem\t# byte" %}
6612
6613 ins_encode(aarch64_enc_ldrb(dst, mem));
6614
6615 ins_pipe(iload_reg_mem);
6616 %}
6617
6618 // Load Short (16 bit signed)
6619 instruct loadS(iRegINoSp dst, memory2 mem)
6620 %{
6621 match(Set dst (LoadS mem));
6622 predicate(!needs_acquiring_load(n));
6623
6624 ins_cost(4 * INSN_COST);
6625 format %{ "ldrshw $dst, $mem\t# short" %}
6626
6627 ins_encode(aarch64_enc_ldrshw(dst, mem));
6628
6629 ins_pipe(iload_reg_mem);
6630 %}
6631
6632 // Load Short (16 bit signed) into long
6633 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6634 %{
6635 match(Set dst (ConvI2L (LoadS mem)));
6636 predicate(!needs_acquiring_load(n->in(1)));
6637
6638 ins_cost(4 * INSN_COST);
6639 format %{ "ldrsh $dst, $mem\t# short" %}
6640
6641 ins_encode(aarch64_enc_ldrsh(dst, mem));
6642
6643 ins_pipe(iload_reg_mem);
6644 %}
6645
6646 // Load Char (16 bit unsigned)
6647 instruct loadUS(iRegINoSp dst, memory2 mem)
6648 %{
6649 match(Set dst (LoadUS mem));
6650 predicate(!needs_acquiring_load(n));
6651
6652 ins_cost(4 * INSN_COST);
6653 format %{ "ldrh $dst, $mem\t# short" %}
6654
6655 ins_encode(aarch64_enc_ldrh(dst, mem));
6656
6657 ins_pipe(iload_reg_mem);
6658 %}
6659
6660 // Load Short/Char (16 bit unsigned) into long
6661 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6662 %{
6663 match(Set dst (ConvI2L (LoadUS mem)));
6664 predicate(!needs_acquiring_load(n->in(1)));
6665
6666 ins_cost(4 * INSN_COST);
6667 format %{ "ldrh $dst, $mem\t# short" %}
6668
6669 ins_encode(aarch64_enc_ldrh(dst, mem));
6670
6671 ins_pipe(iload_reg_mem);
6672 %}
6673
6674 // Load Integer (32 bit signed)
6675 instruct loadI(iRegINoSp dst, memory4 mem)
6676 %{
6677 match(Set dst (LoadI mem));
6678 predicate(!needs_acquiring_load(n));
6679
6680 ins_cost(4 * INSN_COST);
6681 format %{ "ldrw $dst, $mem\t# int" %}
6682
6683 ins_encode(aarch64_enc_ldrw(dst, mem));
6684
6685 ins_pipe(iload_reg_mem);
6686 %}
6687
6688 // Load Integer (32 bit signed) into long
6689 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6690 %{
6691 match(Set dst (ConvI2L (LoadI mem)));
6692 predicate(!needs_acquiring_load(n->in(1)));
6693
6694 ins_cost(4 * INSN_COST);
6695 format %{ "ldrsw $dst, $mem\t# int" %}
6696
6697 ins_encode(aarch64_enc_ldrsw(dst, mem));
6698
6699 ins_pipe(iload_reg_mem);
6700 %}
6701
6702 // Load Integer (32 bit unsigned) into long
6703 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6704 %{
6705 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6706 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6707
6708 ins_cost(4 * INSN_COST);
6709 format %{ "ldrw $dst, $mem\t# int" %}
6710
6711 ins_encode(aarch64_enc_ldrw(dst, mem));
6712
6713 ins_pipe(iload_reg_mem);
6714 %}
6715
6716 // Load Long (64 bit signed)
6717 instruct loadL(iRegLNoSp dst, memory8 mem)
6718 %{
6719 match(Set dst (LoadL mem));
6720 predicate(!needs_acquiring_load(n));
6721
6722 ins_cost(4 * INSN_COST);
6723 format %{ "ldr $dst, $mem\t# int" %}
6724
6725 ins_encode(aarch64_enc_ldr(dst, mem));
6726
6727 ins_pipe(iload_reg_mem);
6728 %}
6729
6730 // Load Range
6731 instruct loadRange(iRegINoSp dst, memory4 mem)
6732 %{
6733 match(Set dst (LoadRange mem));
6734
6735 ins_cost(4 * INSN_COST);
6736 format %{ "ldrw $dst, $mem\t# range" %}
6737
6738 ins_encode(aarch64_enc_ldrw(dst, mem));
6739
6740 ins_pipe(iload_reg_mem);
6741 %}
6742
6743 // Load Pointer
6744 instruct loadP(iRegPNoSp dst, memory8 mem)
6745 %{
6746 match(Set dst (LoadP mem));
6747 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6748
6749 ins_cost(4 * INSN_COST);
6750 format %{ "ldr $dst, $mem\t# ptr" %}
6751
6752 ins_encode(aarch64_enc_ldr(dst, mem));
6753
6754 ins_pipe(iload_reg_mem);
6755 %}
6756
6757 // Load Compressed Pointer
6758 instruct loadN(iRegNNoSp dst, memory4 mem)
6759 %{
6760 match(Set dst (LoadN mem));
6761 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6762
6763 ins_cost(4 * INSN_COST);
6764 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6765
6766 ins_encode(aarch64_enc_ldrw(dst, mem));
6767
6768 ins_pipe(iload_reg_mem);
6769 %}
6770
6771 // Load Klass Pointer
6772 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6773 %{
6774 match(Set dst (LoadKlass mem));
6775 predicate(!needs_acquiring_load(n));
6776
6777 ins_cost(4 * INSN_COST);
6778 format %{ "ldr $dst, $mem\t# class" %}
6779
6780 ins_encode(aarch64_enc_ldr(dst, mem));
6781
6782 ins_pipe(iload_reg_mem);
6783 %}
6784
6785 // Load Narrow Klass Pointer
6786 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6787 %{
6788 match(Set dst (LoadNKlass mem));
6789 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6790
6791 ins_cost(4 * INSN_COST);
6792 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6793
6794 ins_encode(aarch64_enc_ldrw(dst, mem));
6795
6796 ins_pipe(iload_reg_mem);
6797 %}
6798
6799 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory_noindex mem)
6800 %{
6801 match(Set dst (LoadNKlass mem));
6802 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6803
6804 ins_cost(4 * INSN_COST);
6805 format %{
6806 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6807 "lsrw $dst, $dst, markWord::klass_shift"
6808 %}
6809 ins_encode %{
6810 assert($mem$$index$$Register == noreg, "must not have indexed address");
6811 // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
6812 // obj-start, so that we can load from the object's mark-word instead.
6813 __ ldrw($dst$$Register, Address($mem$$base$$Register, $mem$$disp - Type::klass_offset()));
6814 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift);
6815 %}
6816 ins_pipe(iload_reg_mem);
6817 %}
6818
6819 // Load Float
6820 instruct loadF(vRegF dst, memory4 mem)
6821 %{
6822 match(Set dst (LoadF mem));
6823 predicate(!needs_acquiring_load(n));
6824
6825 ins_cost(4 * INSN_COST);
6826 format %{ "ldrs $dst, $mem\t# float" %}
6827
6828 ins_encode( aarch64_enc_ldrs(dst, mem) );
6829
6830 ins_pipe(pipe_class_memory);
6831 %}
6832
6833 // Load Double
6834 instruct loadD(vRegD dst, memory8 mem)
6835 %{
6836 match(Set dst (LoadD mem));
6837 predicate(!needs_acquiring_load(n));
6838
6839 ins_cost(4 * INSN_COST);
6840 format %{ "ldrd $dst, $mem\t# double" %}
6841
6842 ins_encode( aarch64_enc_ldrd(dst, mem) );
6843
6844 ins_pipe(pipe_class_memory);
6845 %}
6846
6847
6848 // Load Int Constant
6849 instruct loadConI(iRegINoSp dst, immI src)
6850 %{
6851 match(Set dst src);
6852
6853 ins_cost(INSN_COST);
6854 format %{ "mov $dst, $src\t# int" %}
6855
6856 ins_encode( aarch64_enc_movw_imm(dst, src) );
6857
6858 ins_pipe(ialu_imm);
6859 %}
6860
6861 // Load Long Constant
6862 instruct loadConL(iRegLNoSp dst, immL src)
6863 %{
6864 match(Set dst src);
6865
6866 ins_cost(INSN_COST);
6867 format %{ "mov $dst, $src\t# long" %}
6868
6869 ins_encode( aarch64_enc_mov_imm(dst, src) );
6870
6871 ins_pipe(ialu_imm);
6872 %}
6873
6874 // Load Pointer Constant
6875
6876 instruct loadConP(iRegPNoSp dst, immP con)
6877 %{
6878 match(Set dst con);
6879
6880 ins_cost(INSN_COST * 4);
6881 format %{
6882 "mov $dst, $con\t# ptr\n\t"
6883 %}
6884
6885 ins_encode(aarch64_enc_mov_p(dst, con));
6886
6887 ins_pipe(ialu_imm);
6888 %}
6889
6890 // Load Null Pointer Constant
6891
6892 instruct loadConP0(iRegPNoSp dst, immP0 con)
6893 %{
6894 match(Set dst con);
6895
6896 ins_cost(INSN_COST);
6897 format %{ "mov $dst, $con\t# nullptr ptr" %}
6898
6899 ins_encode(aarch64_enc_mov_p0(dst, con));
6900
6901 ins_pipe(ialu_imm);
6902 %}
6903
6904 // Load Pointer Constant One
6905
6906 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6907 %{
6908 match(Set dst con);
6909
6910 ins_cost(INSN_COST);
6911 format %{ "mov $dst, $con\t# nullptr ptr" %}
6912
6913 ins_encode(aarch64_enc_mov_p1(dst, con));
6914
6915 ins_pipe(ialu_imm);
6916 %}
6917
6918 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6919 %{
6920 match(Set dst con);
6921
6922 ins_cost(INSN_COST);
6923 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6924
6925 ins_encode %{
6926 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6927 %}
6928
6929 ins_pipe(ialu_imm);
6930 %}
6931
6932 // Load Narrow Pointer Constant
6933
6934 instruct loadConN(iRegNNoSp dst, immN con)
6935 %{
6936 match(Set dst con);
6937
6938 ins_cost(INSN_COST * 4);
6939 format %{ "mov $dst, $con\t# compressed ptr" %}
6940
6941 ins_encode(aarch64_enc_mov_n(dst, con));
6942
6943 ins_pipe(ialu_imm);
6944 %}
6945
6946 // Load Narrow Null Pointer Constant
6947
6948 instruct loadConN0(iRegNNoSp dst, immN0 con)
6949 %{
6950 match(Set dst con);
6951
6952 ins_cost(INSN_COST);
6953 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6954
6955 ins_encode(aarch64_enc_mov_n0(dst, con));
6956
6957 ins_pipe(ialu_imm);
6958 %}
6959
6960 // Load Narrow Klass Constant
6961
6962 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6963 %{
6964 match(Set dst con);
6965
6966 ins_cost(INSN_COST);
6967 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6968
6969 ins_encode(aarch64_enc_mov_nk(dst, con));
6970
6971 ins_pipe(ialu_imm);
6972 %}
6973
6974 // Load Packed Float Constant
6975
6976 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6977 match(Set dst con);
6978 ins_cost(INSN_COST * 4);
6979 format %{ "fmovs $dst, $con"%}
6980 ins_encode %{
6981 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6982 %}
6983
6984 ins_pipe(fp_imm_s);
6985 %}
6986
6987 // Load Float Constant
6988
6989 instruct loadConF(vRegF dst, immF con) %{
6990 match(Set dst con);
6991
6992 ins_cost(INSN_COST * 4);
6993
6994 format %{
6995 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6996 %}
6997
6998 ins_encode %{
6999 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7000 %}
7001
7002 ins_pipe(fp_load_constant_s);
7003 %}
7004
7005 // Load Packed Double Constant
7006
7007 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7008 match(Set dst con);
7009 ins_cost(INSN_COST);
7010 format %{ "fmovd $dst, $con"%}
7011 ins_encode %{
7012 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7013 %}
7014
7015 ins_pipe(fp_imm_d);
7016 %}
7017
7018 // Load Double Constant
7019
7020 instruct loadConD(vRegD dst, immD con) %{
7021 match(Set dst con);
7022
7023 ins_cost(INSN_COST * 5);
7024 format %{
7025 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7026 %}
7027
7028 ins_encode %{
7029 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7030 %}
7031
7032 ins_pipe(fp_load_constant_d);
7033 %}
7034
7035 // Load Half Float Constant
7036 instruct loadConH(vRegF dst, immH con) %{
7037 match(Set dst con);
7038 format %{ "mov rscratch1, $con\n\t"
7039 "fmov $dst, rscratch1"
7040 %}
7041 ins_encode %{
7042 __ movw(rscratch1, (uint32_t)$con$$constant);
7043 __ fmovs($dst$$FloatRegister, rscratch1);
7044 %}
7045 ins_pipe(pipe_class_default);
7046 %}
7047
7048 // Store Instructions
7049
7050 // Store Byte
7051 instruct storeB(iRegIorL2I src, memory1 mem)
7052 %{
7053 match(Set mem (StoreB mem src));
7054 predicate(!needs_releasing_store(n));
7055
7056 ins_cost(INSN_COST);
7057 format %{ "strb $src, $mem\t# byte" %}
7058
7059 ins_encode(aarch64_enc_strb(src, mem));
7060
7061 ins_pipe(istore_reg_mem);
7062 %}
7063
7064
7065 instruct storeimmB0(immI0 zero, memory1 mem)
7066 %{
7067 match(Set mem (StoreB mem zero));
7068 predicate(!needs_releasing_store(n));
7069
7070 ins_cost(INSN_COST);
7071 format %{ "strb rscractch2, $mem\t# byte" %}
7072
7073 ins_encode(aarch64_enc_strb0(mem));
7074
7075 ins_pipe(istore_mem);
7076 %}
7077
7078 // Store Char/Short
7079 instruct storeC(iRegIorL2I src, memory2 mem)
7080 %{
7081 match(Set mem (StoreC mem src));
7082 predicate(!needs_releasing_store(n));
7083
7084 ins_cost(INSN_COST);
7085 format %{ "strh $src, $mem\t# short" %}
7086
7087 ins_encode(aarch64_enc_strh(src, mem));
7088
7089 ins_pipe(istore_reg_mem);
7090 %}
7091
7092 instruct storeimmC0(immI0 zero, memory2 mem)
7093 %{
7094 match(Set mem (StoreC mem zero));
7095 predicate(!needs_releasing_store(n));
7096
7097 ins_cost(INSN_COST);
7098 format %{ "strh zr, $mem\t# short" %}
7099
7100 ins_encode(aarch64_enc_strh0(mem));
7101
7102 ins_pipe(istore_mem);
7103 %}
7104
7105 // Store Integer
7106
7107 instruct storeI(iRegIorL2I src, memory4 mem)
7108 %{
7109 match(Set mem(StoreI mem src));
7110 predicate(!needs_releasing_store(n));
7111
7112 ins_cost(INSN_COST);
7113 format %{ "strw $src, $mem\t# int" %}
7114
7115 ins_encode(aarch64_enc_strw(src, mem));
7116
7117 ins_pipe(istore_reg_mem);
7118 %}
7119
7120 instruct storeimmI0(immI0 zero, memory4 mem)
7121 %{
7122 match(Set mem(StoreI mem zero));
7123 predicate(!needs_releasing_store(n));
7124
7125 ins_cost(INSN_COST);
7126 format %{ "strw zr, $mem\t# int" %}
7127
7128 ins_encode(aarch64_enc_strw0(mem));
7129
7130 ins_pipe(istore_mem);
7131 %}
7132
7133 // Store Long (64 bit signed)
7134 instruct storeL(iRegL src, memory8 mem)
7135 %{
7136 match(Set mem (StoreL mem src));
7137 predicate(!needs_releasing_store(n));
7138
7139 ins_cost(INSN_COST);
7140 format %{ "str $src, $mem\t# int" %}
7141
7142 ins_encode(aarch64_enc_str(src, mem));
7143
7144 ins_pipe(istore_reg_mem);
7145 %}
7146
7147 // Store Long (64 bit signed)
7148 instruct storeimmL0(immL0 zero, memory8 mem)
7149 %{
7150 match(Set mem (StoreL mem zero));
7151 predicate(!needs_releasing_store(n));
7152
7153 ins_cost(INSN_COST);
7154 format %{ "str zr, $mem\t# int" %}
7155
7156 ins_encode(aarch64_enc_str0(mem));
7157
7158 ins_pipe(istore_mem);
7159 %}
7160
7161 // Store Pointer
7162 instruct storeP(iRegP src, memory8 mem)
7163 %{
7164 match(Set mem (StoreP mem src));
7165 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7166
7167 ins_cost(INSN_COST);
7168 format %{ "str $src, $mem\t# ptr" %}
7169
7170 ins_encode(aarch64_enc_str(src, mem));
7171
7172 ins_pipe(istore_reg_mem);
7173 %}
7174
7175 // Store Pointer
7176 instruct storeimmP0(immP0 zero, memory8 mem)
7177 %{
7178 match(Set mem (StoreP mem zero));
7179 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7180
7181 ins_cost(INSN_COST);
7182 format %{ "str zr, $mem\t# ptr" %}
7183
7184 ins_encode(aarch64_enc_str0(mem));
7185
7186 ins_pipe(istore_mem);
7187 %}
7188
7189 // Store Compressed Pointer
7190 instruct storeN(iRegN src, memory4 mem)
7191 %{
7192 match(Set mem (StoreN mem src));
7193 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7194
7195 ins_cost(INSN_COST);
7196 format %{ "strw $src, $mem\t# compressed ptr" %}
7197
7198 ins_encode(aarch64_enc_strw(src, mem));
7199
7200 ins_pipe(istore_reg_mem);
7201 %}
7202
7203 instruct storeImmN0(immN0 zero, memory4 mem)
7204 %{
7205 match(Set mem (StoreN mem zero));
7206 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7207
7208 ins_cost(INSN_COST);
7209 format %{ "strw zr, $mem\t# compressed ptr" %}
7210
7211 ins_encode(aarch64_enc_strw0(mem));
7212
7213 ins_pipe(istore_mem);
7214 %}
7215
7216 // Store Float
7217 instruct storeF(vRegF src, memory4 mem)
7218 %{
7219 match(Set mem (StoreF mem src));
7220 predicate(!needs_releasing_store(n));
7221
7222 ins_cost(INSN_COST);
7223 format %{ "strs $src, $mem\t# float" %}
7224
7225 ins_encode( aarch64_enc_strs(src, mem) );
7226
7227 ins_pipe(pipe_class_memory);
7228 %}
7229
7230 // TODO
7231 // implement storeImmF0 and storeFImmPacked
7232
7233 // Store Double
7234 instruct storeD(vRegD src, memory8 mem)
7235 %{
7236 match(Set mem (StoreD mem src));
7237 predicate(!needs_releasing_store(n));
7238
7239 ins_cost(INSN_COST);
7240 format %{ "strd $src, $mem\t# double" %}
7241
7242 ins_encode( aarch64_enc_strd(src, mem) );
7243
7244 ins_pipe(pipe_class_memory);
7245 %}
7246
7247 // Store Compressed Klass Pointer
7248 instruct storeNKlass(iRegN src, memory4 mem)
7249 %{
7250 predicate(!needs_releasing_store(n));
7251 match(Set mem (StoreNKlass mem src));
7252
7253 ins_cost(INSN_COST);
7254 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7255
7256 ins_encode(aarch64_enc_strw(src, mem));
7257
7258 ins_pipe(istore_reg_mem);
7259 %}
7260
7261 // TODO
7262 // implement storeImmD0 and storeDImmPacked
7263
7264 // prefetch instructions
7265 // Must be safe to execute with invalid address (cannot fault).
7266
7267 instruct prefetchalloc( memory8 mem ) %{
7268 match(PrefetchAllocation mem);
7269
7270 ins_cost(INSN_COST);
7271 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7272
7273 ins_encode( aarch64_enc_prefetchw(mem) );
7274
7275 ins_pipe(iload_prefetch);
7276 %}
7277
7278 // ---------------- volatile loads and stores ----------------
7279
7280 // Load Byte (8 bit signed)
7281 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7282 %{
7283 match(Set dst (LoadB mem));
7284
7285 ins_cost(VOLATILE_REF_COST);
7286 format %{ "ldarsb $dst, $mem\t# byte" %}
7287
7288 ins_encode(aarch64_enc_ldarsb(dst, mem));
7289
7290 ins_pipe(pipe_serial);
7291 %}
7292
7293 // Load Byte (8 bit signed) into long
7294 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7295 %{
7296 match(Set dst (ConvI2L (LoadB mem)));
7297
7298 ins_cost(VOLATILE_REF_COST);
7299 format %{ "ldarsb $dst, $mem\t# byte" %}
7300
7301 ins_encode(aarch64_enc_ldarsb(dst, mem));
7302
7303 ins_pipe(pipe_serial);
7304 %}
7305
7306 // Load Byte (8 bit unsigned)
7307 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7308 %{
7309 match(Set dst (LoadUB mem));
7310
7311 ins_cost(VOLATILE_REF_COST);
7312 format %{ "ldarb $dst, $mem\t# byte" %}
7313
7314 ins_encode(aarch64_enc_ldarb(dst, mem));
7315
7316 ins_pipe(pipe_serial);
7317 %}
7318
7319 // Load Byte (8 bit unsigned) into long
7320 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7321 %{
7322 match(Set dst (ConvI2L (LoadUB mem)));
7323
7324 ins_cost(VOLATILE_REF_COST);
7325 format %{ "ldarb $dst, $mem\t# byte" %}
7326
7327 ins_encode(aarch64_enc_ldarb(dst, mem));
7328
7329 ins_pipe(pipe_serial);
7330 %}
7331
7332 // Load Short (16 bit signed)
7333 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7334 %{
7335 match(Set dst (LoadS mem));
7336
7337 ins_cost(VOLATILE_REF_COST);
7338 format %{ "ldarshw $dst, $mem\t# short" %}
7339
7340 ins_encode(aarch64_enc_ldarshw(dst, mem));
7341
7342 ins_pipe(pipe_serial);
7343 %}
7344
7345 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7346 %{
7347 match(Set dst (LoadUS mem));
7348
7349 ins_cost(VOLATILE_REF_COST);
7350 format %{ "ldarhw $dst, $mem\t# short" %}
7351
7352 ins_encode(aarch64_enc_ldarhw(dst, mem));
7353
7354 ins_pipe(pipe_serial);
7355 %}
7356
7357 // Load Short/Char (16 bit unsigned) into long
7358 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7359 %{
7360 match(Set dst (ConvI2L (LoadUS mem)));
7361
7362 ins_cost(VOLATILE_REF_COST);
7363 format %{ "ldarh $dst, $mem\t# short" %}
7364
7365 ins_encode(aarch64_enc_ldarh(dst, mem));
7366
7367 ins_pipe(pipe_serial);
7368 %}
7369
7370 // Load Short/Char (16 bit signed) into long
7371 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7372 %{
7373 match(Set dst (ConvI2L (LoadS mem)));
7374
7375 ins_cost(VOLATILE_REF_COST);
7376 format %{ "ldarh $dst, $mem\t# short" %}
7377
7378 ins_encode(aarch64_enc_ldarsh(dst, mem));
7379
7380 ins_pipe(pipe_serial);
7381 %}
7382
7383 // Load Integer (32 bit signed)
7384 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7385 %{
7386 match(Set dst (LoadI mem));
7387
7388 ins_cost(VOLATILE_REF_COST);
7389 format %{ "ldarw $dst, $mem\t# int" %}
7390
7391 ins_encode(aarch64_enc_ldarw(dst, mem));
7392
7393 ins_pipe(pipe_serial);
7394 %}
7395
7396 // Load Integer (32 bit unsigned) into long
7397 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7398 %{
7399 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7400
7401 ins_cost(VOLATILE_REF_COST);
7402 format %{ "ldarw $dst, $mem\t# int" %}
7403
7404 ins_encode(aarch64_enc_ldarw(dst, mem));
7405
7406 ins_pipe(pipe_serial);
7407 %}
7408
7409 // Load Long (64 bit signed)
7410 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7411 %{
7412 match(Set dst (LoadL mem));
7413
7414 ins_cost(VOLATILE_REF_COST);
7415 format %{ "ldar $dst, $mem\t# int" %}
7416
7417 ins_encode(aarch64_enc_ldar(dst, mem));
7418
7419 ins_pipe(pipe_serial);
7420 %}
7421
7422 // Load Pointer
7423 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7424 %{
7425 match(Set dst (LoadP mem));
7426 predicate(n->as_Load()->barrier_data() == 0);
7427
7428 ins_cost(VOLATILE_REF_COST);
7429 format %{ "ldar $dst, $mem\t# ptr" %}
7430
7431 ins_encode(aarch64_enc_ldar(dst, mem));
7432
7433 ins_pipe(pipe_serial);
7434 %}
7435
7436 // Load Compressed Pointer
7437 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7438 %{
7439 match(Set dst (LoadN mem));
7440 predicate(n->as_Load()->barrier_data() == 0);
7441
7442 ins_cost(VOLATILE_REF_COST);
7443 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7444
7445 ins_encode(aarch64_enc_ldarw(dst, mem));
7446
7447 ins_pipe(pipe_serial);
7448 %}
7449
7450 // Load Float
7451 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7452 %{
7453 match(Set dst (LoadF mem));
7454
7455 ins_cost(VOLATILE_REF_COST);
7456 format %{ "ldars $dst, $mem\t# float" %}
7457
7458 ins_encode( aarch64_enc_fldars(dst, mem) );
7459
7460 ins_pipe(pipe_serial);
7461 %}
7462
7463 // Load Double
7464 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7465 %{
7466 match(Set dst (LoadD mem));
7467
7468 ins_cost(VOLATILE_REF_COST);
7469 format %{ "ldard $dst, $mem\t# double" %}
7470
7471 ins_encode( aarch64_enc_fldard(dst, mem) );
7472
7473 ins_pipe(pipe_serial);
7474 %}
7475
7476 // Store Byte
7477 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7478 %{
7479 match(Set mem (StoreB mem src));
7480
7481 ins_cost(VOLATILE_REF_COST);
7482 format %{ "stlrb $src, $mem\t# byte" %}
7483
7484 ins_encode(aarch64_enc_stlrb(src, mem));
7485
7486 ins_pipe(pipe_class_memory);
7487 %}
7488
7489 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7490 %{
7491 match(Set mem (StoreB mem zero));
7492
7493 ins_cost(VOLATILE_REF_COST);
7494 format %{ "stlrb zr, $mem\t# byte" %}
7495
7496 ins_encode(aarch64_enc_stlrb0(mem));
7497
7498 ins_pipe(pipe_class_memory);
7499 %}
7500
7501 // Store Char/Short
7502 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7503 %{
7504 match(Set mem (StoreC mem src));
7505
7506 ins_cost(VOLATILE_REF_COST);
7507 format %{ "stlrh $src, $mem\t# short" %}
7508
7509 ins_encode(aarch64_enc_stlrh(src, mem));
7510
7511 ins_pipe(pipe_class_memory);
7512 %}
7513
7514 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7515 %{
7516 match(Set mem (StoreC mem zero));
7517
7518 ins_cost(VOLATILE_REF_COST);
7519 format %{ "stlrh zr, $mem\t# short" %}
7520
7521 ins_encode(aarch64_enc_stlrh0(mem));
7522
7523 ins_pipe(pipe_class_memory);
7524 %}
7525
7526 // Store Integer
7527
7528 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7529 %{
7530 match(Set mem(StoreI mem src));
7531
7532 ins_cost(VOLATILE_REF_COST);
7533 format %{ "stlrw $src, $mem\t# int" %}
7534
7535 ins_encode(aarch64_enc_stlrw(src, mem));
7536
7537 ins_pipe(pipe_class_memory);
7538 %}
7539
7540 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7541 %{
7542 match(Set mem(StoreI mem zero));
7543
7544 ins_cost(VOLATILE_REF_COST);
7545 format %{ "stlrw zr, $mem\t# int" %}
7546
7547 ins_encode(aarch64_enc_stlrw0(mem));
7548
7549 ins_pipe(pipe_class_memory);
7550 %}
7551
7552 // Store Long (64 bit signed)
7553 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7554 %{
7555 match(Set mem (StoreL mem src));
7556
7557 ins_cost(VOLATILE_REF_COST);
7558 format %{ "stlr $src, $mem\t# int" %}
7559
7560 ins_encode(aarch64_enc_stlr(src, mem));
7561
7562 ins_pipe(pipe_class_memory);
7563 %}
7564
7565 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7566 %{
7567 match(Set mem (StoreL mem zero));
7568
7569 ins_cost(VOLATILE_REF_COST);
7570 format %{ "stlr zr, $mem\t# int" %}
7571
7572 ins_encode(aarch64_enc_stlr0(mem));
7573
7574 ins_pipe(pipe_class_memory);
7575 %}
7576
7577 // Store Pointer
7578 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7579 %{
7580 match(Set mem (StoreP mem src));
7581 predicate(n->as_Store()->barrier_data() == 0);
7582
7583 ins_cost(VOLATILE_REF_COST);
7584 format %{ "stlr $src, $mem\t# ptr" %}
7585
7586 ins_encode(aarch64_enc_stlr(src, mem));
7587
7588 ins_pipe(pipe_class_memory);
7589 %}
7590
7591 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7592 %{
7593 match(Set mem (StoreP mem zero));
7594 predicate(n->as_Store()->barrier_data() == 0);
7595
7596 ins_cost(VOLATILE_REF_COST);
7597 format %{ "stlr zr, $mem\t# ptr" %}
7598
7599 ins_encode(aarch64_enc_stlr0(mem));
7600
7601 ins_pipe(pipe_class_memory);
7602 %}
7603
7604 // Store Compressed Pointer
7605 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7606 %{
7607 match(Set mem (StoreN mem src));
7608 predicate(n->as_Store()->barrier_data() == 0);
7609
7610 ins_cost(VOLATILE_REF_COST);
7611 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7612
7613 ins_encode(aarch64_enc_stlrw(src, mem));
7614
7615 ins_pipe(pipe_class_memory);
7616 %}
7617
7618 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7619 %{
7620 match(Set mem (StoreN mem zero));
7621 predicate(n->as_Store()->barrier_data() == 0);
7622
7623 ins_cost(VOLATILE_REF_COST);
7624 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7625
7626 ins_encode(aarch64_enc_stlrw0(mem));
7627
7628 ins_pipe(pipe_class_memory);
7629 %}
7630
7631 // Store Float
7632 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7633 %{
7634 match(Set mem (StoreF mem src));
7635
7636 ins_cost(VOLATILE_REF_COST);
7637 format %{ "stlrs $src, $mem\t# float" %}
7638
7639 ins_encode( aarch64_enc_fstlrs(src, mem) );
7640
7641 ins_pipe(pipe_class_memory);
7642 %}
7643
7644 // TODO
7645 // implement storeImmF0 and storeFImmPacked
7646
7647 // Store Double
7648 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7649 %{
7650 match(Set mem (StoreD mem src));
7651
7652 ins_cost(VOLATILE_REF_COST);
7653 format %{ "stlrd $src, $mem\t# double" %}
7654
7655 ins_encode( aarch64_enc_fstlrd(src, mem) );
7656
7657 ins_pipe(pipe_class_memory);
7658 %}
7659
7660 // ---------------- end of volatile loads and stores ----------------
7661
7662 instruct cacheWB(indirect addr)
7663 %{
7664 predicate(VM_Version::supports_data_cache_line_flush());
7665 match(CacheWB addr);
7666
7667 ins_cost(100);
7668 format %{"cache wb $addr" %}
7669 ins_encode %{
7670 assert($addr->index_position() < 0, "should be");
7671 assert($addr$$disp == 0, "should be");
7672 __ cache_wb(Address($addr$$base$$Register, 0));
7673 %}
7674 ins_pipe(pipe_slow); // XXX
7675 %}
7676
7677 instruct cacheWBPreSync()
7678 %{
7679 predicate(VM_Version::supports_data_cache_line_flush());
7680 match(CacheWBPreSync);
7681
7682 ins_cost(100);
7683 format %{"cache wb presync" %}
7684 ins_encode %{
7685 __ cache_wbsync(true);
7686 %}
7687 ins_pipe(pipe_slow); // XXX
7688 %}
7689
7690 instruct cacheWBPostSync()
7691 %{
7692 predicate(VM_Version::supports_data_cache_line_flush());
7693 match(CacheWBPostSync);
7694
7695 ins_cost(100);
7696 format %{"cache wb postsync" %}
7697 ins_encode %{
7698 __ cache_wbsync(false);
7699 %}
7700 ins_pipe(pipe_slow); // XXX
7701 %}
7702
7703 // ============================================================================
7704 // BSWAP Instructions
7705
7706 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7707 match(Set dst (ReverseBytesI src));
7708
7709 ins_cost(INSN_COST);
7710 format %{ "revw $dst, $src" %}
7711
7712 ins_encode %{
7713 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7714 %}
7715
7716 ins_pipe(ialu_reg);
7717 %}
7718
7719 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7720 match(Set dst (ReverseBytesL src));
7721
7722 ins_cost(INSN_COST);
7723 format %{ "rev $dst, $src" %}
7724
7725 ins_encode %{
7726 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7727 %}
7728
7729 ins_pipe(ialu_reg);
7730 %}
7731
7732 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7733 match(Set dst (ReverseBytesUS src));
7734
7735 ins_cost(INSN_COST);
7736 format %{ "rev16w $dst, $src\t# $dst -> unsigned short" %}
7737
7738 ins_encode %{
7739 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7740 __ narrow_subword_type(as_Register($dst$$reg), T_CHAR);
7741 %}
7742
7743 ins_pipe(ialu_reg);
7744 %}
7745
7746 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7747 match(Set dst (ReverseBytesS src));
7748
7749 ins_cost(INSN_COST);
7750 format %{ "rev16w $dst, $src\n\t"
7751 "sbfmw $dst, $dst, #0, #15" %}
7752
7753 ins_encode %{
7754 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7755 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7756 %}
7757
7758 ins_pipe(ialu_reg);
7759 %}
7760
7761 // ============================================================================
7762 // Zero Count Instructions
7763
7764 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7765 match(Set dst (CountLeadingZerosI src));
7766
7767 ins_cost(INSN_COST);
7768 format %{ "clzw $dst, $src" %}
7769 ins_encode %{
7770 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7771 %}
7772
7773 ins_pipe(ialu_reg);
7774 %}
7775
7776 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7777 match(Set dst (CountLeadingZerosL src));
7778
7779 ins_cost(INSN_COST);
7780 format %{ "clz $dst, $src" %}
7781 ins_encode %{
7782 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7783 %}
7784
7785 ins_pipe(ialu_reg);
7786 %}
7787
7788 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7789 match(Set dst (CountTrailingZerosI src));
7790
7791 ins_cost(INSN_COST * 2);
7792 format %{ "rbitw $dst, $src\n\t"
7793 "clzw $dst, $dst" %}
7794 ins_encode %{
7795 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7796 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7797 %}
7798
7799 ins_pipe(ialu_reg);
7800 %}
7801
7802 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7803 match(Set dst (CountTrailingZerosL src));
7804
7805 ins_cost(INSN_COST * 2);
7806 format %{ "rbit $dst, $src\n\t"
7807 "clz $dst, $dst" %}
7808 ins_encode %{
7809 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7810 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7811 %}
7812
7813 ins_pipe(ialu_reg);
7814 %}
7815
7816 //---------- Population Count Instructions -------------------------------------
7817 //
7818
7819 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7820 match(Set dst (PopCountI src));
7821 effect(TEMP tmp);
7822 ins_cost(INSN_COST * 13);
7823
7824 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7825 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7826 "addv $tmp, $tmp\t# vector (8B)\n\t"
7827 "mov $dst, $tmp\t# vector (1D)" %}
7828 ins_encode %{
7829 __ fmovs($tmp$$FloatRegister, $src$$Register);
7830 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7831 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7832 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7833 %}
7834
7835 ins_pipe(pipe_class_default);
7836 %}
7837
7838 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7839 match(Set dst (PopCountI (LoadI mem)));
7840 effect(TEMP tmp);
7841 ins_cost(INSN_COST * 13);
7842
7843 format %{ "ldrs $tmp, $mem\n\t"
7844 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7845 "addv $tmp, $tmp\t# vector (8B)\n\t"
7846 "mov $dst, $tmp\t# vector (1D)" %}
7847 ins_encode %{
7848 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7849 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7850 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7851 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7852 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7853 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7854 %}
7855
7856 ins_pipe(pipe_class_default);
7857 %}
7858
7859 // Note: Long.bitCount(long) returns an int.
7860 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7861 match(Set dst (PopCountL src));
7862 effect(TEMP tmp);
7863 ins_cost(INSN_COST * 13);
7864
7865 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7866 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7867 "addv $tmp, $tmp\t# vector (8B)\n\t"
7868 "mov $dst, $tmp\t# vector (1D)" %}
7869 ins_encode %{
7870 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7871 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7872 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7873 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7874 %}
7875
7876 ins_pipe(pipe_class_default);
7877 %}
7878
7879 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7880 match(Set dst (PopCountL (LoadL mem)));
7881 effect(TEMP tmp);
7882 ins_cost(INSN_COST * 13);
7883
7884 format %{ "ldrd $tmp, $mem\n\t"
7885 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7886 "addv $tmp, $tmp\t# vector (8B)\n\t"
7887 "mov $dst, $tmp\t# vector (1D)" %}
7888 ins_encode %{
7889 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7890 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7891 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7892 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7893 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7894 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7895 %}
7896
7897 ins_pipe(pipe_class_default);
7898 %}
7899
7900 // ============================================================================
7901 // VerifyVectorAlignment Instruction
7902
7903 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7904 match(Set addr (VerifyVectorAlignment addr mask));
7905 effect(KILL cr);
7906 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7907 ins_encode %{
7908 Label Lskip;
7909 // check if masked bits of addr are zero
7910 __ tst($addr$$Register, $mask$$constant);
7911 __ br(Assembler::EQ, Lskip);
7912 __ stop("verify_vector_alignment found a misaligned vector memory access");
7913 __ bind(Lskip);
7914 %}
7915 ins_pipe(pipe_slow);
7916 %}
7917
7918 // ============================================================================
7919 // MemBar Instruction
7920
7921 instruct load_fence() %{
7922 match(LoadFence);
7923 ins_cost(VOLATILE_REF_COST);
7924
7925 format %{ "load_fence" %}
7926
7927 ins_encode %{
7928 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7929 %}
7930 ins_pipe(pipe_serial);
7931 %}
7932
7933 instruct unnecessary_membar_acquire() %{
7934 predicate(unnecessary_acquire(n));
7935 match(MemBarAcquire);
7936 ins_cost(0);
7937
7938 format %{ "membar_acquire (elided)" %}
7939
7940 ins_encode %{
7941 __ block_comment("membar_acquire (elided)");
7942 %}
7943
7944 ins_pipe(pipe_class_empty);
7945 %}
7946
7947 instruct membar_acquire() %{
7948 match(MemBarAcquire);
7949 ins_cost(VOLATILE_REF_COST);
7950
7951 format %{ "membar_acquire\n\t"
7952 "dmb ishld" %}
7953
7954 ins_encode %{
7955 __ block_comment("membar_acquire");
7956 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7957 %}
7958
7959 ins_pipe(pipe_serial);
7960 %}
7961
7962
7963 instruct membar_acquire_lock() %{
7964 match(MemBarAcquireLock);
7965 ins_cost(VOLATILE_REF_COST);
7966
7967 format %{ "membar_acquire_lock (elided)" %}
7968
7969 ins_encode %{
7970 __ block_comment("membar_acquire_lock (elided)");
7971 %}
7972
7973 ins_pipe(pipe_serial);
7974 %}
7975
7976 instruct store_fence() %{
7977 match(StoreFence);
7978 ins_cost(VOLATILE_REF_COST);
7979
7980 format %{ "store_fence" %}
7981
7982 ins_encode %{
7983 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7984 %}
7985 ins_pipe(pipe_serial);
7986 %}
7987
7988 instruct unnecessary_membar_release() %{
7989 predicate(unnecessary_release(n));
7990 match(MemBarRelease);
7991 ins_cost(0);
7992
7993 format %{ "membar_release (elided)" %}
7994
7995 ins_encode %{
7996 __ block_comment("membar_release (elided)");
7997 %}
7998 ins_pipe(pipe_serial);
7999 %}
8000
8001 instruct membar_release() %{
8002 match(MemBarRelease);
8003 ins_cost(VOLATILE_REF_COST);
8004
8005 format %{ "membar_release\n\t"
8006 "dmb ishst\n\tdmb ishld" %}
8007
8008 ins_encode %{
8009 __ block_comment("membar_release");
8010 // These will be merged if AlwaysMergeDMB is enabled.
8011 __ membar(Assembler::StoreStore);
8012 __ membar(Assembler::LoadStore);
8013 %}
8014 ins_pipe(pipe_serial);
8015 %}
8016
8017 instruct membar_storestore() %{
8018 match(MemBarStoreStore);
8019 match(StoreStoreFence);
8020 ins_cost(VOLATILE_REF_COST);
8021
8022 format %{ "MEMBAR-store-store" %}
8023
8024 ins_encode %{
8025 __ membar(Assembler::StoreStore);
8026 %}
8027 ins_pipe(pipe_serial);
8028 %}
8029
8030 instruct membar_release_lock() %{
8031 match(MemBarReleaseLock);
8032 ins_cost(VOLATILE_REF_COST);
8033
8034 format %{ "membar_release_lock (elided)" %}
8035
8036 ins_encode %{
8037 __ block_comment("membar_release_lock (elided)");
8038 %}
8039
8040 ins_pipe(pipe_serial);
8041 %}
8042
8043 instruct membar_storeload() %{
8044 match(MemBarStoreLoad);
8045 ins_cost(VOLATILE_REF_COST*100);
8046
8047 format %{ "MEMBAR-store-load\n\t"
8048 "dmb ish" %}
8049
8050 ins_encode %{
8051 __ block_comment("membar_storeload");
8052 __ membar(Assembler::StoreLoad);
8053 %}
8054
8055 ins_pipe(pipe_serial);
8056 %}
8057
8058 instruct unnecessary_membar_volatile() %{
8059 predicate(unnecessary_volatile(n));
8060 match(MemBarVolatile);
8061 ins_cost(0);
8062
8063 format %{ "membar_volatile (elided)" %}
8064
8065 ins_encode %{
8066 __ block_comment("membar_volatile (elided)");
8067 %}
8068
8069 ins_pipe(pipe_serial);
8070 %}
8071
8072 instruct membar_volatile() %{
8073 match(MemBarVolatile);
8074 ins_cost(VOLATILE_REF_COST*100);
8075
8076 format %{ "membar_volatile\n\t"
8077 "dmb ish"%}
8078
8079 ins_encode %{
8080 __ block_comment("membar_volatile");
8081 __ membar(Assembler::StoreLoad);
8082 %}
8083
8084 ins_pipe(pipe_serial);
8085 %}
8086
8087 instruct membar_full() %{
8088 match(MemBarFull);
8089 ins_cost(VOLATILE_REF_COST*100);
8090
8091 format %{ "membar_full\n\t"
8092 "dmb ish" %}
8093 ins_encode %{
8094 __ block_comment("membar_full");
8095 __ membar(Assembler::AnyAny);
8096 %}
8097
8098 ins_pipe(pipe_serial);
8099 %}
8100
8101 // ============================================================================
8102 // Cast/Convert Instructions
8103
8104 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8105 match(Set dst (CastX2P src));
8106
8107 ins_cost(INSN_COST);
8108 format %{ "mov $dst, $src\t# long -> ptr" %}
8109
8110 ins_encode %{
8111 if ($dst$$reg != $src$$reg) {
8112 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8113 }
8114 %}
8115
8116 ins_pipe(ialu_reg);
8117 %}
8118
8119 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8120 match(Set dst (CastP2X src));
8121
8122 ins_cost(INSN_COST);
8123 format %{ "mov $dst, $src\t# ptr -> long" %}
8124
8125 ins_encode %{
8126 if ($dst$$reg != $src$$reg) {
8127 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8128 }
8129 %}
8130
8131 ins_pipe(ialu_reg);
8132 %}
8133
8134 // Convert oop into int for vectors alignment masking
8135 instruct convP2I(iRegINoSp dst, iRegP src) %{
8136 match(Set dst (ConvL2I (CastP2X src)));
8137
8138 ins_cost(INSN_COST);
8139 format %{ "movw $dst, $src\t# ptr -> int" %}
8140 ins_encode %{
8141 __ movw($dst$$Register, $src$$Register);
8142 %}
8143
8144 ins_pipe(ialu_reg);
8145 %}
8146
8147 // Convert compressed oop into int for vectors alignment masking
8148 // in case of 32bit oops (heap < 4Gb).
8149 instruct convN2I(iRegINoSp dst, iRegN src)
8150 %{
8151 predicate(CompressedOops::shift() == 0);
8152 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8153
8154 ins_cost(INSN_COST);
8155 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8156 ins_encode %{
8157 __ movw($dst$$Register, $src$$Register);
8158 %}
8159
8160 ins_pipe(ialu_reg);
8161 %}
8162
8163
8164 // Convert oop pointer into compressed form
8165 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8166 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8167 match(Set dst (EncodeP src));
8168 effect(KILL cr);
8169 ins_cost(INSN_COST * 3);
8170 format %{ "encode_heap_oop $dst, $src" %}
8171 ins_encode %{
8172 Register s = $src$$Register;
8173 Register d = $dst$$Register;
8174 __ encode_heap_oop(d, s);
8175 %}
8176 ins_pipe(ialu_reg);
8177 %}
8178
8179 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8180 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8181 match(Set dst (EncodeP src));
8182 ins_cost(INSN_COST * 3);
8183 format %{ "encode_heap_oop_not_null $dst, $src" %}
8184 ins_encode %{
8185 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8186 %}
8187 ins_pipe(ialu_reg);
8188 %}
8189
8190 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8191 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8192 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8193 match(Set dst (DecodeN src));
8194 ins_cost(INSN_COST * 3);
8195 format %{ "decode_heap_oop $dst, $src" %}
8196 ins_encode %{
8197 Register s = $src$$Register;
8198 Register d = $dst$$Register;
8199 __ decode_heap_oop(d, s);
8200 %}
8201 ins_pipe(ialu_reg);
8202 %}
8203
8204 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8205 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8206 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8207 match(Set dst (DecodeN src));
8208 ins_cost(INSN_COST * 3);
8209 format %{ "decode_heap_oop_not_null $dst, $src" %}
8210 ins_encode %{
8211 Register s = $src$$Register;
8212 Register d = $dst$$Register;
8213 __ decode_heap_oop_not_null(d, s);
8214 %}
8215 ins_pipe(ialu_reg);
8216 %}
8217
8218 // n.b. AArch64 implementations of encode_klass_not_null and
8219 // decode_klass_not_null do not modify the flags register so, unlike
8220 // Intel, we don't kill CR as a side effect here
8221
8222 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8223 match(Set dst (EncodePKlass src));
8224
8225 ins_cost(INSN_COST * 3);
8226 format %{ "encode_klass_not_null $dst,$src" %}
8227
8228 ins_encode %{
8229 Register src_reg = as_Register($src$$reg);
8230 Register dst_reg = as_Register($dst$$reg);
8231 __ encode_klass_not_null(dst_reg, src_reg);
8232 %}
8233
8234 ins_pipe(ialu_reg);
8235 %}
8236
8237 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8238 match(Set dst (DecodeNKlass src));
8239
8240 ins_cost(INSN_COST * 3);
8241 format %{ "decode_klass_not_null $dst,$src" %}
8242
8243 ins_encode %{
8244 Register src_reg = as_Register($src$$reg);
8245 Register dst_reg = as_Register($dst$$reg);
8246 if (dst_reg != src_reg) {
8247 __ decode_klass_not_null(dst_reg, src_reg);
8248 } else {
8249 __ decode_klass_not_null(dst_reg);
8250 }
8251 %}
8252
8253 ins_pipe(ialu_reg);
8254 %}
8255
8256 instruct checkCastPP(iRegPNoSp dst)
8257 %{
8258 match(Set dst (CheckCastPP dst));
8259
8260 size(0);
8261 format %{ "# checkcastPP of $dst" %}
8262 ins_encode(/* empty encoding */);
8263 ins_pipe(pipe_class_empty);
8264 %}
8265
8266 instruct castPP(iRegPNoSp dst)
8267 %{
8268 match(Set dst (CastPP dst));
8269
8270 size(0);
8271 format %{ "# castPP of $dst" %}
8272 ins_encode(/* empty encoding */);
8273 ins_pipe(pipe_class_empty);
8274 %}
8275
8276 instruct castII(iRegI dst)
8277 %{
8278 predicate(VerifyConstraintCasts == 0);
8279 match(Set dst (CastII dst));
8280
8281 size(0);
8282 format %{ "# castII of $dst" %}
8283 ins_encode(/* empty encoding */);
8284 ins_cost(0);
8285 ins_pipe(pipe_class_empty);
8286 %}
8287
8288 instruct castII_checked(iRegI dst, rFlagsReg cr)
8289 %{
8290 predicate(VerifyConstraintCasts > 0);
8291 match(Set dst (CastII dst));
8292 effect(KILL cr);
8293
8294 format %{ "# castII_checked of $dst" %}
8295 ins_encode %{
8296 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8297 %}
8298 ins_pipe(pipe_slow);
8299 %}
8300
8301 // The unchecked and checked variants for CastII below both use iRegINoSp for src and dst
8302 // as some consumers of CastII node like ConvHF2F forbid the stack pointer as an input
8303 // (please see convHF2F_reg_reg rule which requires input to be in an iRegINoSp register).
8304 instruct castII_nosp(iRegINoSp dst)
8305 %{
8306 predicate(VerifyConstraintCasts == 0);
8307 match(Set dst (CastII dst));
8308
8309 size(0);
8310 format %{ "# castII of $dst" %}
8311 ins_encode(/* empty encoding */);
8312 ins_cost(0);
8313 ins_pipe(pipe_class_empty);
8314 %}
8315
8316 instruct castII_checked_nosp(iRegINoSp dst, rFlagsReg cr)
8317 %{
8318 predicate(VerifyConstraintCasts > 0);
8319 match(Set dst (CastII dst));
8320 effect(KILL cr);
8321
8322 format %{ "# castII_checked of $dst" %}
8323 ins_encode %{
8324 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8325 %}
8326 ins_pipe(pipe_slow);
8327 %}
8328
8329 instruct castLL(iRegL dst)
8330 %{
8331 predicate(VerifyConstraintCasts == 0);
8332 match(Set dst (CastLL dst));
8333
8334 size(0);
8335 format %{ "# castLL of $dst" %}
8336 ins_encode(/* empty encoding */);
8337 ins_cost(0);
8338 ins_pipe(pipe_class_empty);
8339 %}
8340
8341 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8342 %{
8343 predicate(VerifyConstraintCasts > 0);
8344 match(Set dst (CastLL dst));
8345 effect(KILL cr);
8346
8347 format %{ "# castLL_checked of $dst" %}
8348 ins_encode %{
8349 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8350 %}
8351 ins_pipe(pipe_slow);
8352 %}
8353
8354 instruct castHH(vRegF dst)
8355 %{
8356 match(Set dst (CastHH dst));
8357 size(0);
8358 format %{ "# castHH of $dst" %}
8359 ins_encode(/* empty encoding */);
8360 ins_cost(0);
8361 ins_pipe(pipe_class_empty);
8362 %}
8363
8364 instruct castFF(vRegF dst)
8365 %{
8366 match(Set dst (CastFF dst));
8367
8368 size(0);
8369 format %{ "# castFF of $dst" %}
8370 ins_encode(/* empty encoding */);
8371 ins_cost(0);
8372 ins_pipe(pipe_class_empty);
8373 %}
8374
8375 instruct castDD(vRegD dst)
8376 %{
8377 match(Set dst (CastDD dst));
8378
8379 size(0);
8380 format %{ "# castDD of $dst" %}
8381 ins_encode(/* empty encoding */);
8382 ins_cost(0);
8383 ins_pipe(pipe_class_empty);
8384 %}
8385
8386 instruct castVV(vReg dst)
8387 %{
8388 match(Set dst (CastVV dst));
8389
8390 size(0);
8391 format %{ "# castVV of $dst" %}
8392 ins_encode(/* empty encoding */);
8393 ins_cost(0);
8394 ins_pipe(pipe_class_empty);
8395 %}
8396
8397 instruct castVVMask(pRegGov dst)
8398 %{
8399 match(Set dst (CastVV dst));
8400
8401 size(0);
8402 format %{ "# castVV of $dst" %}
8403 ins_encode(/* empty encoding */);
8404 ins_cost(0);
8405 ins_pipe(pipe_class_empty);
8406 %}
8407
8408 // Manifest a CmpU result in an integer register.
8409 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8410 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8411 %{
8412 match(Set dst (CmpU3 src1 src2));
8413 effect(KILL flags);
8414
8415 ins_cost(INSN_COST * 3);
8416 format %{
8417 "cmpw $src1, $src2\n\t"
8418 "csetw $dst, ne\n\t"
8419 "cnegw $dst, lo\t# CmpU3(reg)"
8420 %}
8421 ins_encode %{
8422 __ cmpw($src1$$Register, $src2$$Register);
8423 __ csetw($dst$$Register, Assembler::NE);
8424 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8425 %}
8426
8427 ins_pipe(pipe_class_default);
8428 %}
8429
8430 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8431 %{
8432 match(Set dst (CmpU3 src1 src2));
8433 effect(KILL flags);
8434
8435 ins_cost(INSN_COST * 3);
8436 format %{
8437 "subsw zr, $src1, $src2\n\t"
8438 "csetw $dst, ne\n\t"
8439 "cnegw $dst, lo\t# CmpU3(imm)"
8440 %}
8441 ins_encode %{
8442 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8443 __ csetw($dst$$Register, Assembler::NE);
8444 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8445 %}
8446
8447 ins_pipe(pipe_class_default);
8448 %}
8449
8450 // Manifest a CmpUL result in an integer register.
8451 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8452 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8453 %{
8454 match(Set dst (CmpUL3 src1 src2));
8455 effect(KILL flags);
8456
8457 ins_cost(INSN_COST * 3);
8458 format %{
8459 "cmp $src1, $src2\n\t"
8460 "csetw $dst, ne\n\t"
8461 "cnegw $dst, lo\t# CmpUL3(reg)"
8462 %}
8463 ins_encode %{
8464 __ cmp($src1$$Register, $src2$$Register);
8465 __ csetw($dst$$Register, Assembler::NE);
8466 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8467 %}
8468
8469 ins_pipe(pipe_class_default);
8470 %}
8471
8472 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8473 %{
8474 match(Set dst (CmpUL3 src1 src2));
8475 effect(KILL flags);
8476
8477 ins_cost(INSN_COST * 3);
8478 format %{
8479 "subs zr, $src1, $src2\n\t"
8480 "csetw $dst, ne\n\t"
8481 "cnegw $dst, lo\t# CmpUL3(imm)"
8482 %}
8483 ins_encode %{
8484 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8485 __ csetw($dst$$Register, Assembler::NE);
8486 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8487 %}
8488
8489 ins_pipe(pipe_class_default);
8490 %}
8491
8492 // Manifest a CmpL result in an integer register.
8493 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8494 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8495 %{
8496 match(Set dst (CmpL3 src1 src2));
8497 effect(KILL flags);
8498
8499 ins_cost(INSN_COST * 3);
8500 format %{
8501 "cmp $src1, $src2\n\t"
8502 "csetw $dst, ne\n\t"
8503 "cnegw $dst, lt\t# CmpL3(reg)"
8504 %}
8505 ins_encode %{
8506 __ cmp($src1$$Register, $src2$$Register);
8507 __ csetw($dst$$Register, Assembler::NE);
8508 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8509 %}
8510
8511 ins_pipe(pipe_class_default);
8512 %}
8513
8514 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8515 %{
8516 match(Set dst (CmpL3 src1 src2));
8517 effect(KILL flags);
8518
8519 ins_cost(INSN_COST * 3);
8520 format %{
8521 "subs zr, $src1, $src2\n\t"
8522 "csetw $dst, ne\n\t"
8523 "cnegw $dst, lt\t# CmpL3(imm)"
8524 %}
8525 ins_encode %{
8526 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8527 __ csetw($dst$$Register, Assembler::NE);
8528 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8529 %}
8530
8531 ins_pipe(pipe_class_default);
8532 %}
8533
8534 // ============================================================================
8535 // Conditional Move Instructions
8536
8537 // n.b. we have identical rules for both a signed compare op (cmpOp)
8538 // and an unsigned compare op (cmpOpU). it would be nice if we could
8539 // define an op class which merged both inputs and use it to type the
8540 // argument to a single rule. unfortunatelyt his fails because the
8541 // opclass does not live up to the COND_INTER interface of its
8542 // component operands. When the generic code tries to negate the
8543 // operand it ends up running the generci Machoper::negate method
8544 // which throws a ShouldNotHappen. So, we have to provide two flavours
8545 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8546
8547 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8548 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8549
8550 ins_cost(INSN_COST * 2);
8551 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8552
8553 ins_encode %{
8554 __ cselw(as_Register($dst$$reg),
8555 as_Register($src2$$reg),
8556 as_Register($src1$$reg),
8557 (Assembler::Condition)$cmp$$cmpcode);
8558 %}
8559
8560 ins_pipe(icond_reg_reg);
8561 %}
8562
8563 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8564 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8565
8566 ins_cost(INSN_COST * 2);
8567 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8568
8569 ins_encode %{
8570 __ cselw(as_Register($dst$$reg),
8571 as_Register($src2$$reg),
8572 as_Register($src1$$reg),
8573 (Assembler::Condition)$cmp$$cmpcode);
8574 %}
8575
8576 ins_pipe(icond_reg_reg);
8577 %}
8578
8579 // special cases where one arg is zero
8580
8581 // n.b. this is selected in preference to the rule above because it
8582 // avoids loading constant 0 into a source register
8583
8584 // TODO
8585 // we ought only to be able to cull one of these variants as the ideal
8586 // transforms ought always to order the zero consistently (to left/right?)
8587
8588 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8589 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8590
8591 ins_cost(INSN_COST * 2);
8592 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8593
8594 ins_encode %{
8595 __ cselw(as_Register($dst$$reg),
8596 as_Register($src$$reg),
8597 zr,
8598 (Assembler::Condition)$cmp$$cmpcode);
8599 %}
8600
8601 ins_pipe(icond_reg);
8602 %}
8603
8604 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8605 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8606
8607 ins_cost(INSN_COST * 2);
8608 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8609
8610 ins_encode %{
8611 __ cselw(as_Register($dst$$reg),
8612 as_Register($src$$reg),
8613 zr,
8614 (Assembler::Condition)$cmp$$cmpcode);
8615 %}
8616
8617 ins_pipe(icond_reg);
8618 %}
8619
8620 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8621 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8622
8623 ins_cost(INSN_COST * 2);
8624 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8625
8626 ins_encode %{
8627 __ cselw(as_Register($dst$$reg),
8628 zr,
8629 as_Register($src$$reg),
8630 (Assembler::Condition)$cmp$$cmpcode);
8631 %}
8632
8633 ins_pipe(icond_reg);
8634 %}
8635
8636 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8637 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8638
8639 ins_cost(INSN_COST * 2);
8640 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8641
8642 ins_encode %{
8643 __ cselw(as_Register($dst$$reg),
8644 zr,
8645 as_Register($src$$reg),
8646 (Assembler::Condition)$cmp$$cmpcode);
8647 %}
8648
8649 ins_pipe(icond_reg);
8650 %}
8651
8652 // special case for creating a boolean 0 or 1
8653
8654 // n.b. this is selected in preference to the rule above because it
8655 // avoids loading constants 0 and 1 into a source register
8656
8657 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8658 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8659
8660 ins_cost(INSN_COST * 2);
8661 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8662
8663 ins_encode %{
8664 // equivalently
8665 // cset(as_Register($dst$$reg),
8666 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8667 __ csincw(as_Register($dst$$reg),
8668 zr,
8669 zr,
8670 (Assembler::Condition)$cmp$$cmpcode);
8671 %}
8672
8673 ins_pipe(icond_none);
8674 %}
8675
8676 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8677 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8678
8679 ins_cost(INSN_COST * 2);
8680 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8681
8682 ins_encode %{
8683 // equivalently
8684 // cset(as_Register($dst$$reg),
8685 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8686 __ csincw(as_Register($dst$$reg),
8687 zr,
8688 zr,
8689 (Assembler::Condition)$cmp$$cmpcode);
8690 %}
8691
8692 ins_pipe(icond_none);
8693 %}
8694
8695 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8696 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8697
8698 ins_cost(INSN_COST * 2);
8699 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8700
8701 ins_encode %{
8702 __ csel(as_Register($dst$$reg),
8703 as_Register($src2$$reg),
8704 as_Register($src1$$reg),
8705 (Assembler::Condition)$cmp$$cmpcode);
8706 %}
8707
8708 ins_pipe(icond_reg_reg);
8709 %}
8710
8711 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8712 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8713
8714 ins_cost(INSN_COST * 2);
8715 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8716
8717 ins_encode %{
8718 __ csel(as_Register($dst$$reg),
8719 as_Register($src2$$reg),
8720 as_Register($src1$$reg),
8721 (Assembler::Condition)$cmp$$cmpcode);
8722 %}
8723
8724 ins_pipe(icond_reg_reg);
8725 %}
8726
8727 // special cases where one arg is zero
8728
8729 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8730 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8731
8732 ins_cost(INSN_COST * 2);
8733 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8734
8735 ins_encode %{
8736 __ csel(as_Register($dst$$reg),
8737 zr,
8738 as_Register($src$$reg),
8739 (Assembler::Condition)$cmp$$cmpcode);
8740 %}
8741
8742 ins_pipe(icond_reg);
8743 %}
8744
8745 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8746 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8747
8748 ins_cost(INSN_COST * 2);
8749 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8750
8751 ins_encode %{
8752 __ csel(as_Register($dst$$reg),
8753 zr,
8754 as_Register($src$$reg),
8755 (Assembler::Condition)$cmp$$cmpcode);
8756 %}
8757
8758 ins_pipe(icond_reg);
8759 %}
8760
8761 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8762 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8763
8764 ins_cost(INSN_COST * 2);
8765 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8766
8767 ins_encode %{
8768 __ csel(as_Register($dst$$reg),
8769 as_Register($src$$reg),
8770 zr,
8771 (Assembler::Condition)$cmp$$cmpcode);
8772 %}
8773
8774 ins_pipe(icond_reg);
8775 %}
8776
8777 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8778 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8779
8780 ins_cost(INSN_COST * 2);
8781 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8782
8783 ins_encode %{
8784 __ csel(as_Register($dst$$reg),
8785 as_Register($src$$reg),
8786 zr,
8787 (Assembler::Condition)$cmp$$cmpcode);
8788 %}
8789
8790 ins_pipe(icond_reg);
8791 %}
8792
8793 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8794 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8795
8796 ins_cost(INSN_COST * 2);
8797 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8798
8799 ins_encode %{
8800 __ csel(as_Register($dst$$reg),
8801 as_Register($src2$$reg),
8802 as_Register($src1$$reg),
8803 (Assembler::Condition)$cmp$$cmpcode);
8804 %}
8805
8806 ins_pipe(icond_reg_reg);
8807 %}
8808
8809 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8810 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8811
8812 ins_cost(INSN_COST * 2);
8813 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8814
8815 ins_encode %{
8816 __ csel(as_Register($dst$$reg),
8817 as_Register($src2$$reg),
8818 as_Register($src1$$reg),
8819 (Assembler::Condition)$cmp$$cmpcode);
8820 %}
8821
8822 ins_pipe(icond_reg_reg);
8823 %}
8824
8825 // special cases where one arg is zero
8826
8827 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8828 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8829
8830 ins_cost(INSN_COST * 2);
8831 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8832
8833 ins_encode %{
8834 __ csel(as_Register($dst$$reg),
8835 zr,
8836 as_Register($src$$reg),
8837 (Assembler::Condition)$cmp$$cmpcode);
8838 %}
8839
8840 ins_pipe(icond_reg);
8841 %}
8842
8843 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8844 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8845
8846 ins_cost(INSN_COST * 2);
8847 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8848
8849 ins_encode %{
8850 __ csel(as_Register($dst$$reg),
8851 zr,
8852 as_Register($src$$reg),
8853 (Assembler::Condition)$cmp$$cmpcode);
8854 %}
8855
8856 ins_pipe(icond_reg);
8857 %}
8858
8859 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8860 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8861
8862 ins_cost(INSN_COST * 2);
8863 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8864
8865 ins_encode %{
8866 __ csel(as_Register($dst$$reg),
8867 as_Register($src$$reg),
8868 zr,
8869 (Assembler::Condition)$cmp$$cmpcode);
8870 %}
8871
8872 ins_pipe(icond_reg);
8873 %}
8874
8875 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8876 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8877
8878 ins_cost(INSN_COST * 2);
8879 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8880
8881 ins_encode %{
8882 __ csel(as_Register($dst$$reg),
8883 as_Register($src$$reg),
8884 zr,
8885 (Assembler::Condition)$cmp$$cmpcode);
8886 %}
8887
8888 ins_pipe(icond_reg);
8889 %}
8890
8891 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8892 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8893
8894 ins_cost(INSN_COST * 2);
8895 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8896
8897 ins_encode %{
8898 __ cselw(as_Register($dst$$reg),
8899 as_Register($src2$$reg),
8900 as_Register($src1$$reg),
8901 (Assembler::Condition)$cmp$$cmpcode);
8902 %}
8903
8904 ins_pipe(icond_reg_reg);
8905 %}
8906
8907 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8908 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8909
8910 ins_cost(INSN_COST * 2);
8911 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8912
8913 ins_encode %{
8914 __ cselw(as_Register($dst$$reg),
8915 as_Register($src2$$reg),
8916 as_Register($src1$$reg),
8917 (Assembler::Condition)$cmp$$cmpcode);
8918 %}
8919
8920 ins_pipe(icond_reg_reg);
8921 %}
8922
8923 // special cases where one arg is zero
8924
8925 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8926 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8927
8928 ins_cost(INSN_COST * 2);
8929 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8930
8931 ins_encode %{
8932 __ cselw(as_Register($dst$$reg),
8933 zr,
8934 as_Register($src$$reg),
8935 (Assembler::Condition)$cmp$$cmpcode);
8936 %}
8937
8938 ins_pipe(icond_reg);
8939 %}
8940
8941 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8942 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8943
8944 ins_cost(INSN_COST * 2);
8945 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8946
8947 ins_encode %{
8948 __ cselw(as_Register($dst$$reg),
8949 zr,
8950 as_Register($src$$reg),
8951 (Assembler::Condition)$cmp$$cmpcode);
8952 %}
8953
8954 ins_pipe(icond_reg);
8955 %}
8956
8957 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8958 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8959
8960 ins_cost(INSN_COST * 2);
8961 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8962
8963 ins_encode %{
8964 __ cselw(as_Register($dst$$reg),
8965 as_Register($src$$reg),
8966 zr,
8967 (Assembler::Condition)$cmp$$cmpcode);
8968 %}
8969
8970 ins_pipe(icond_reg);
8971 %}
8972
8973 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8974 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8975
8976 ins_cost(INSN_COST * 2);
8977 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8978
8979 ins_encode %{
8980 __ cselw(as_Register($dst$$reg),
8981 as_Register($src$$reg),
8982 zr,
8983 (Assembler::Condition)$cmp$$cmpcode);
8984 %}
8985
8986 ins_pipe(icond_reg);
8987 %}
8988
8989 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8990 %{
8991 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8992
8993 ins_cost(INSN_COST * 3);
8994
8995 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8996 ins_encode %{
8997 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8998 __ fcsels(as_FloatRegister($dst$$reg),
8999 as_FloatRegister($src2$$reg),
9000 as_FloatRegister($src1$$reg),
9001 cond);
9002 %}
9003
9004 ins_pipe(fp_cond_reg_reg_s);
9005 %}
9006
9007 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9008 %{
9009 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9010
9011 ins_cost(INSN_COST * 3);
9012
9013 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9014 ins_encode %{
9015 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9016 __ fcsels(as_FloatRegister($dst$$reg),
9017 as_FloatRegister($src2$$reg),
9018 as_FloatRegister($src1$$reg),
9019 cond);
9020 %}
9021
9022 ins_pipe(fp_cond_reg_reg_s);
9023 %}
9024
9025 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9026 %{
9027 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9028
9029 ins_cost(INSN_COST * 3);
9030
9031 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9032 ins_encode %{
9033 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9034 __ fcseld(as_FloatRegister($dst$$reg),
9035 as_FloatRegister($src2$$reg),
9036 as_FloatRegister($src1$$reg),
9037 cond);
9038 %}
9039
9040 ins_pipe(fp_cond_reg_reg_d);
9041 %}
9042
9043 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9044 %{
9045 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9046
9047 ins_cost(INSN_COST * 3);
9048
9049 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9050 ins_encode %{
9051 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9052 __ fcseld(as_FloatRegister($dst$$reg),
9053 as_FloatRegister($src2$$reg),
9054 as_FloatRegister($src1$$reg),
9055 cond);
9056 %}
9057
9058 ins_pipe(fp_cond_reg_reg_d);
9059 %}
9060
9061 // ============================================================================
9062 // Arithmetic Instructions
9063 //
9064
9065 // Integer Addition
9066
9067 // TODO
9068 // these currently employ operations which do not set CR and hence are
9069 // not flagged as killing CR but we would like to isolate the cases
9070 // where we want to set flags from those where we don't. need to work
9071 // out how to do that.
9072
9073 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9074 match(Set dst (AddI src1 src2));
9075
9076 ins_cost(INSN_COST);
9077 format %{ "addw $dst, $src1, $src2" %}
9078
9079 ins_encode %{
9080 __ addw(as_Register($dst$$reg),
9081 as_Register($src1$$reg),
9082 as_Register($src2$$reg));
9083 %}
9084
9085 ins_pipe(ialu_reg_reg);
9086 %}
9087
9088 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9089 match(Set dst (AddI src1 src2));
9090
9091 ins_cost(INSN_COST);
9092 format %{ "addw $dst, $src1, $src2" %}
9093
9094 // use opcode to indicate that this is an add not a sub
9095 opcode(0x0);
9096
9097 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9098
9099 ins_pipe(ialu_reg_imm);
9100 %}
9101
9102 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9103 match(Set dst (AddI (ConvL2I src1) src2));
9104
9105 ins_cost(INSN_COST);
9106 format %{ "addw $dst, $src1, $src2" %}
9107
9108 // use opcode to indicate that this is an add not a sub
9109 opcode(0x0);
9110
9111 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9112
9113 ins_pipe(ialu_reg_imm);
9114 %}
9115
9116 // Pointer Addition
9117 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9118 match(Set dst (AddP src1 src2));
9119
9120 ins_cost(INSN_COST);
9121 format %{ "add $dst, $src1, $src2\t# ptr" %}
9122
9123 ins_encode %{
9124 __ add(as_Register($dst$$reg),
9125 as_Register($src1$$reg),
9126 as_Register($src2$$reg));
9127 %}
9128
9129 ins_pipe(ialu_reg_reg);
9130 %}
9131
9132 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9133 match(Set dst (AddP src1 (ConvI2L src2)));
9134
9135 ins_cost(1.9 * INSN_COST);
9136 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9137
9138 ins_encode %{
9139 __ add(as_Register($dst$$reg),
9140 as_Register($src1$$reg),
9141 as_Register($src2$$reg), ext::sxtw);
9142 %}
9143
9144 ins_pipe(ialu_reg_reg);
9145 %}
9146
9147 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9148 match(Set dst (AddP src1 (LShiftL src2 scale)));
9149
9150 ins_cost(1.9 * INSN_COST);
9151 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9152
9153 ins_encode %{
9154 __ lea(as_Register($dst$$reg),
9155 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9156 Address::lsl($scale$$constant)));
9157 %}
9158
9159 ins_pipe(ialu_reg_reg_shift);
9160 %}
9161
9162 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9163 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9164
9165 ins_cost(1.9 * INSN_COST);
9166 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9167
9168 ins_encode %{
9169 __ lea(as_Register($dst$$reg),
9170 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9171 Address::sxtw($scale$$constant)));
9172 %}
9173
9174 ins_pipe(ialu_reg_reg_shift);
9175 %}
9176
9177 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9178 match(Set dst (LShiftL (ConvI2L src) scale));
9179
9180 ins_cost(INSN_COST);
9181 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9182
9183 ins_encode %{
9184 __ sbfiz(as_Register($dst$$reg),
9185 as_Register($src$$reg),
9186 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9187 %}
9188
9189 ins_pipe(ialu_reg_shift);
9190 %}
9191
9192 // Pointer Immediate Addition
9193 // n.b. this needs to be more expensive than using an indirect memory
9194 // operand
9195 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9196 match(Set dst (AddP src1 src2));
9197
9198 ins_cost(INSN_COST);
9199 format %{ "add $dst, $src1, $src2\t# ptr" %}
9200
9201 // use opcode to indicate that this is an add not a sub
9202 opcode(0x0);
9203
9204 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9205
9206 ins_pipe(ialu_reg_imm);
9207 %}
9208
9209 // Long Addition
9210 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9211
9212 match(Set dst (AddL src1 src2));
9213
9214 ins_cost(INSN_COST);
9215 format %{ "add $dst, $src1, $src2" %}
9216
9217 ins_encode %{
9218 __ add(as_Register($dst$$reg),
9219 as_Register($src1$$reg),
9220 as_Register($src2$$reg));
9221 %}
9222
9223 ins_pipe(ialu_reg_reg);
9224 %}
9225
9226 // No constant pool entries requiredLong Immediate Addition.
9227 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9228 match(Set dst (AddL src1 src2));
9229
9230 ins_cost(INSN_COST);
9231 format %{ "add $dst, $src1, $src2" %}
9232
9233 // use opcode to indicate that this is an add not a sub
9234 opcode(0x0);
9235
9236 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9237
9238 ins_pipe(ialu_reg_imm);
9239 %}
9240
9241 // Integer Subtraction
9242 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9243 match(Set dst (SubI src1 src2));
9244
9245 ins_cost(INSN_COST);
9246 format %{ "subw $dst, $src1, $src2" %}
9247
9248 ins_encode %{
9249 __ subw(as_Register($dst$$reg),
9250 as_Register($src1$$reg),
9251 as_Register($src2$$reg));
9252 %}
9253
9254 ins_pipe(ialu_reg_reg);
9255 %}
9256
9257 // Immediate Subtraction
9258 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9259 match(Set dst (SubI src1 src2));
9260
9261 ins_cost(INSN_COST);
9262 format %{ "subw $dst, $src1, $src2" %}
9263
9264 // use opcode to indicate that this is a sub not an add
9265 opcode(0x1);
9266
9267 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9268
9269 ins_pipe(ialu_reg_imm);
9270 %}
9271
9272 // Long Subtraction
9273 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9274
9275 match(Set dst (SubL src1 src2));
9276
9277 ins_cost(INSN_COST);
9278 format %{ "sub $dst, $src1, $src2" %}
9279
9280 ins_encode %{
9281 __ sub(as_Register($dst$$reg),
9282 as_Register($src1$$reg),
9283 as_Register($src2$$reg));
9284 %}
9285
9286 ins_pipe(ialu_reg_reg);
9287 %}
9288
9289 // No constant pool entries requiredLong Immediate Subtraction.
9290 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9291 match(Set dst (SubL src1 src2));
9292
9293 ins_cost(INSN_COST);
9294 format %{ "sub$dst, $src1, $src2" %}
9295
9296 // use opcode to indicate that this is a sub not an add
9297 opcode(0x1);
9298
9299 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9300
9301 ins_pipe(ialu_reg_imm);
9302 %}
9303
9304 // Integer Negation (special case for sub)
9305
9306 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9307 match(Set dst (SubI zero src));
9308
9309 ins_cost(INSN_COST);
9310 format %{ "negw $dst, $src\t# int" %}
9311
9312 ins_encode %{
9313 __ negw(as_Register($dst$$reg),
9314 as_Register($src$$reg));
9315 %}
9316
9317 ins_pipe(ialu_reg);
9318 %}
9319
9320 // Long Negation
9321
9322 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9323 match(Set dst (SubL zero src));
9324
9325 ins_cost(INSN_COST);
9326 format %{ "neg $dst, $src\t# long" %}
9327
9328 ins_encode %{
9329 __ neg(as_Register($dst$$reg),
9330 as_Register($src$$reg));
9331 %}
9332
9333 ins_pipe(ialu_reg);
9334 %}
9335
9336 // Integer Multiply
9337
9338 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9339 match(Set dst (MulI src1 src2));
9340
9341 ins_cost(INSN_COST * 3);
9342 format %{ "mulw $dst, $src1, $src2" %}
9343
9344 ins_encode %{
9345 __ mulw(as_Register($dst$$reg),
9346 as_Register($src1$$reg),
9347 as_Register($src2$$reg));
9348 %}
9349
9350 ins_pipe(imul_reg_reg);
9351 %}
9352
9353 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9354 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9355
9356 ins_cost(INSN_COST * 3);
9357 format %{ "smull $dst, $src1, $src2" %}
9358
9359 ins_encode %{
9360 __ smull(as_Register($dst$$reg),
9361 as_Register($src1$$reg),
9362 as_Register($src2$$reg));
9363 %}
9364
9365 ins_pipe(imul_reg_reg);
9366 %}
9367
9368 // Long Multiply
9369
9370 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9371 match(Set dst (MulL src1 src2));
9372
9373 ins_cost(INSN_COST * 5);
9374 format %{ "mul $dst, $src1, $src2" %}
9375
9376 ins_encode %{
9377 __ mul(as_Register($dst$$reg),
9378 as_Register($src1$$reg),
9379 as_Register($src2$$reg));
9380 %}
9381
9382 ins_pipe(lmul_reg_reg);
9383 %}
9384
9385 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9386 %{
9387 match(Set dst (MulHiL src1 src2));
9388
9389 ins_cost(INSN_COST * 7);
9390 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9391
9392 ins_encode %{
9393 __ smulh(as_Register($dst$$reg),
9394 as_Register($src1$$reg),
9395 as_Register($src2$$reg));
9396 %}
9397
9398 ins_pipe(lmul_reg_reg);
9399 %}
9400
9401 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9402 %{
9403 match(Set dst (UMulHiL src1 src2));
9404
9405 ins_cost(INSN_COST * 7);
9406 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9407
9408 ins_encode %{
9409 __ umulh(as_Register($dst$$reg),
9410 as_Register($src1$$reg),
9411 as_Register($src2$$reg));
9412 %}
9413
9414 ins_pipe(lmul_reg_reg);
9415 %}
9416
9417 // Combined Integer Multiply & Add/Sub
9418
9419 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9420 match(Set dst (AddI src3 (MulI src1 src2)));
9421
9422 ins_cost(INSN_COST * 3);
9423 format %{ "madd $dst, $src1, $src2, $src3" %}
9424
9425 ins_encode %{
9426 __ maddw(as_Register($dst$$reg),
9427 as_Register($src1$$reg),
9428 as_Register($src2$$reg),
9429 as_Register($src3$$reg));
9430 %}
9431
9432 ins_pipe(imac_reg_reg);
9433 %}
9434
9435 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9436 match(Set dst (SubI src3 (MulI src1 src2)));
9437
9438 ins_cost(INSN_COST * 3);
9439 format %{ "msub $dst, $src1, $src2, $src3" %}
9440
9441 ins_encode %{
9442 __ msubw(as_Register($dst$$reg),
9443 as_Register($src1$$reg),
9444 as_Register($src2$$reg),
9445 as_Register($src3$$reg));
9446 %}
9447
9448 ins_pipe(imac_reg_reg);
9449 %}
9450
9451 // Combined Integer Multiply & Neg
9452
9453 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9454 match(Set dst (MulI (SubI zero src1) src2));
9455
9456 ins_cost(INSN_COST * 3);
9457 format %{ "mneg $dst, $src1, $src2" %}
9458
9459 ins_encode %{
9460 __ mnegw(as_Register($dst$$reg),
9461 as_Register($src1$$reg),
9462 as_Register($src2$$reg));
9463 %}
9464
9465 ins_pipe(imac_reg_reg);
9466 %}
9467
9468 // Combined Long Multiply & Add/Sub
9469
9470 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9471 match(Set dst (AddL src3 (MulL src1 src2)));
9472
9473 ins_cost(INSN_COST * 5);
9474 format %{ "madd $dst, $src1, $src2, $src3" %}
9475
9476 ins_encode %{
9477 __ madd(as_Register($dst$$reg),
9478 as_Register($src1$$reg),
9479 as_Register($src2$$reg),
9480 as_Register($src3$$reg));
9481 %}
9482
9483 ins_pipe(lmac_reg_reg);
9484 %}
9485
9486 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9487 match(Set dst (SubL src3 (MulL src1 src2)));
9488
9489 ins_cost(INSN_COST * 5);
9490 format %{ "msub $dst, $src1, $src2, $src3" %}
9491
9492 ins_encode %{
9493 __ msub(as_Register($dst$$reg),
9494 as_Register($src1$$reg),
9495 as_Register($src2$$reg),
9496 as_Register($src3$$reg));
9497 %}
9498
9499 ins_pipe(lmac_reg_reg);
9500 %}
9501
9502 // Combined Long Multiply & Neg
9503
9504 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9505 match(Set dst (MulL (SubL zero src1) src2));
9506
9507 ins_cost(INSN_COST * 5);
9508 format %{ "mneg $dst, $src1, $src2" %}
9509
9510 ins_encode %{
9511 __ mneg(as_Register($dst$$reg),
9512 as_Register($src1$$reg),
9513 as_Register($src2$$reg));
9514 %}
9515
9516 ins_pipe(lmac_reg_reg);
9517 %}
9518
9519 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9520
9521 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9522 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9523
9524 ins_cost(INSN_COST * 3);
9525 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9526
9527 ins_encode %{
9528 __ smaddl(as_Register($dst$$reg),
9529 as_Register($src1$$reg),
9530 as_Register($src2$$reg),
9531 as_Register($src3$$reg));
9532 %}
9533
9534 ins_pipe(imac_reg_reg);
9535 %}
9536
9537 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9538 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9539
9540 ins_cost(INSN_COST * 3);
9541 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9542
9543 ins_encode %{
9544 __ smsubl(as_Register($dst$$reg),
9545 as_Register($src1$$reg),
9546 as_Register($src2$$reg),
9547 as_Register($src3$$reg));
9548 %}
9549
9550 ins_pipe(imac_reg_reg);
9551 %}
9552
9553 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9554 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9555
9556 ins_cost(INSN_COST * 3);
9557 format %{ "smnegl $dst, $src1, $src2" %}
9558
9559 ins_encode %{
9560 __ smnegl(as_Register($dst$$reg),
9561 as_Register($src1$$reg),
9562 as_Register($src2$$reg));
9563 %}
9564
9565 ins_pipe(imac_reg_reg);
9566 %}
9567
9568 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9569
9570 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9571 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9572
9573 ins_cost(INSN_COST * 5);
9574 format %{ "mulw rscratch1, $src1, $src2\n\t"
9575 "maddw $dst, $src3, $src4, rscratch1" %}
9576
9577 ins_encode %{
9578 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9579 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9580
9581 ins_pipe(imac_reg_reg);
9582 %}
9583
9584 // Integer Divide
9585
9586 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9587 match(Set dst (DivI src1 src2));
9588
9589 ins_cost(INSN_COST * 19);
9590 format %{ "sdivw $dst, $src1, $src2" %}
9591
9592 ins_encode(aarch64_enc_divw(dst, src1, src2));
9593 ins_pipe(idiv_reg_reg);
9594 %}
9595
9596 // Long Divide
9597
9598 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9599 match(Set dst (DivL src1 src2));
9600
9601 ins_cost(INSN_COST * 35);
9602 format %{ "sdiv $dst, $src1, $src2" %}
9603
9604 ins_encode(aarch64_enc_div(dst, src1, src2));
9605 ins_pipe(ldiv_reg_reg);
9606 %}
9607
9608 // Integer Remainder
9609
9610 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9611 match(Set dst (ModI src1 src2));
9612
9613 ins_cost(INSN_COST * 22);
9614 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9615 "msubw $dst, rscratch1, $src2, $src1" %}
9616
9617 ins_encode(aarch64_enc_modw(dst, src1, src2));
9618 ins_pipe(idiv_reg_reg);
9619 %}
9620
9621 // Long Remainder
9622
9623 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9624 match(Set dst (ModL src1 src2));
9625
9626 ins_cost(INSN_COST * 38);
9627 format %{ "sdiv rscratch1, $src1, $src2\n"
9628 "msub $dst, rscratch1, $src2, $src1" %}
9629
9630 ins_encode(aarch64_enc_mod(dst, src1, src2));
9631 ins_pipe(ldiv_reg_reg);
9632 %}
9633
9634 // Unsigned Integer Divide
9635
9636 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9637 match(Set dst (UDivI src1 src2));
9638
9639 ins_cost(INSN_COST * 19);
9640 format %{ "udivw $dst, $src1, $src2" %}
9641
9642 ins_encode %{
9643 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9644 %}
9645
9646 ins_pipe(idiv_reg_reg);
9647 %}
9648
9649 // Unsigned Long Divide
9650
9651 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9652 match(Set dst (UDivL src1 src2));
9653
9654 ins_cost(INSN_COST * 35);
9655 format %{ "udiv $dst, $src1, $src2" %}
9656
9657 ins_encode %{
9658 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9659 %}
9660
9661 ins_pipe(ldiv_reg_reg);
9662 %}
9663
9664 // Unsigned Integer Remainder
9665
9666 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9667 match(Set dst (UModI src1 src2));
9668
9669 ins_cost(INSN_COST * 22);
9670 format %{ "udivw rscratch1, $src1, $src2\n\t"
9671 "msubw $dst, rscratch1, $src2, $src1" %}
9672
9673 ins_encode %{
9674 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9675 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9676 %}
9677
9678 ins_pipe(idiv_reg_reg);
9679 %}
9680
9681 // Unsigned Long Remainder
9682
9683 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9684 match(Set dst (UModL src1 src2));
9685
9686 ins_cost(INSN_COST * 38);
9687 format %{ "udiv rscratch1, $src1, $src2\n"
9688 "msub $dst, rscratch1, $src2, $src1" %}
9689
9690 ins_encode %{
9691 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9692 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9693 %}
9694
9695 ins_pipe(ldiv_reg_reg);
9696 %}
9697
9698 // Integer Shifts
9699
9700 // Shift Left Register
9701 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9702 match(Set dst (LShiftI src1 src2));
9703
9704 ins_cost(INSN_COST * 2);
9705 format %{ "lslvw $dst, $src1, $src2" %}
9706
9707 ins_encode %{
9708 __ lslvw(as_Register($dst$$reg),
9709 as_Register($src1$$reg),
9710 as_Register($src2$$reg));
9711 %}
9712
9713 ins_pipe(ialu_reg_reg_vshift);
9714 %}
9715
9716 // Shift Left Immediate
9717 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9718 match(Set dst (LShiftI src1 src2));
9719
9720 ins_cost(INSN_COST);
9721 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9722
9723 ins_encode %{
9724 __ lslw(as_Register($dst$$reg),
9725 as_Register($src1$$reg),
9726 $src2$$constant & 0x1f);
9727 %}
9728
9729 ins_pipe(ialu_reg_shift);
9730 %}
9731
9732 // Shift Right Logical Register
9733 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9734 match(Set dst (URShiftI src1 src2));
9735
9736 ins_cost(INSN_COST * 2);
9737 format %{ "lsrvw $dst, $src1, $src2" %}
9738
9739 ins_encode %{
9740 __ lsrvw(as_Register($dst$$reg),
9741 as_Register($src1$$reg),
9742 as_Register($src2$$reg));
9743 %}
9744
9745 ins_pipe(ialu_reg_reg_vshift);
9746 %}
9747
9748 // Shift Right Logical Immediate
9749 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9750 match(Set dst (URShiftI src1 src2));
9751
9752 ins_cost(INSN_COST);
9753 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9754
9755 ins_encode %{
9756 __ lsrw(as_Register($dst$$reg),
9757 as_Register($src1$$reg),
9758 $src2$$constant & 0x1f);
9759 %}
9760
9761 ins_pipe(ialu_reg_shift);
9762 %}
9763
9764 // Shift Right Arithmetic Register
9765 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9766 match(Set dst (RShiftI src1 src2));
9767
9768 ins_cost(INSN_COST * 2);
9769 format %{ "asrvw $dst, $src1, $src2" %}
9770
9771 ins_encode %{
9772 __ asrvw(as_Register($dst$$reg),
9773 as_Register($src1$$reg),
9774 as_Register($src2$$reg));
9775 %}
9776
9777 ins_pipe(ialu_reg_reg_vshift);
9778 %}
9779
9780 // Shift Right Arithmetic Immediate
9781 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9782 match(Set dst (RShiftI src1 src2));
9783
9784 ins_cost(INSN_COST);
9785 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9786
9787 ins_encode %{
9788 __ asrw(as_Register($dst$$reg),
9789 as_Register($src1$$reg),
9790 $src2$$constant & 0x1f);
9791 %}
9792
9793 ins_pipe(ialu_reg_shift);
9794 %}
9795
9796 // Combined Int Mask and Right Shift (using UBFM)
9797 // TODO
9798
9799 // Long Shifts
9800
9801 // Shift Left Register
9802 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9803 match(Set dst (LShiftL src1 src2));
9804
9805 ins_cost(INSN_COST * 2);
9806 format %{ "lslv $dst, $src1, $src2" %}
9807
9808 ins_encode %{
9809 __ lslv(as_Register($dst$$reg),
9810 as_Register($src1$$reg),
9811 as_Register($src2$$reg));
9812 %}
9813
9814 ins_pipe(ialu_reg_reg_vshift);
9815 %}
9816
9817 // Shift Left Immediate
9818 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9819 match(Set dst (LShiftL src1 src2));
9820
9821 ins_cost(INSN_COST);
9822 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9823
9824 ins_encode %{
9825 __ lsl(as_Register($dst$$reg),
9826 as_Register($src1$$reg),
9827 $src2$$constant & 0x3f);
9828 %}
9829
9830 ins_pipe(ialu_reg_shift);
9831 %}
9832
9833 // Shift Right Logical Register
9834 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9835 match(Set dst (URShiftL src1 src2));
9836
9837 ins_cost(INSN_COST * 2);
9838 format %{ "lsrv $dst, $src1, $src2" %}
9839
9840 ins_encode %{
9841 __ lsrv(as_Register($dst$$reg),
9842 as_Register($src1$$reg),
9843 as_Register($src2$$reg));
9844 %}
9845
9846 ins_pipe(ialu_reg_reg_vshift);
9847 %}
9848
9849 // Shift Right Logical Immediate
9850 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9851 match(Set dst (URShiftL src1 src2));
9852
9853 ins_cost(INSN_COST);
9854 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9855
9856 ins_encode %{
9857 __ lsr(as_Register($dst$$reg),
9858 as_Register($src1$$reg),
9859 $src2$$constant & 0x3f);
9860 %}
9861
9862 ins_pipe(ialu_reg_shift);
9863 %}
9864
9865 // A special-case pattern for card table stores.
9866 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9867 match(Set dst (URShiftL (CastP2X src1) src2));
9868
9869 ins_cost(INSN_COST);
9870 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9871
9872 ins_encode %{
9873 __ lsr(as_Register($dst$$reg),
9874 as_Register($src1$$reg),
9875 $src2$$constant & 0x3f);
9876 %}
9877
9878 ins_pipe(ialu_reg_shift);
9879 %}
9880
9881 // Shift Right Arithmetic Register
9882 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9883 match(Set dst (RShiftL src1 src2));
9884
9885 ins_cost(INSN_COST * 2);
9886 format %{ "asrv $dst, $src1, $src2" %}
9887
9888 ins_encode %{
9889 __ asrv(as_Register($dst$$reg),
9890 as_Register($src1$$reg),
9891 as_Register($src2$$reg));
9892 %}
9893
9894 ins_pipe(ialu_reg_reg_vshift);
9895 %}
9896
9897 // Shift Right Arithmetic Immediate
9898 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9899 match(Set dst (RShiftL src1 src2));
9900
9901 ins_cost(INSN_COST);
9902 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9903
9904 ins_encode %{
9905 __ asr(as_Register($dst$$reg),
9906 as_Register($src1$$reg),
9907 $src2$$constant & 0x3f);
9908 %}
9909
9910 ins_pipe(ialu_reg_shift);
9911 %}
9912
9913 // BEGIN This section of the file is automatically generated. Do not edit --------------
9914 // This section is generated from aarch64_ad.m4
9915
9916 // This pattern is automatically generated from aarch64_ad.m4
9917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9918 instruct regL_not_reg(iRegLNoSp dst,
9919 iRegL src1, immL_M1 m1,
9920 rFlagsReg cr) %{
9921 match(Set dst (XorL src1 m1));
9922 ins_cost(INSN_COST);
9923 format %{ "eon $dst, $src1, zr" %}
9924
9925 ins_encode %{
9926 __ eon(as_Register($dst$$reg),
9927 as_Register($src1$$reg),
9928 zr,
9929 Assembler::LSL, 0);
9930 %}
9931
9932 ins_pipe(ialu_reg);
9933 %}
9934
9935 // This pattern is automatically generated from aarch64_ad.m4
9936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9937 instruct regI_not_reg(iRegINoSp dst,
9938 iRegIorL2I src1, immI_M1 m1,
9939 rFlagsReg cr) %{
9940 match(Set dst (XorI src1 m1));
9941 ins_cost(INSN_COST);
9942 format %{ "eonw $dst, $src1, zr" %}
9943
9944 ins_encode %{
9945 __ eonw(as_Register($dst$$reg),
9946 as_Register($src1$$reg),
9947 zr,
9948 Assembler::LSL, 0);
9949 %}
9950
9951 ins_pipe(ialu_reg);
9952 %}
9953
9954 // This pattern is automatically generated from aarch64_ad.m4
9955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9956 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9957 immI0 zero, iRegIorL2I src1, immI src2) %{
9958 match(Set dst (SubI zero (URShiftI src1 src2)));
9959
9960 ins_cost(1.9 * INSN_COST);
9961 format %{ "negw $dst, $src1, LSR $src2" %}
9962
9963 ins_encode %{
9964 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9965 Assembler::LSR, $src2$$constant & 0x1f);
9966 %}
9967
9968 ins_pipe(ialu_reg_shift);
9969 %}
9970
9971 // This pattern is automatically generated from aarch64_ad.m4
9972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9973 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9974 immI0 zero, iRegIorL2I src1, immI src2) %{
9975 match(Set dst (SubI zero (RShiftI src1 src2)));
9976
9977 ins_cost(1.9 * INSN_COST);
9978 format %{ "negw $dst, $src1, ASR $src2" %}
9979
9980 ins_encode %{
9981 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9982 Assembler::ASR, $src2$$constant & 0x1f);
9983 %}
9984
9985 ins_pipe(ialu_reg_shift);
9986 %}
9987
9988 // This pattern is automatically generated from aarch64_ad.m4
9989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9990 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9991 immI0 zero, iRegIorL2I src1, immI src2) %{
9992 match(Set dst (SubI zero (LShiftI src1 src2)));
9993
9994 ins_cost(1.9 * INSN_COST);
9995 format %{ "negw $dst, $src1, LSL $src2" %}
9996
9997 ins_encode %{
9998 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9999 Assembler::LSL, $src2$$constant & 0x1f);
10000 %}
10001
10002 ins_pipe(ialu_reg_shift);
10003 %}
10004
10005 // This pattern is automatically generated from aarch64_ad.m4
10006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10007 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10008 immL0 zero, iRegL src1, immI src2) %{
10009 match(Set dst (SubL zero (URShiftL src1 src2)));
10010
10011 ins_cost(1.9 * INSN_COST);
10012 format %{ "neg $dst, $src1, LSR $src2" %}
10013
10014 ins_encode %{
10015 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10016 Assembler::LSR, $src2$$constant & 0x3f);
10017 %}
10018
10019 ins_pipe(ialu_reg_shift);
10020 %}
10021
10022 // This pattern is automatically generated from aarch64_ad.m4
10023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10024 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10025 immL0 zero, iRegL src1, immI src2) %{
10026 match(Set dst (SubL zero (RShiftL src1 src2)));
10027
10028 ins_cost(1.9 * INSN_COST);
10029 format %{ "neg $dst, $src1, ASR $src2" %}
10030
10031 ins_encode %{
10032 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10033 Assembler::ASR, $src2$$constant & 0x3f);
10034 %}
10035
10036 ins_pipe(ialu_reg_shift);
10037 %}
10038
10039 // This pattern is automatically generated from aarch64_ad.m4
10040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10041 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10042 immL0 zero, iRegL src1, immI src2) %{
10043 match(Set dst (SubL zero (LShiftL src1 src2)));
10044
10045 ins_cost(1.9 * INSN_COST);
10046 format %{ "neg $dst, $src1, LSL $src2" %}
10047
10048 ins_encode %{
10049 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10050 Assembler::LSL, $src2$$constant & 0x3f);
10051 %}
10052
10053 ins_pipe(ialu_reg_shift);
10054 %}
10055
10056 // This pattern is automatically generated from aarch64_ad.m4
10057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10058 instruct AndI_reg_not_reg(iRegINoSp dst,
10059 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10060 match(Set dst (AndI src1 (XorI src2 m1)));
10061 ins_cost(INSN_COST);
10062 format %{ "bicw $dst, $src1, $src2" %}
10063
10064 ins_encode %{
10065 __ bicw(as_Register($dst$$reg),
10066 as_Register($src1$$reg),
10067 as_Register($src2$$reg),
10068 Assembler::LSL, 0);
10069 %}
10070
10071 ins_pipe(ialu_reg_reg);
10072 %}
10073
10074 // This pattern is automatically generated from aarch64_ad.m4
10075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10076 instruct AndL_reg_not_reg(iRegLNoSp dst,
10077 iRegL src1, iRegL src2, immL_M1 m1) %{
10078 match(Set dst (AndL src1 (XorL src2 m1)));
10079 ins_cost(INSN_COST);
10080 format %{ "bic $dst, $src1, $src2" %}
10081
10082 ins_encode %{
10083 __ bic(as_Register($dst$$reg),
10084 as_Register($src1$$reg),
10085 as_Register($src2$$reg),
10086 Assembler::LSL, 0);
10087 %}
10088
10089 ins_pipe(ialu_reg_reg);
10090 %}
10091
10092 // This pattern is automatically generated from aarch64_ad.m4
10093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10094 instruct OrI_reg_not_reg(iRegINoSp dst,
10095 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10096 match(Set dst (OrI src1 (XorI src2 m1)));
10097 ins_cost(INSN_COST);
10098 format %{ "ornw $dst, $src1, $src2" %}
10099
10100 ins_encode %{
10101 __ ornw(as_Register($dst$$reg),
10102 as_Register($src1$$reg),
10103 as_Register($src2$$reg),
10104 Assembler::LSL, 0);
10105 %}
10106
10107 ins_pipe(ialu_reg_reg);
10108 %}
10109
10110 // This pattern is automatically generated from aarch64_ad.m4
10111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10112 instruct OrL_reg_not_reg(iRegLNoSp dst,
10113 iRegL src1, iRegL src2, immL_M1 m1) %{
10114 match(Set dst (OrL src1 (XorL src2 m1)));
10115 ins_cost(INSN_COST);
10116 format %{ "orn $dst, $src1, $src2" %}
10117
10118 ins_encode %{
10119 __ orn(as_Register($dst$$reg),
10120 as_Register($src1$$reg),
10121 as_Register($src2$$reg),
10122 Assembler::LSL, 0);
10123 %}
10124
10125 ins_pipe(ialu_reg_reg);
10126 %}
10127
10128 // This pattern is automatically generated from aarch64_ad.m4
10129 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10130 instruct XorI_reg_not_reg(iRegINoSp dst,
10131 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10132 match(Set dst (XorI m1 (XorI src2 src1)));
10133 ins_cost(INSN_COST);
10134 format %{ "eonw $dst, $src1, $src2" %}
10135
10136 ins_encode %{
10137 __ eonw(as_Register($dst$$reg),
10138 as_Register($src1$$reg),
10139 as_Register($src2$$reg),
10140 Assembler::LSL, 0);
10141 %}
10142
10143 ins_pipe(ialu_reg_reg);
10144 %}
10145
10146 // This pattern is automatically generated from aarch64_ad.m4
10147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10148 instruct XorL_reg_not_reg(iRegLNoSp dst,
10149 iRegL src1, iRegL src2, immL_M1 m1) %{
10150 match(Set dst (XorL m1 (XorL src2 src1)));
10151 ins_cost(INSN_COST);
10152 format %{ "eon $dst, $src1, $src2" %}
10153
10154 ins_encode %{
10155 __ eon(as_Register($dst$$reg),
10156 as_Register($src1$$reg),
10157 as_Register($src2$$reg),
10158 Assembler::LSL, 0);
10159 %}
10160
10161 ins_pipe(ialu_reg_reg);
10162 %}
10163
10164 // This pattern is automatically generated from aarch64_ad.m4
10165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10166 // val & (-1 ^ (val >>> shift)) ==> bicw
10167 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10168 iRegIorL2I src1, iRegIorL2I src2,
10169 immI src3, immI_M1 src4) %{
10170 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10171 ins_cost(1.9 * INSN_COST);
10172 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10173
10174 ins_encode %{
10175 __ bicw(as_Register($dst$$reg),
10176 as_Register($src1$$reg),
10177 as_Register($src2$$reg),
10178 Assembler::LSR,
10179 $src3$$constant & 0x1f);
10180 %}
10181
10182 ins_pipe(ialu_reg_reg_shift);
10183 %}
10184
10185 // This pattern is automatically generated from aarch64_ad.m4
10186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10187 // val & (-1 ^ (val >>> shift)) ==> bic
10188 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10189 iRegL src1, iRegL src2,
10190 immI src3, immL_M1 src4) %{
10191 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10192 ins_cost(1.9 * INSN_COST);
10193 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10194
10195 ins_encode %{
10196 __ bic(as_Register($dst$$reg),
10197 as_Register($src1$$reg),
10198 as_Register($src2$$reg),
10199 Assembler::LSR,
10200 $src3$$constant & 0x3f);
10201 %}
10202
10203 ins_pipe(ialu_reg_reg_shift);
10204 %}
10205
10206 // This pattern is automatically generated from aarch64_ad.m4
10207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10208 // val & (-1 ^ (val >> shift)) ==> bicw
10209 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10210 iRegIorL2I src1, iRegIorL2I src2,
10211 immI src3, immI_M1 src4) %{
10212 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10213 ins_cost(1.9 * INSN_COST);
10214 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10215
10216 ins_encode %{
10217 __ bicw(as_Register($dst$$reg),
10218 as_Register($src1$$reg),
10219 as_Register($src2$$reg),
10220 Assembler::ASR,
10221 $src3$$constant & 0x1f);
10222 %}
10223
10224 ins_pipe(ialu_reg_reg_shift);
10225 %}
10226
10227 // This pattern is automatically generated from aarch64_ad.m4
10228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10229 // val & (-1 ^ (val >> shift)) ==> bic
10230 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10231 iRegL src1, iRegL src2,
10232 immI src3, immL_M1 src4) %{
10233 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10234 ins_cost(1.9 * INSN_COST);
10235 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10236
10237 ins_encode %{
10238 __ bic(as_Register($dst$$reg),
10239 as_Register($src1$$reg),
10240 as_Register($src2$$reg),
10241 Assembler::ASR,
10242 $src3$$constant & 0x3f);
10243 %}
10244
10245 ins_pipe(ialu_reg_reg_shift);
10246 %}
10247
10248 // This pattern is automatically generated from aarch64_ad.m4
10249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10250 // val & (-1 ^ (val ror shift)) ==> bicw
10251 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10252 iRegIorL2I src1, iRegIorL2I src2,
10253 immI src3, immI_M1 src4) %{
10254 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10255 ins_cost(1.9 * INSN_COST);
10256 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10257
10258 ins_encode %{
10259 __ bicw(as_Register($dst$$reg),
10260 as_Register($src1$$reg),
10261 as_Register($src2$$reg),
10262 Assembler::ROR,
10263 $src3$$constant & 0x1f);
10264 %}
10265
10266 ins_pipe(ialu_reg_reg_shift);
10267 %}
10268
10269 // This pattern is automatically generated from aarch64_ad.m4
10270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10271 // val & (-1 ^ (val ror shift)) ==> bic
10272 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10273 iRegL src1, iRegL src2,
10274 immI src3, immL_M1 src4) %{
10275 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10276 ins_cost(1.9 * INSN_COST);
10277 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10278
10279 ins_encode %{
10280 __ bic(as_Register($dst$$reg),
10281 as_Register($src1$$reg),
10282 as_Register($src2$$reg),
10283 Assembler::ROR,
10284 $src3$$constant & 0x3f);
10285 %}
10286
10287 ins_pipe(ialu_reg_reg_shift);
10288 %}
10289
10290 // This pattern is automatically generated from aarch64_ad.m4
10291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10292 // val & (-1 ^ (val << shift)) ==> bicw
10293 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10294 iRegIorL2I src1, iRegIorL2I src2,
10295 immI src3, immI_M1 src4) %{
10296 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10297 ins_cost(1.9 * INSN_COST);
10298 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10299
10300 ins_encode %{
10301 __ bicw(as_Register($dst$$reg),
10302 as_Register($src1$$reg),
10303 as_Register($src2$$reg),
10304 Assembler::LSL,
10305 $src3$$constant & 0x1f);
10306 %}
10307
10308 ins_pipe(ialu_reg_reg_shift);
10309 %}
10310
10311 // This pattern is automatically generated from aarch64_ad.m4
10312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10313 // val & (-1 ^ (val << shift)) ==> bic
10314 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10315 iRegL src1, iRegL src2,
10316 immI src3, immL_M1 src4) %{
10317 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10318 ins_cost(1.9 * INSN_COST);
10319 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10320
10321 ins_encode %{
10322 __ bic(as_Register($dst$$reg),
10323 as_Register($src1$$reg),
10324 as_Register($src2$$reg),
10325 Assembler::LSL,
10326 $src3$$constant & 0x3f);
10327 %}
10328
10329 ins_pipe(ialu_reg_reg_shift);
10330 %}
10331
10332 // This pattern is automatically generated from aarch64_ad.m4
10333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10334 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10335 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10336 iRegIorL2I src1, iRegIorL2I src2,
10337 immI src3, immI_M1 src4) %{
10338 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10339 ins_cost(1.9 * INSN_COST);
10340 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10341
10342 ins_encode %{
10343 __ eonw(as_Register($dst$$reg),
10344 as_Register($src1$$reg),
10345 as_Register($src2$$reg),
10346 Assembler::LSR,
10347 $src3$$constant & 0x1f);
10348 %}
10349
10350 ins_pipe(ialu_reg_reg_shift);
10351 %}
10352
10353 // This pattern is automatically generated from aarch64_ad.m4
10354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10355 // val ^ (-1 ^ (val >>> shift)) ==> eon
10356 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10357 iRegL src1, iRegL src2,
10358 immI src3, immL_M1 src4) %{
10359 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10360 ins_cost(1.9 * INSN_COST);
10361 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10362
10363 ins_encode %{
10364 __ eon(as_Register($dst$$reg),
10365 as_Register($src1$$reg),
10366 as_Register($src2$$reg),
10367 Assembler::LSR,
10368 $src3$$constant & 0x3f);
10369 %}
10370
10371 ins_pipe(ialu_reg_reg_shift);
10372 %}
10373
10374 // This pattern is automatically generated from aarch64_ad.m4
10375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10376 // val ^ (-1 ^ (val >> shift)) ==> eonw
10377 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10378 iRegIorL2I src1, iRegIorL2I src2,
10379 immI src3, immI_M1 src4) %{
10380 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10381 ins_cost(1.9 * INSN_COST);
10382 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10383
10384 ins_encode %{
10385 __ eonw(as_Register($dst$$reg),
10386 as_Register($src1$$reg),
10387 as_Register($src2$$reg),
10388 Assembler::ASR,
10389 $src3$$constant & 0x1f);
10390 %}
10391
10392 ins_pipe(ialu_reg_reg_shift);
10393 %}
10394
10395 // This pattern is automatically generated from aarch64_ad.m4
10396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10397 // val ^ (-1 ^ (val >> shift)) ==> eon
10398 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10399 iRegL src1, iRegL src2,
10400 immI src3, immL_M1 src4) %{
10401 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10402 ins_cost(1.9 * INSN_COST);
10403 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10404
10405 ins_encode %{
10406 __ eon(as_Register($dst$$reg),
10407 as_Register($src1$$reg),
10408 as_Register($src2$$reg),
10409 Assembler::ASR,
10410 $src3$$constant & 0x3f);
10411 %}
10412
10413 ins_pipe(ialu_reg_reg_shift);
10414 %}
10415
10416 // This pattern is automatically generated from aarch64_ad.m4
10417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10418 // val ^ (-1 ^ (val ror shift)) ==> eonw
10419 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10420 iRegIorL2I src1, iRegIorL2I src2,
10421 immI src3, immI_M1 src4) %{
10422 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10423 ins_cost(1.9 * INSN_COST);
10424 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10425
10426 ins_encode %{
10427 __ eonw(as_Register($dst$$reg),
10428 as_Register($src1$$reg),
10429 as_Register($src2$$reg),
10430 Assembler::ROR,
10431 $src3$$constant & 0x1f);
10432 %}
10433
10434 ins_pipe(ialu_reg_reg_shift);
10435 %}
10436
10437 // This pattern is automatically generated from aarch64_ad.m4
10438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10439 // val ^ (-1 ^ (val ror shift)) ==> eon
10440 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10441 iRegL src1, iRegL src2,
10442 immI src3, immL_M1 src4) %{
10443 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10444 ins_cost(1.9 * INSN_COST);
10445 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10446
10447 ins_encode %{
10448 __ eon(as_Register($dst$$reg),
10449 as_Register($src1$$reg),
10450 as_Register($src2$$reg),
10451 Assembler::ROR,
10452 $src3$$constant & 0x3f);
10453 %}
10454
10455 ins_pipe(ialu_reg_reg_shift);
10456 %}
10457
10458 // This pattern is automatically generated from aarch64_ad.m4
10459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10460 // val ^ (-1 ^ (val << shift)) ==> eonw
10461 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10462 iRegIorL2I src1, iRegIorL2I src2,
10463 immI src3, immI_M1 src4) %{
10464 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10465 ins_cost(1.9 * INSN_COST);
10466 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10467
10468 ins_encode %{
10469 __ eonw(as_Register($dst$$reg),
10470 as_Register($src1$$reg),
10471 as_Register($src2$$reg),
10472 Assembler::LSL,
10473 $src3$$constant & 0x1f);
10474 %}
10475
10476 ins_pipe(ialu_reg_reg_shift);
10477 %}
10478
10479 // This pattern is automatically generated from aarch64_ad.m4
10480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10481 // val ^ (-1 ^ (val << shift)) ==> eon
10482 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10483 iRegL src1, iRegL src2,
10484 immI src3, immL_M1 src4) %{
10485 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10486 ins_cost(1.9 * INSN_COST);
10487 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10488
10489 ins_encode %{
10490 __ eon(as_Register($dst$$reg),
10491 as_Register($src1$$reg),
10492 as_Register($src2$$reg),
10493 Assembler::LSL,
10494 $src3$$constant & 0x3f);
10495 %}
10496
10497 ins_pipe(ialu_reg_reg_shift);
10498 %}
10499
10500 // This pattern is automatically generated from aarch64_ad.m4
10501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10502 // val | (-1 ^ (val >>> shift)) ==> ornw
10503 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10504 iRegIorL2I src1, iRegIorL2I src2,
10505 immI src3, immI_M1 src4) %{
10506 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10507 ins_cost(1.9 * INSN_COST);
10508 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10509
10510 ins_encode %{
10511 __ ornw(as_Register($dst$$reg),
10512 as_Register($src1$$reg),
10513 as_Register($src2$$reg),
10514 Assembler::LSR,
10515 $src3$$constant & 0x1f);
10516 %}
10517
10518 ins_pipe(ialu_reg_reg_shift);
10519 %}
10520
10521 // This pattern is automatically generated from aarch64_ad.m4
10522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10523 // val | (-1 ^ (val >>> shift)) ==> orn
10524 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10525 iRegL src1, iRegL src2,
10526 immI src3, immL_M1 src4) %{
10527 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10528 ins_cost(1.9 * INSN_COST);
10529 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10530
10531 ins_encode %{
10532 __ orn(as_Register($dst$$reg),
10533 as_Register($src1$$reg),
10534 as_Register($src2$$reg),
10535 Assembler::LSR,
10536 $src3$$constant & 0x3f);
10537 %}
10538
10539 ins_pipe(ialu_reg_reg_shift);
10540 %}
10541
10542 // This pattern is automatically generated from aarch64_ad.m4
10543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10544 // val | (-1 ^ (val >> shift)) ==> ornw
10545 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10546 iRegIorL2I src1, iRegIorL2I src2,
10547 immI src3, immI_M1 src4) %{
10548 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10549 ins_cost(1.9 * INSN_COST);
10550 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10551
10552 ins_encode %{
10553 __ ornw(as_Register($dst$$reg),
10554 as_Register($src1$$reg),
10555 as_Register($src2$$reg),
10556 Assembler::ASR,
10557 $src3$$constant & 0x1f);
10558 %}
10559
10560 ins_pipe(ialu_reg_reg_shift);
10561 %}
10562
10563 // This pattern is automatically generated from aarch64_ad.m4
10564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10565 // val | (-1 ^ (val >> shift)) ==> orn
10566 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10567 iRegL src1, iRegL src2,
10568 immI src3, immL_M1 src4) %{
10569 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10570 ins_cost(1.9 * INSN_COST);
10571 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10572
10573 ins_encode %{
10574 __ orn(as_Register($dst$$reg),
10575 as_Register($src1$$reg),
10576 as_Register($src2$$reg),
10577 Assembler::ASR,
10578 $src3$$constant & 0x3f);
10579 %}
10580
10581 ins_pipe(ialu_reg_reg_shift);
10582 %}
10583
10584 // This pattern is automatically generated from aarch64_ad.m4
10585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10586 // val | (-1 ^ (val ror shift)) ==> ornw
10587 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10588 iRegIorL2I src1, iRegIorL2I src2,
10589 immI src3, immI_M1 src4) %{
10590 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10591 ins_cost(1.9 * INSN_COST);
10592 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10593
10594 ins_encode %{
10595 __ ornw(as_Register($dst$$reg),
10596 as_Register($src1$$reg),
10597 as_Register($src2$$reg),
10598 Assembler::ROR,
10599 $src3$$constant & 0x1f);
10600 %}
10601
10602 ins_pipe(ialu_reg_reg_shift);
10603 %}
10604
10605 // This pattern is automatically generated from aarch64_ad.m4
10606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10607 // val | (-1 ^ (val ror shift)) ==> orn
10608 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10609 iRegL src1, iRegL src2,
10610 immI src3, immL_M1 src4) %{
10611 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10612 ins_cost(1.9 * INSN_COST);
10613 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10614
10615 ins_encode %{
10616 __ orn(as_Register($dst$$reg),
10617 as_Register($src1$$reg),
10618 as_Register($src2$$reg),
10619 Assembler::ROR,
10620 $src3$$constant & 0x3f);
10621 %}
10622
10623 ins_pipe(ialu_reg_reg_shift);
10624 %}
10625
10626 // This pattern is automatically generated from aarch64_ad.m4
10627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10628 // val | (-1 ^ (val << shift)) ==> ornw
10629 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10630 iRegIorL2I src1, iRegIorL2I src2,
10631 immI src3, immI_M1 src4) %{
10632 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10633 ins_cost(1.9 * INSN_COST);
10634 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10635
10636 ins_encode %{
10637 __ ornw(as_Register($dst$$reg),
10638 as_Register($src1$$reg),
10639 as_Register($src2$$reg),
10640 Assembler::LSL,
10641 $src3$$constant & 0x1f);
10642 %}
10643
10644 ins_pipe(ialu_reg_reg_shift);
10645 %}
10646
10647 // This pattern is automatically generated from aarch64_ad.m4
10648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10649 // val | (-1 ^ (val << shift)) ==> orn
10650 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10651 iRegL src1, iRegL src2,
10652 immI src3, immL_M1 src4) %{
10653 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10654 ins_cost(1.9 * INSN_COST);
10655 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10656
10657 ins_encode %{
10658 __ orn(as_Register($dst$$reg),
10659 as_Register($src1$$reg),
10660 as_Register($src2$$reg),
10661 Assembler::LSL,
10662 $src3$$constant & 0x3f);
10663 %}
10664
10665 ins_pipe(ialu_reg_reg_shift);
10666 %}
10667
10668 // This pattern is automatically generated from aarch64_ad.m4
10669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10670 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10671 iRegIorL2I src1, iRegIorL2I src2,
10672 immI src3) %{
10673 match(Set dst (AndI src1 (URShiftI src2 src3)));
10674
10675 ins_cost(1.9 * INSN_COST);
10676 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10677
10678 ins_encode %{
10679 __ andw(as_Register($dst$$reg),
10680 as_Register($src1$$reg),
10681 as_Register($src2$$reg),
10682 Assembler::LSR,
10683 $src3$$constant & 0x1f);
10684 %}
10685
10686 ins_pipe(ialu_reg_reg_shift);
10687 %}
10688
10689 // This pattern is automatically generated from aarch64_ad.m4
10690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10691 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10692 iRegL src1, iRegL src2,
10693 immI src3) %{
10694 match(Set dst (AndL src1 (URShiftL src2 src3)));
10695
10696 ins_cost(1.9 * INSN_COST);
10697 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10698
10699 ins_encode %{
10700 __ andr(as_Register($dst$$reg),
10701 as_Register($src1$$reg),
10702 as_Register($src2$$reg),
10703 Assembler::LSR,
10704 $src3$$constant & 0x3f);
10705 %}
10706
10707 ins_pipe(ialu_reg_reg_shift);
10708 %}
10709
10710 // This pattern is automatically generated from aarch64_ad.m4
10711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10712 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10713 iRegIorL2I src1, iRegIorL2I src2,
10714 immI src3) %{
10715 match(Set dst (AndI src1 (RShiftI src2 src3)));
10716
10717 ins_cost(1.9 * INSN_COST);
10718 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10719
10720 ins_encode %{
10721 __ andw(as_Register($dst$$reg),
10722 as_Register($src1$$reg),
10723 as_Register($src2$$reg),
10724 Assembler::ASR,
10725 $src3$$constant & 0x1f);
10726 %}
10727
10728 ins_pipe(ialu_reg_reg_shift);
10729 %}
10730
10731 // This pattern is automatically generated from aarch64_ad.m4
10732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10733 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10734 iRegL src1, iRegL src2,
10735 immI src3) %{
10736 match(Set dst (AndL src1 (RShiftL src2 src3)));
10737
10738 ins_cost(1.9 * INSN_COST);
10739 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10740
10741 ins_encode %{
10742 __ andr(as_Register($dst$$reg),
10743 as_Register($src1$$reg),
10744 as_Register($src2$$reg),
10745 Assembler::ASR,
10746 $src3$$constant & 0x3f);
10747 %}
10748
10749 ins_pipe(ialu_reg_reg_shift);
10750 %}
10751
10752 // This pattern is automatically generated from aarch64_ad.m4
10753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10754 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10755 iRegIorL2I src1, iRegIorL2I src2,
10756 immI src3) %{
10757 match(Set dst (AndI src1 (LShiftI src2 src3)));
10758
10759 ins_cost(1.9 * INSN_COST);
10760 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10761
10762 ins_encode %{
10763 __ andw(as_Register($dst$$reg),
10764 as_Register($src1$$reg),
10765 as_Register($src2$$reg),
10766 Assembler::LSL,
10767 $src3$$constant & 0x1f);
10768 %}
10769
10770 ins_pipe(ialu_reg_reg_shift);
10771 %}
10772
10773 // This pattern is automatically generated from aarch64_ad.m4
10774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10775 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10776 iRegL src1, iRegL src2,
10777 immI src3) %{
10778 match(Set dst (AndL src1 (LShiftL src2 src3)));
10779
10780 ins_cost(1.9 * INSN_COST);
10781 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10782
10783 ins_encode %{
10784 __ andr(as_Register($dst$$reg),
10785 as_Register($src1$$reg),
10786 as_Register($src2$$reg),
10787 Assembler::LSL,
10788 $src3$$constant & 0x3f);
10789 %}
10790
10791 ins_pipe(ialu_reg_reg_shift);
10792 %}
10793
10794 // This pattern is automatically generated from aarch64_ad.m4
10795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10796 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10797 iRegIorL2I src1, iRegIorL2I src2,
10798 immI src3) %{
10799 match(Set dst (AndI src1 (RotateRight src2 src3)));
10800
10801 ins_cost(1.9 * INSN_COST);
10802 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10803
10804 ins_encode %{
10805 __ andw(as_Register($dst$$reg),
10806 as_Register($src1$$reg),
10807 as_Register($src2$$reg),
10808 Assembler::ROR,
10809 $src3$$constant & 0x1f);
10810 %}
10811
10812 ins_pipe(ialu_reg_reg_shift);
10813 %}
10814
10815 // This pattern is automatically generated from aarch64_ad.m4
10816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10817 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10818 iRegL src1, iRegL src2,
10819 immI src3) %{
10820 match(Set dst (AndL src1 (RotateRight src2 src3)));
10821
10822 ins_cost(1.9 * INSN_COST);
10823 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10824
10825 ins_encode %{
10826 __ andr(as_Register($dst$$reg),
10827 as_Register($src1$$reg),
10828 as_Register($src2$$reg),
10829 Assembler::ROR,
10830 $src3$$constant & 0x3f);
10831 %}
10832
10833 ins_pipe(ialu_reg_reg_shift);
10834 %}
10835
10836 // This pattern is automatically generated from aarch64_ad.m4
10837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10838 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10839 iRegIorL2I src1, iRegIorL2I src2,
10840 immI src3) %{
10841 match(Set dst (XorI src1 (URShiftI src2 src3)));
10842
10843 ins_cost(1.9 * INSN_COST);
10844 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10845
10846 ins_encode %{
10847 __ eorw(as_Register($dst$$reg),
10848 as_Register($src1$$reg),
10849 as_Register($src2$$reg),
10850 Assembler::LSR,
10851 $src3$$constant & 0x1f);
10852 %}
10853
10854 ins_pipe(ialu_reg_reg_shift);
10855 %}
10856
10857 // This pattern is automatically generated from aarch64_ad.m4
10858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10859 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10860 iRegL src1, iRegL src2,
10861 immI src3) %{
10862 match(Set dst (XorL src1 (URShiftL src2 src3)));
10863
10864 ins_cost(1.9 * INSN_COST);
10865 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10866
10867 ins_encode %{
10868 __ eor(as_Register($dst$$reg),
10869 as_Register($src1$$reg),
10870 as_Register($src2$$reg),
10871 Assembler::LSR,
10872 $src3$$constant & 0x3f);
10873 %}
10874
10875 ins_pipe(ialu_reg_reg_shift);
10876 %}
10877
10878 // This pattern is automatically generated from aarch64_ad.m4
10879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10880 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10881 iRegIorL2I src1, iRegIorL2I src2,
10882 immI src3) %{
10883 match(Set dst (XorI src1 (RShiftI src2 src3)));
10884
10885 ins_cost(1.9 * INSN_COST);
10886 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10887
10888 ins_encode %{
10889 __ eorw(as_Register($dst$$reg),
10890 as_Register($src1$$reg),
10891 as_Register($src2$$reg),
10892 Assembler::ASR,
10893 $src3$$constant & 0x1f);
10894 %}
10895
10896 ins_pipe(ialu_reg_reg_shift);
10897 %}
10898
10899 // This pattern is automatically generated from aarch64_ad.m4
10900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10901 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10902 iRegL src1, iRegL src2,
10903 immI src3) %{
10904 match(Set dst (XorL src1 (RShiftL src2 src3)));
10905
10906 ins_cost(1.9 * INSN_COST);
10907 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10908
10909 ins_encode %{
10910 __ eor(as_Register($dst$$reg),
10911 as_Register($src1$$reg),
10912 as_Register($src2$$reg),
10913 Assembler::ASR,
10914 $src3$$constant & 0x3f);
10915 %}
10916
10917 ins_pipe(ialu_reg_reg_shift);
10918 %}
10919
10920 // This pattern is automatically generated from aarch64_ad.m4
10921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10922 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10923 iRegIorL2I src1, iRegIorL2I src2,
10924 immI src3) %{
10925 match(Set dst (XorI src1 (LShiftI src2 src3)));
10926
10927 ins_cost(1.9 * INSN_COST);
10928 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10929
10930 ins_encode %{
10931 __ eorw(as_Register($dst$$reg),
10932 as_Register($src1$$reg),
10933 as_Register($src2$$reg),
10934 Assembler::LSL,
10935 $src3$$constant & 0x1f);
10936 %}
10937
10938 ins_pipe(ialu_reg_reg_shift);
10939 %}
10940
10941 // This pattern is automatically generated from aarch64_ad.m4
10942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10943 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10944 iRegL src1, iRegL src2,
10945 immI src3) %{
10946 match(Set dst (XorL src1 (LShiftL src2 src3)));
10947
10948 ins_cost(1.9 * INSN_COST);
10949 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10950
10951 ins_encode %{
10952 __ eor(as_Register($dst$$reg),
10953 as_Register($src1$$reg),
10954 as_Register($src2$$reg),
10955 Assembler::LSL,
10956 $src3$$constant & 0x3f);
10957 %}
10958
10959 ins_pipe(ialu_reg_reg_shift);
10960 %}
10961
10962 // This pattern is automatically generated from aarch64_ad.m4
10963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10964 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10965 iRegIorL2I src1, iRegIorL2I src2,
10966 immI src3) %{
10967 match(Set dst (XorI src1 (RotateRight src2 src3)));
10968
10969 ins_cost(1.9 * INSN_COST);
10970 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10971
10972 ins_encode %{
10973 __ eorw(as_Register($dst$$reg),
10974 as_Register($src1$$reg),
10975 as_Register($src2$$reg),
10976 Assembler::ROR,
10977 $src3$$constant & 0x1f);
10978 %}
10979
10980 ins_pipe(ialu_reg_reg_shift);
10981 %}
10982
10983 // This pattern is automatically generated from aarch64_ad.m4
10984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10985 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10986 iRegL src1, iRegL src2,
10987 immI src3) %{
10988 match(Set dst (XorL src1 (RotateRight src2 src3)));
10989
10990 ins_cost(1.9 * INSN_COST);
10991 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10992
10993 ins_encode %{
10994 __ eor(as_Register($dst$$reg),
10995 as_Register($src1$$reg),
10996 as_Register($src2$$reg),
10997 Assembler::ROR,
10998 $src3$$constant & 0x3f);
10999 %}
11000
11001 ins_pipe(ialu_reg_reg_shift);
11002 %}
11003
11004 // This pattern is automatically generated from aarch64_ad.m4
11005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11006 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11007 iRegIorL2I src1, iRegIorL2I src2,
11008 immI src3) %{
11009 match(Set dst (OrI src1 (URShiftI src2 src3)));
11010
11011 ins_cost(1.9 * INSN_COST);
11012 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11013
11014 ins_encode %{
11015 __ orrw(as_Register($dst$$reg),
11016 as_Register($src1$$reg),
11017 as_Register($src2$$reg),
11018 Assembler::LSR,
11019 $src3$$constant & 0x1f);
11020 %}
11021
11022 ins_pipe(ialu_reg_reg_shift);
11023 %}
11024
11025 // This pattern is automatically generated from aarch64_ad.m4
11026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11027 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11028 iRegL src1, iRegL src2,
11029 immI src3) %{
11030 match(Set dst (OrL src1 (URShiftL src2 src3)));
11031
11032 ins_cost(1.9 * INSN_COST);
11033 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11034
11035 ins_encode %{
11036 __ orr(as_Register($dst$$reg),
11037 as_Register($src1$$reg),
11038 as_Register($src2$$reg),
11039 Assembler::LSR,
11040 $src3$$constant & 0x3f);
11041 %}
11042
11043 ins_pipe(ialu_reg_reg_shift);
11044 %}
11045
11046 // This pattern is automatically generated from aarch64_ad.m4
11047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11048 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11049 iRegIorL2I src1, iRegIorL2I src2,
11050 immI src3) %{
11051 match(Set dst (OrI src1 (RShiftI src2 src3)));
11052
11053 ins_cost(1.9 * INSN_COST);
11054 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11055
11056 ins_encode %{
11057 __ orrw(as_Register($dst$$reg),
11058 as_Register($src1$$reg),
11059 as_Register($src2$$reg),
11060 Assembler::ASR,
11061 $src3$$constant & 0x1f);
11062 %}
11063
11064 ins_pipe(ialu_reg_reg_shift);
11065 %}
11066
11067 // This pattern is automatically generated from aarch64_ad.m4
11068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11069 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11070 iRegL src1, iRegL src2,
11071 immI src3) %{
11072 match(Set dst (OrL src1 (RShiftL src2 src3)));
11073
11074 ins_cost(1.9 * INSN_COST);
11075 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11076
11077 ins_encode %{
11078 __ orr(as_Register($dst$$reg),
11079 as_Register($src1$$reg),
11080 as_Register($src2$$reg),
11081 Assembler::ASR,
11082 $src3$$constant & 0x3f);
11083 %}
11084
11085 ins_pipe(ialu_reg_reg_shift);
11086 %}
11087
11088 // This pattern is automatically generated from aarch64_ad.m4
11089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11090 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11091 iRegIorL2I src1, iRegIorL2I src2,
11092 immI src3) %{
11093 match(Set dst (OrI src1 (LShiftI src2 src3)));
11094
11095 ins_cost(1.9 * INSN_COST);
11096 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11097
11098 ins_encode %{
11099 __ orrw(as_Register($dst$$reg),
11100 as_Register($src1$$reg),
11101 as_Register($src2$$reg),
11102 Assembler::LSL,
11103 $src3$$constant & 0x1f);
11104 %}
11105
11106 ins_pipe(ialu_reg_reg_shift);
11107 %}
11108
11109 // This pattern is automatically generated from aarch64_ad.m4
11110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11111 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11112 iRegL src1, iRegL src2,
11113 immI src3) %{
11114 match(Set dst (OrL src1 (LShiftL src2 src3)));
11115
11116 ins_cost(1.9 * INSN_COST);
11117 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11118
11119 ins_encode %{
11120 __ orr(as_Register($dst$$reg),
11121 as_Register($src1$$reg),
11122 as_Register($src2$$reg),
11123 Assembler::LSL,
11124 $src3$$constant & 0x3f);
11125 %}
11126
11127 ins_pipe(ialu_reg_reg_shift);
11128 %}
11129
11130 // This pattern is automatically generated from aarch64_ad.m4
11131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11132 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11133 iRegIorL2I src1, iRegIorL2I src2,
11134 immI src3) %{
11135 match(Set dst (OrI src1 (RotateRight src2 src3)));
11136
11137 ins_cost(1.9 * INSN_COST);
11138 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11139
11140 ins_encode %{
11141 __ orrw(as_Register($dst$$reg),
11142 as_Register($src1$$reg),
11143 as_Register($src2$$reg),
11144 Assembler::ROR,
11145 $src3$$constant & 0x1f);
11146 %}
11147
11148 ins_pipe(ialu_reg_reg_shift);
11149 %}
11150
11151 // This pattern is automatically generated from aarch64_ad.m4
11152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11153 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11154 iRegL src1, iRegL src2,
11155 immI src3) %{
11156 match(Set dst (OrL src1 (RotateRight src2 src3)));
11157
11158 ins_cost(1.9 * INSN_COST);
11159 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11160
11161 ins_encode %{
11162 __ orr(as_Register($dst$$reg),
11163 as_Register($src1$$reg),
11164 as_Register($src2$$reg),
11165 Assembler::ROR,
11166 $src3$$constant & 0x3f);
11167 %}
11168
11169 ins_pipe(ialu_reg_reg_shift);
11170 %}
11171
11172 // This pattern is automatically generated from aarch64_ad.m4
11173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11174 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11175 iRegIorL2I src1, iRegIorL2I src2,
11176 immI src3) %{
11177 match(Set dst (AddI src1 (URShiftI src2 src3)));
11178
11179 ins_cost(1.9 * INSN_COST);
11180 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11181
11182 ins_encode %{
11183 __ addw(as_Register($dst$$reg),
11184 as_Register($src1$$reg),
11185 as_Register($src2$$reg),
11186 Assembler::LSR,
11187 $src3$$constant & 0x1f);
11188 %}
11189
11190 ins_pipe(ialu_reg_reg_shift);
11191 %}
11192
11193 // This pattern is automatically generated from aarch64_ad.m4
11194 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11195 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11196 iRegL src1, iRegL src2,
11197 immI src3) %{
11198 match(Set dst (AddL src1 (URShiftL src2 src3)));
11199
11200 ins_cost(1.9 * INSN_COST);
11201 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11202
11203 ins_encode %{
11204 __ add(as_Register($dst$$reg),
11205 as_Register($src1$$reg),
11206 as_Register($src2$$reg),
11207 Assembler::LSR,
11208 $src3$$constant & 0x3f);
11209 %}
11210
11211 ins_pipe(ialu_reg_reg_shift);
11212 %}
11213
11214 // This pattern is automatically generated from aarch64_ad.m4
11215 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11216 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11217 iRegIorL2I src1, iRegIorL2I src2,
11218 immI src3) %{
11219 match(Set dst (AddI src1 (RShiftI src2 src3)));
11220
11221 ins_cost(1.9 * INSN_COST);
11222 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11223
11224 ins_encode %{
11225 __ addw(as_Register($dst$$reg),
11226 as_Register($src1$$reg),
11227 as_Register($src2$$reg),
11228 Assembler::ASR,
11229 $src3$$constant & 0x1f);
11230 %}
11231
11232 ins_pipe(ialu_reg_reg_shift);
11233 %}
11234
11235 // This pattern is automatically generated from aarch64_ad.m4
11236 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11237 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11238 iRegL src1, iRegL src2,
11239 immI src3) %{
11240 match(Set dst (AddL src1 (RShiftL src2 src3)));
11241
11242 ins_cost(1.9 * INSN_COST);
11243 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11244
11245 ins_encode %{
11246 __ add(as_Register($dst$$reg),
11247 as_Register($src1$$reg),
11248 as_Register($src2$$reg),
11249 Assembler::ASR,
11250 $src3$$constant & 0x3f);
11251 %}
11252
11253 ins_pipe(ialu_reg_reg_shift);
11254 %}
11255
11256 // This pattern is automatically generated from aarch64_ad.m4
11257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11258 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11259 iRegIorL2I src1, iRegIorL2I src2,
11260 immI src3) %{
11261 match(Set dst (AddI src1 (LShiftI src2 src3)));
11262
11263 ins_cost(1.9 * INSN_COST);
11264 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11265
11266 ins_encode %{
11267 __ addw(as_Register($dst$$reg),
11268 as_Register($src1$$reg),
11269 as_Register($src2$$reg),
11270 Assembler::LSL,
11271 $src3$$constant & 0x1f);
11272 %}
11273
11274 ins_pipe(ialu_reg_reg_shift);
11275 %}
11276
11277 // This pattern is automatically generated from aarch64_ad.m4
11278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11279 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11280 iRegL src1, iRegL src2,
11281 immI src3) %{
11282 match(Set dst (AddL src1 (LShiftL src2 src3)));
11283
11284 ins_cost(1.9 * INSN_COST);
11285 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11286
11287 ins_encode %{
11288 __ add(as_Register($dst$$reg),
11289 as_Register($src1$$reg),
11290 as_Register($src2$$reg),
11291 Assembler::LSL,
11292 $src3$$constant & 0x3f);
11293 %}
11294
11295 ins_pipe(ialu_reg_reg_shift);
11296 %}
11297
11298 // This pattern is automatically generated from aarch64_ad.m4
11299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11300 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11301 iRegIorL2I src1, iRegIorL2I src2,
11302 immI src3) %{
11303 match(Set dst (SubI src1 (URShiftI src2 src3)));
11304
11305 ins_cost(1.9 * INSN_COST);
11306 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11307
11308 ins_encode %{
11309 __ subw(as_Register($dst$$reg),
11310 as_Register($src1$$reg),
11311 as_Register($src2$$reg),
11312 Assembler::LSR,
11313 $src3$$constant & 0x1f);
11314 %}
11315
11316 ins_pipe(ialu_reg_reg_shift);
11317 %}
11318
11319 // This pattern is automatically generated from aarch64_ad.m4
11320 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11321 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11322 iRegL src1, iRegL src2,
11323 immI src3) %{
11324 match(Set dst (SubL src1 (URShiftL src2 src3)));
11325
11326 ins_cost(1.9 * INSN_COST);
11327 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11328
11329 ins_encode %{
11330 __ sub(as_Register($dst$$reg),
11331 as_Register($src1$$reg),
11332 as_Register($src2$$reg),
11333 Assembler::LSR,
11334 $src3$$constant & 0x3f);
11335 %}
11336
11337 ins_pipe(ialu_reg_reg_shift);
11338 %}
11339
11340 // This pattern is automatically generated from aarch64_ad.m4
11341 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11342 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11343 iRegIorL2I src1, iRegIorL2I src2,
11344 immI src3) %{
11345 match(Set dst (SubI src1 (RShiftI src2 src3)));
11346
11347 ins_cost(1.9 * INSN_COST);
11348 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11349
11350 ins_encode %{
11351 __ subw(as_Register($dst$$reg),
11352 as_Register($src1$$reg),
11353 as_Register($src2$$reg),
11354 Assembler::ASR,
11355 $src3$$constant & 0x1f);
11356 %}
11357
11358 ins_pipe(ialu_reg_reg_shift);
11359 %}
11360
11361 // This pattern is automatically generated from aarch64_ad.m4
11362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11363 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11364 iRegL src1, iRegL src2,
11365 immI src3) %{
11366 match(Set dst (SubL src1 (RShiftL src2 src3)));
11367
11368 ins_cost(1.9 * INSN_COST);
11369 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11370
11371 ins_encode %{
11372 __ sub(as_Register($dst$$reg),
11373 as_Register($src1$$reg),
11374 as_Register($src2$$reg),
11375 Assembler::ASR,
11376 $src3$$constant & 0x3f);
11377 %}
11378
11379 ins_pipe(ialu_reg_reg_shift);
11380 %}
11381
11382 // This pattern is automatically generated from aarch64_ad.m4
11383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11384 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11385 iRegIorL2I src1, iRegIorL2I src2,
11386 immI src3) %{
11387 match(Set dst (SubI src1 (LShiftI src2 src3)));
11388
11389 ins_cost(1.9 * INSN_COST);
11390 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11391
11392 ins_encode %{
11393 __ subw(as_Register($dst$$reg),
11394 as_Register($src1$$reg),
11395 as_Register($src2$$reg),
11396 Assembler::LSL,
11397 $src3$$constant & 0x1f);
11398 %}
11399
11400 ins_pipe(ialu_reg_reg_shift);
11401 %}
11402
11403 // This pattern is automatically generated from aarch64_ad.m4
11404 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11405 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11406 iRegL src1, iRegL src2,
11407 immI src3) %{
11408 match(Set dst (SubL src1 (LShiftL src2 src3)));
11409
11410 ins_cost(1.9 * INSN_COST);
11411 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11412
11413 ins_encode %{
11414 __ sub(as_Register($dst$$reg),
11415 as_Register($src1$$reg),
11416 as_Register($src2$$reg),
11417 Assembler::LSL,
11418 $src3$$constant & 0x3f);
11419 %}
11420
11421 ins_pipe(ialu_reg_reg_shift);
11422 %}
11423
11424 // This pattern is automatically generated from aarch64_ad.m4
11425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11426
11427 // Shift Left followed by Shift Right.
11428 // This idiom is used by the compiler for the i2b bytecode etc.
11429 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11430 %{
11431 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11432 ins_cost(INSN_COST * 2);
11433 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11434 ins_encode %{
11435 int lshift = $lshift_count$$constant & 63;
11436 int rshift = $rshift_count$$constant & 63;
11437 int s = 63 - lshift;
11438 int r = (rshift - lshift) & 63;
11439 __ sbfm(as_Register($dst$$reg),
11440 as_Register($src$$reg),
11441 r, s);
11442 %}
11443
11444 ins_pipe(ialu_reg_shift);
11445 %}
11446
11447 // This pattern is automatically generated from aarch64_ad.m4
11448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11449
11450 // Shift Left followed by Shift Right.
11451 // This idiom is used by the compiler for the i2b bytecode etc.
11452 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11453 %{
11454 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11455 ins_cost(INSN_COST * 2);
11456 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11457 ins_encode %{
11458 int lshift = $lshift_count$$constant & 31;
11459 int rshift = $rshift_count$$constant & 31;
11460 int s = 31 - lshift;
11461 int r = (rshift - lshift) & 31;
11462 __ sbfmw(as_Register($dst$$reg),
11463 as_Register($src$$reg),
11464 r, s);
11465 %}
11466
11467 ins_pipe(ialu_reg_shift);
11468 %}
11469
11470 // This pattern is automatically generated from aarch64_ad.m4
11471 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11472
11473 // Shift Left followed by Shift Right.
11474 // This idiom is used by the compiler for the i2b bytecode etc.
11475 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11476 %{
11477 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11478 ins_cost(INSN_COST * 2);
11479 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11480 ins_encode %{
11481 int lshift = $lshift_count$$constant & 63;
11482 int rshift = $rshift_count$$constant & 63;
11483 int s = 63 - lshift;
11484 int r = (rshift - lshift) & 63;
11485 __ ubfm(as_Register($dst$$reg),
11486 as_Register($src$$reg),
11487 r, s);
11488 %}
11489
11490 ins_pipe(ialu_reg_shift);
11491 %}
11492
11493 // This pattern is automatically generated from aarch64_ad.m4
11494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11495
11496 // Shift Left followed by Shift Right.
11497 // This idiom is used by the compiler for the i2b bytecode etc.
11498 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11499 %{
11500 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11501 ins_cost(INSN_COST * 2);
11502 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11503 ins_encode %{
11504 int lshift = $lshift_count$$constant & 31;
11505 int rshift = $rshift_count$$constant & 31;
11506 int s = 31 - lshift;
11507 int r = (rshift - lshift) & 31;
11508 __ ubfmw(as_Register($dst$$reg),
11509 as_Register($src$$reg),
11510 r, s);
11511 %}
11512
11513 ins_pipe(ialu_reg_shift);
11514 %}
11515
11516 // Bitfield extract with shift & mask
11517
11518 // This pattern is automatically generated from aarch64_ad.m4
11519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11520 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11521 %{
11522 match(Set dst (AndI (URShiftI src rshift) mask));
11523 // Make sure we are not going to exceed what ubfxw can do.
11524 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11525
11526 ins_cost(INSN_COST);
11527 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11528 ins_encode %{
11529 int rshift = $rshift$$constant & 31;
11530 intptr_t mask = $mask$$constant;
11531 int width = exact_log2(mask+1);
11532 __ ubfxw(as_Register($dst$$reg),
11533 as_Register($src$$reg), rshift, width);
11534 %}
11535 ins_pipe(ialu_reg_shift);
11536 %}
11537
11538 // This pattern is automatically generated from aarch64_ad.m4
11539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11540 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11541 %{
11542 match(Set dst (AndL (URShiftL src rshift) mask));
11543 // Make sure we are not going to exceed what ubfx can do.
11544 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11545
11546 ins_cost(INSN_COST);
11547 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11548 ins_encode %{
11549 int rshift = $rshift$$constant & 63;
11550 intptr_t mask = $mask$$constant;
11551 int width = exact_log2_long(mask+1);
11552 __ ubfx(as_Register($dst$$reg),
11553 as_Register($src$$reg), rshift, width);
11554 %}
11555 ins_pipe(ialu_reg_shift);
11556 %}
11557
11558
11559 // This pattern is automatically generated from aarch64_ad.m4
11560 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11561
11562 // We can use ubfx when extending an And with a mask when we know mask
11563 // is positive. We know that because immI_bitmask guarantees it.
11564 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11565 %{
11566 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11567 // Make sure we are not going to exceed what ubfxw can do.
11568 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11569
11570 ins_cost(INSN_COST * 2);
11571 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11572 ins_encode %{
11573 int rshift = $rshift$$constant & 31;
11574 intptr_t mask = $mask$$constant;
11575 int width = exact_log2(mask+1);
11576 __ ubfx(as_Register($dst$$reg),
11577 as_Register($src$$reg), rshift, width);
11578 %}
11579 ins_pipe(ialu_reg_shift);
11580 %}
11581
11582
11583 // This pattern is automatically generated from aarch64_ad.m4
11584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11585
11586 // We can use ubfiz when masking by a positive number and then left shifting the result.
11587 // We know that the mask is positive because immI_bitmask guarantees it.
11588 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11589 %{
11590 match(Set dst (LShiftI (AndI src mask) lshift));
11591 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11592
11593 ins_cost(INSN_COST);
11594 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11595 ins_encode %{
11596 int lshift = $lshift$$constant & 31;
11597 intptr_t mask = $mask$$constant;
11598 int width = exact_log2(mask+1);
11599 __ ubfizw(as_Register($dst$$reg),
11600 as_Register($src$$reg), lshift, width);
11601 %}
11602 ins_pipe(ialu_reg_shift);
11603 %}
11604
11605 // This pattern is automatically generated from aarch64_ad.m4
11606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11607
11608 // We can use ubfiz when masking by a positive number and then left shifting the result.
11609 // We know that the mask is positive because immL_bitmask guarantees it.
11610 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11611 %{
11612 match(Set dst (LShiftL (AndL src mask) lshift));
11613 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11614
11615 ins_cost(INSN_COST);
11616 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11617 ins_encode %{
11618 int lshift = $lshift$$constant & 63;
11619 intptr_t mask = $mask$$constant;
11620 int width = exact_log2_long(mask+1);
11621 __ ubfiz(as_Register($dst$$reg),
11622 as_Register($src$$reg), lshift, width);
11623 %}
11624 ins_pipe(ialu_reg_shift);
11625 %}
11626
11627 // This pattern is automatically generated from aarch64_ad.m4
11628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11629
11630 // We can use ubfiz when masking by a positive number and then left shifting the result.
11631 // We know that the mask is positive because immI_bitmask guarantees it.
11632 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11633 %{
11634 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11635 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11636
11637 ins_cost(INSN_COST);
11638 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11639 ins_encode %{
11640 int lshift = $lshift$$constant & 31;
11641 intptr_t mask = $mask$$constant;
11642 int width = exact_log2(mask+1);
11643 __ ubfizw(as_Register($dst$$reg),
11644 as_Register($src$$reg), lshift, width);
11645 %}
11646 ins_pipe(ialu_reg_shift);
11647 %}
11648
11649 // This pattern is automatically generated from aarch64_ad.m4
11650 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11651
11652 // We can use ubfiz when masking by a positive number and then left shifting the result.
11653 // We know that the mask is positive because immL_bitmask guarantees it.
11654 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11655 %{
11656 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11657 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11658
11659 ins_cost(INSN_COST);
11660 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11661 ins_encode %{
11662 int lshift = $lshift$$constant & 63;
11663 intptr_t mask = $mask$$constant;
11664 int width = exact_log2_long(mask+1);
11665 __ ubfiz(as_Register($dst$$reg),
11666 as_Register($src$$reg), lshift, width);
11667 %}
11668 ins_pipe(ialu_reg_shift);
11669 %}
11670
11671
11672 // This pattern is automatically generated from aarch64_ad.m4
11673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11674
11675 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11676 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11677 %{
11678 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11679 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11680
11681 ins_cost(INSN_COST);
11682 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11683 ins_encode %{
11684 int lshift = $lshift$$constant & 63;
11685 intptr_t mask = $mask$$constant;
11686 int width = exact_log2(mask+1);
11687 __ ubfiz(as_Register($dst$$reg),
11688 as_Register($src$$reg), lshift, width);
11689 %}
11690 ins_pipe(ialu_reg_shift);
11691 %}
11692
11693 // This pattern is automatically generated from aarch64_ad.m4
11694 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11695
11696 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11697 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11698 %{
11699 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11700 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11701
11702 ins_cost(INSN_COST);
11703 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11704 ins_encode %{
11705 int lshift = $lshift$$constant & 31;
11706 intptr_t mask = $mask$$constant;
11707 int width = exact_log2(mask+1);
11708 __ ubfiz(as_Register($dst$$reg),
11709 as_Register($src$$reg), lshift, width);
11710 %}
11711 ins_pipe(ialu_reg_shift);
11712 %}
11713
11714 // This pattern is automatically generated from aarch64_ad.m4
11715 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11716
11717 // Can skip int2long conversions after AND with small bitmask
11718 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11719 %{
11720 match(Set dst (ConvI2L (AndI src msk)));
11721 ins_cost(INSN_COST);
11722 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11723 ins_encode %{
11724 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11725 %}
11726 ins_pipe(ialu_reg_shift);
11727 %}
11728
11729
11730 // Rotations
11731
11732 // This pattern is automatically generated from aarch64_ad.m4
11733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11734 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11735 %{
11736 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11737 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11738
11739 ins_cost(INSN_COST);
11740 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11741
11742 ins_encode %{
11743 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11744 $rshift$$constant & 63);
11745 %}
11746 ins_pipe(ialu_reg_reg_extr);
11747 %}
11748
11749
11750 // This pattern is automatically generated from aarch64_ad.m4
11751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11752 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11753 %{
11754 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11755 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11756
11757 ins_cost(INSN_COST);
11758 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11759
11760 ins_encode %{
11761 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11762 $rshift$$constant & 31);
11763 %}
11764 ins_pipe(ialu_reg_reg_extr);
11765 %}
11766
11767
11768 // This pattern is automatically generated from aarch64_ad.m4
11769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11770 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11771 %{
11772 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11773 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11774
11775 ins_cost(INSN_COST);
11776 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11777
11778 ins_encode %{
11779 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11780 $rshift$$constant & 63);
11781 %}
11782 ins_pipe(ialu_reg_reg_extr);
11783 %}
11784
11785
11786 // This pattern is automatically generated from aarch64_ad.m4
11787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11788 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11789 %{
11790 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11791 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11792
11793 ins_cost(INSN_COST);
11794 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11795
11796 ins_encode %{
11797 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11798 $rshift$$constant & 31);
11799 %}
11800 ins_pipe(ialu_reg_reg_extr);
11801 %}
11802
11803 // This pattern is automatically generated from aarch64_ad.m4
11804 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11805 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11806 %{
11807 match(Set dst (RotateRight src shift));
11808
11809 ins_cost(INSN_COST);
11810 format %{ "ror $dst, $src, $shift" %}
11811
11812 ins_encode %{
11813 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11814 $shift$$constant & 0x1f);
11815 %}
11816 ins_pipe(ialu_reg_reg_vshift);
11817 %}
11818
11819 // This pattern is automatically generated from aarch64_ad.m4
11820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11821 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11822 %{
11823 match(Set dst (RotateRight src shift));
11824
11825 ins_cost(INSN_COST);
11826 format %{ "ror $dst, $src, $shift" %}
11827
11828 ins_encode %{
11829 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11830 $shift$$constant & 0x3f);
11831 %}
11832 ins_pipe(ialu_reg_reg_vshift);
11833 %}
11834
11835 // This pattern is automatically generated from aarch64_ad.m4
11836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11837 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11838 %{
11839 match(Set dst (RotateRight src shift));
11840
11841 ins_cost(INSN_COST);
11842 format %{ "ror $dst, $src, $shift" %}
11843
11844 ins_encode %{
11845 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11846 %}
11847 ins_pipe(ialu_reg_reg_vshift);
11848 %}
11849
11850 // This pattern is automatically generated from aarch64_ad.m4
11851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11852 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11853 %{
11854 match(Set dst (RotateRight src shift));
11855
11856 ins_cost(INSN_COST);
11857 format %{ "ror $dst, $src, $shift" %}
11858
11859 ins_encode %{
11860 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11861 %}
11862 ins_pipe(ialu_reg_reg_vshift);
11863 %}
11864
11865 // This pattern is automatically generated from aarch64_ad.m4
11866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11867 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11868 %{
11869 match(Set dst (RotateLeft src shift));
11870
11871 ins_cost(INSN_COST);
11872 format %{ "rol $dst, $src, $shift" %}
11873
11874 ins_encode %{
11875 __ subw(rscratch1, zr, as_Register($shift$$reg));
11876 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11877 %}
11878 ins_pipe(ialu_reg_reg_vshift);
11879 %}
11880
11881 // This pattern is automatically generated from aarch64_ad.m4
11882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11883 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11884 %{
11885 match(Set dst (RotateLeft src shift));
11886
11887 ins_cost(INSN_COST);
11888 format %{ "rol $dst, $src, $shift" %}
11889
11890 ins_encode %{
11891 __ subw(rscratch1, zr, as_Register($shift$$reg));
11892 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11893 %}
11894 ins_pipe(ialu_reg_reg_vshift);
11895 %}
11896
11897
11898 // Add/subtract (extended)
11899
11900 // This pattern is automatically generated from aarch64_ad.m4
11901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11902 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11903 %{
11904 match(Set dst (AddL src1 (ConvI2L src2)));
11905 ins_cost(INSN_COST);
11906 format %{ "add $dst, $src1, $src2, sxtw" %}
11907
11908 ins_encode %{
11909 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11910 as_Register($src2$$reg), ext::sxtw);
11911 %}
11912 ins_pipe(ialu_reg_reg);
11913 %}
11914
11915 // This pattern is automatically generated from aarch64_ad.m4
11916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11917 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11918 %{
11919 match(Set dst (SubL src1 (ConvI2L src2)));
11920 ins_cost(INSN_COST);
11921 format %{ "sub $dst, $src1, $src2, sxtw" %}
11922
11923 ins_encode %{
11924 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11925 as_Register($src2$$reg), ext::sxtw);
11926 %}
11927 ins_pipe(ialu_reg_reg);
11928 %}
11929
11930 // This pattern is automatically generated from aarch64_ad.m4
11931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11932 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11933 %{
11934 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11935 ins_cost(INSN_COST);
11936 format %{ "add $dst, $src1, $src2, sxth" %}
11937
11938 ins_encode %{
11939 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11940 as_Register($src2$$reg), ext::sxth);
11941 %}
11942 ins_pipe(ialu_reg_reg);
11943 %}
11944
11945 // This pattern is automatically generated from aarch64_ad.m4
11946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11947 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11948 %{
11949 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11950 ins_cost(INSN_COST);
11951 format %{ "add $dst, $src1, $src2, sxtb" %}
11952
11953 ins_encode %{
11954 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11955 as_Register($src2$$reg), ext::sxtb);
11956 %}
11957 ins_pipe(ialu_reg_reg);
11958 %}
11959
11960 // This pattern is automatically generated from aarch64_ad.m4
11961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11962 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11963 %{
11964 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11965 ins_cost(INSN_COST);
11966 format %{ "add $dst, $src1, $src2, uxtb" %}
11967
11968 ins_encode %{
11969 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11970 as_Register($src2$$reg), ext::uxtb);
11971 %}
11972 ins_pipe(ialu_reg_reg);
11973 %}
11974
11975 // This pattern is automatically generated from aarch64_ad.m4
11976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11977 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11978 %{
11979 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11980 ins_cost(INSN_COST);
11981 format %{ "add $dst, $src1, $src2, sxth" %}
11982
11983 ins_encode %{
11984 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11985 as_Register($src2$$reg), ext::sxth);
11986 %}
11987 ins_pipe(ialu_reg_reg);
11988 %}
11989
11990 // This pattern is automatically generated from aarch64_ad.m4
11991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11992 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11993 %{
11994 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11995 ins_cost(INSN_COST);
11996 format %{ "add $dst, $src1, $src2, sxtw" %}
11997
11998 ins_encode %{
11999 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12000 as_Register($src2$$reg), ext::sxtw);
12001 %}
12002 ins_pipe(ialu_reg_reg);
12003 %}
12004
12005 // This pattern is automatically generated from aarch64_ad.m4
12006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12007 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12008 %{
12009 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12010 ins_cost(INSN_COST);
12011 format %{ "add $dst, $src1, $src2, sxtb" %}
12012
12013 ins_encode %{
12014 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12015 as_Register($src2$$reg), ext::sxtb);
12016 %}
12017 ins_pipe(ialu_reg_reg);
12018 %}
12019
12020 // This pattern is automatically generated from aarch64_ad.m4
12021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12022 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12023 %{
12024 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12025 ins_cost(INSN_COST);
12026 format %{ "add $dst, $src1, $src2, uxtb" %}
12027
12028 ins_encode %{
12029 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12030 as_Register($src2$$reg), ext::uxtb);
12031 %}
12032 ins_pipe(ialu_reg_reg);
12033 %}
12034
12035 // This pattern is automatically generated from aarch64_ad.m4
12036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12037 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12038 %{
12039 match(Set dst (AddI src1 (AndI src2 mask)));
12040 ins_cost(INSN_COST);
12041 format %{ "addw $dst, $src1, $src2, uxtb" %}
12042
12043 ins_encode %{
12044 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12045 as_Register($src2$$reg), ext::uxtb);
12046 %}
12047 ins_pipe(ialu_reg_reg);
12048 %}
12049
12050 // This pattern is automatically generated from aarch64_ad.m4
12051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12052 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12053 %{
12054 match(Set dst (AddI src1 (AndI src2 mask)));
12055 ins_cost(INSN_COST);
12056 format %{ "addw $dst, $src1, $src2, uxth" %}
12057
12058 ins_encode %{
12059 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12060 as_Register($src2$$reg), ext::uxth);
12061 %}
12062 ins_pipe(ialu_reg_reg);
12063 %}
12064
12065 // This pattern is automatically generated from aarch64_ad.m4
12066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12067 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12068 %{
12069 match(Set dst (AddL src1 (AndL src2 mask)));
12070 ins_cost(INSN_COST);
12071 format %{ "add $dst, $src1, $src2, uxtb" %}
12072
12073 ins_encode %{
12074 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12075 as_Register($src2$$reg), ext::uxtb);
12076 %}
12077 ins_pipe(ialu_reg_reg);
12078 %}
12079
12080 // This pattern is automatically generated from aarch64_ad.m4
12081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12082 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12083 %{
12084 match(Set dst (AddL src1 (AndL src2 mask)));
12085 ins_cost(INSN_COST);
12086 format %{ "add $dst, $src1, $src2, uxth" %}
12087
12088 ins_encode %{
12089 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12090 as_Register($src2$$reg), ext::uxth);
12091 %}
12092 ins_pipe(ialu_reg_reg);
12093 %}
12094
12095 // This pattern is automatically generated from aarch64_ad.m4
12096 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12097 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12098 %{
12099 match(Set dst (AddL src1 (AndL src2 mask)));
12100 ins_cost(INSN_COST);
12101 format %{ "add $dst, $src1, $src2, uxtw" %}
12102
12103 ins_encode %{
12104 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12105 as_Register($src2$$reg), ext::uxtw);
12106 %}
12107 ins_pipe(ialu_reg_reg);
12108 %}
12109
12110 // This pattern is automatically generated from aarch64_ad.m4
12111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12112 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12113 %{
12114 match(Set dst (SubI src1 (AndI src2 mask)));
12115 ins_cost(INSN_COST);
12116 format %{ "subw $dst, $src1, $src2, uxtb" %}
12117
12118 ins_encode %{
12119 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12120 as_Register($src2$$reg), ext::uxtb);
12121 %}
12122 ins_pipe(ialu_reg_reg);
12123 %}
12124
12125 // This pattern is automatically generated from aarch64_ad.m4
12126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12127 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12128 %{
12129 match(Set dst (SubI src1 (AndI src2 mask)));
12130 ins_cost(INSN_COST);
12131 format %{ "subw $dst, $src1, $src2, uxth" %}
12132
12133 ins_encode %{
12134 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12135 as_Register($src2$$reg), ext::uxth);
12136 %}
12137 ins_pipe(ialu_reg_reg);
12138 %}
12139
12140 // This pattern is automatically generated from aarch64_ad.m4
12141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12142 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12143 %{
12144 match(Set dst (SubL src1 (AndL src2 mask)));
12145 ins_cost(INSN_COST);
12146 format %{ "sub $dst, $src1, $src2, uxtb" %}
12147
12148 ins_encode %{
12149 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12150 as_Register($src2$$reg), ext::uxtb);
12151 %}
12152 ins_pipe(ialu_reg_reg);
12153 %}
12154
12155 // This pattern is automatically generated from aarch64_ad.m4
12156 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12157 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12158 %{
12159 match(Set dst (SubL src1 (AndL src2 mask)));
12160 ins_cost(INSN_COST);
12161 format %{ "sub $dst, $src1, $src2, uxth" %}
12162
12163 ins_encode %{
12164 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12165 as_Register($src2$$reg), ext::uxth);
12166 %}
12167 ins_pipe(ialu_reg_reg);
12168 %}
12169
12170 // This pattern is automatically generated from aarch64_ad.m4
12171 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12172 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12173 %{
12174 match(Set dst (SubL src1 (AndL src2 mask)));
12175 ins_cost(INSN_COST);
12176 format %{ "sub $dst, $src1, $src2, uxtw" %}
12177
12178 ins_encode %{
12179 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12180 as_Register($src2$$reg), ext::uxtw);
12181 %}
12182 ins_pipe(ialu_reg_reg);
12183 %}
12184
12185
12186 // This pattern is automatically generated from aarch64_ad.m4
12187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12188 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12189 %{
12190 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12191 ins_cost(1.9 * INSN_COST);
12192 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12193
12194 ins_encode %{
12195 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12196 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12197 %}
12198 ins_pipe(ialu_reg_reg_shift);
12199 %}
12200
12201 // This pattern is automatically generated from aarch64_ad.m4
12202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12203 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12204 %{
12205 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12206 ins_cost(1.9 * INSN_COST);
12207 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12208
12209 ins_encode %{
12210 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12211 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12212 %}
12213 ins_pipe(ialu_reg_reg_shift);
12214 %}
12215
12216 // This pattern is automatically generated from aarch64_ad.m4
12217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12218 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12219 %{
12220 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12221 ins_cost(1.9 * INSN_COST);
12222 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12223
12224 ins_encode %{
12225 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12226 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12227 %}
12228 ins_pipe(ialu_reg_reg_shift);
12229 %}
12230
12231 // This pattern is automatically generated from aarch64_ad.m4
12232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12233 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12234 %{
12235 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12236 ins_cost(1.9 * INSN_COST);
12237 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12238
12239 ins_encode %{
12240 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12241 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12242 %}
12243 ins_pipe(ialu_reg_reg_shift);
12244 %}
12245
12246 // This pattern is automatically generated from aarch64_ad.m4
12247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12248 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12249 %{
12250 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12251 ins_cost(1.9 * INSN_COST);
12252 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12253
12254 ins_encode %{
12255 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12256 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12257 %}
12258 ins_pipe(ialu_reg_reg_shift);
12259 %}
12260
12261 // This pattern is automatically generated from aarch64_ad.m4
12262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12263 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12264 %{
12265 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12266 ins_cost(1.9 * INSN_COST);
12267 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12268
12269 ins_encode %{
12270 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12271 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12272 %}
12273 ins_pipe(ialu_reg_reg_shift);
12274 %}
12275
12276 // This pattern is automatically generated from aarch64_ad.m4
12277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12278 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12279 %{
12280 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12281 ins_cost(1.9 * INSN_COST);
12282 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12283
12284 ins_encode %{
12285 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12286 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12287 %}
12288 ins_pipe(ialu_reg_reg_shift);
12289 %}
12290
12291 // This pattern is automatically generated from aarch64_ad.m4
12292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12293 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12294 %{
12295 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12296 ins_cost(1.9 * INSN_COST);
12297 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12298
12299 ins_encode %{
12300 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12301 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12302 %}
12303 ins_pipe(ialu_reg_reg_shift);
12304 %}
12305
12306 // This pattern is automatically generated from aarch64_ad.m4
12307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12308 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12309 %{
12310 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12311 ins_cost(1.9 * INSN_COST);
12312 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12313
12314 ins_encode %{
12315 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12316 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12317 %}
12318 ins_pipe(ialu_reg_reg_shift);
12319 %}
12320
12321 // This pattern is automatically generated from aarch64_ad.m4
12322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12323 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12324 %{
12325 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12326 ins_cost(1.9 * INSN_COST);
12327 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12328
12329 ins_encode %{
12330 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12331 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12332 %}
12333 ins_pipe(ialu_reg_reg_shift);
12334 %}
12335
12336 // This pattern is automatically generated from aarch64_ad.m4
12337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12338 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12339 %{
12340 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12341 ins_cost(1.9 * INSN_COST);
12342 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12343
12344 ins_encode %{
12345 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12346 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12347 %}
12348 ins_pipe(ialu_reg_reg_shift);
12349 %}
12350
12351 // This pattern is automatically generated from aarch64_ad.m4
12352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12353 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12354 %{
12355 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12356 ins_cost(1.9 * INSN_COST);
12357 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12358
12359 ins_encode %{
12360 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12361 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12362 %}
12363 ins_pipe(ialu_reg_reg_shift);
12364 %}
12365
12366 // This pattern is automatically generated from aarch64_ad.m4
12367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12368 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12369 %{
12370 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12371 ins_cost(1.9 * INSN_COST);
12372 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12373
12374 ins_encode %{
12375 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12376 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12377 %}
12378 ins_pipe(ialu_reg_reg_shift);
12379 %}
12380
12381 // This pattern is automatically generated from aarch64_ad.m4
12382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12383 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12384 %{
12385 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12386 ins_cost(1.9 * INSN_COST);
12387 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12388
12389 ins_encode %{
12390 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12391 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12392 %}
12393 ins_pipe(ialu_reg_reg_shift);
12394 %}
12395
12396 // This pattern is automatically generated from aarch64_ad.m4
12397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12398 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12399 %{
12400 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12401 ins_cost(1.9 * INSN_COST);
12402 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12403
12404 ins_encode %{
12405 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12406 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12407 %}
12408 ins_pipe(ialu_reg_reg_shift);
12409 %}
12410
12411 // This pattern is automatically generated from aarch64_ad.m4
12412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12413 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12414 %{
12415 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12416 ins_cost(1.9 * INSN_COST);
12417 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12418
12419 ins_encode %{
12420 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12421 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12422 %}
12423 ins_pipe(ialu_reg_reg_shift);
12424 %}
12425
12426 // This pattern is automatically generated from aarch64_ad.m4
12427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12428 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12429 %{
12430 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12431 ins_cost(1.9 * INSN_COST);
12432 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12433
12434 ins_encode %{
12435 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12436 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12437 %}
12438 ins_pipe(ialu_reg_reg_shift);
12439 %}
12440
12441 // This pattern is automatically generated from aarch64_ad.m4
12442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12443 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12444 %{
12445 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12446 ins_cost(1.9 * INSN_COST);
12447 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12448
12449 ins_encode %{
12450 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12451 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12452 %}
12453 ins_pipe(ialu_reg_reg_shift);
12454 %}
12455
12456 // This pattern is automatically generated from aarch64_ad.m4
12457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12458 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12459 %{
12460 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12461 ins_cost(1.9 * INSN_COST);
12462 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12463
12464 ins_encode %{
12465 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12466 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12467 %}
12468 ins_pipe(ialu_reg_reg_shift);
12469 %}
12470
12471 // This pattern is automatically generated from aarch64_ad.m4
12472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12473 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12474 %{
12475 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12476 ins_cost(1.9 * INSN_COST);
12477 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12478
12479 ins_encode %{
12480 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12481 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12482 %}
12483 ins_pipe(ialu_reg_reg_shift);
12484 %}
12485
12486 // This pattern is automatically generated from aarch64_ad.m4
12487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12488 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12489 %{
12490 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12491 ins_cost(1.9 * INSN_COST);
12492 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12493
12494 ins_encode %{
12495 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12496 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12497 %}
12498 ins_pipe(ialu_reg_reg_shift);
12499 %}
12500
12501 // This pattern is automatically generated from aarch64_ad.m4
12502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12503 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12504 %{
12505 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12506 ins_cost(1.9 * INSN_COST);
12507 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12508
12509 ins_encode %{
12510 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12511 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12512 %}
12513 ins_pipe(ialu_reg_reg_shift);
12514 %}
12515
12516 // This pattern is automatically generated from aarch64_ad.m4
12517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12518 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12519 %{
12520 effect(DEF dst, USE src1, USE src2, USE cr);
12521 ins_cost(INSN_COST * 2);
12522 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12523
12524 ins_encode %{
12525 __ cselw($dst$$Register,
12526 $src1$$Register,
12527 $src2$$Register,
12528 Assembler::LT);
12529 %}
12530 ins_pipe(icond_reg_reg);
12531 %}
12532
12533 // This pattern is automatically generated from aarch64_ad.m4
12534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12535 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12536 %{
12537 effect(DEF dst, USE src1, USE src2, USE cr);
12538 ins_cost(INSN_COST * 2);
12539 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12540
12541 ins_encode %{
12542 __ cselw($dst$$Register,
12543 $src1$$Register,
12544 $src2$$Register,
12545 Assembler::GT);
12546 %}
12547 ins_pipe(icond_reg_reg);
12548 %}
12549
12550 // This pattern is automatically generated from aarch64_ad.m4
12551 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12552 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12553 %{
12554 effect(DEF dst, USE src1, USE cr);
12555 ins_cost(INSN_COST * 2);
12556 format %{ "cselw $dst, $src1, zr lt\t" %}
12557
12558 ins_encode %{
12559 __ cselw($dst$$Register,
12560 $src1$$Register,
12561 zr,
12562 Assembler::LT);
12563 %}
12564 ins_pipe(icond_reg);
12565 %}
12566
12567 // This pattern is automatically generated from aarch64_ad.m4
12568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12569 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12570 %{
12571 effect(DEF dst, USE src1, USE cr);
12572 ins_cost(INSN_COST * 2);
12573 format %{ "cselw $dst, $src1, zr gt\t" %}
12574
12575 ins_encode %{
12576 __ cselw($dst$$Register,
12577 $src1$$Register,
12578 zr,
12579 Assembler::GT);
12580 %}
12581 ins_pipe(icond_reg);
12582 %}
12583
12584 // This pattern is automatically generated from aarch64_ad.m4
12585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12586 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12587 %{
12588 effect(DEF dst, USE src1, USE cr);
12589 ins_cost(INSN_COST * 2);
12590 format %{ "csincw $dst, $src1, zr le\t" %}
12591
12592 ins_encode %{
12593 __ csincw($dst$$Register,
12594 $src1$$Register,
12595 zr,
12596 Assembler::LE);
12597 %}
12598 ins_pipe(icond_reg);
12599 %}
12600
12601 // This pattern is automatically generated from aarch64_ad.m4
12602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12603 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12604 %{
12605 effect(DEF dst, USE src1, USE cr);
12606 ins_cost(INSN_COST * 2);
12607 format %{ "csincw $dst, $src1, zr gt\t" %}
12608
12609 ins_encode %{
12610 __ csincw($dst$$Register,
12611 $src1$$Register,
12612 zr,
12613 Assembler::GT);
12614 %}
12615 ins_pipe(icond_reg);
12616 %}
12617
12618 // This pattern is automatically generated from aarch64_ad.m4
12619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12620 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12621 %{
12622 effect(DEF dst, USE src1, USE cr);
12623 ins_cost(INSN_COST * 2);
12624 format %{ "csinvw $dst, $src1, zr lt\t" %}
12625
12626 ins_encode %{
12627 __ csinvw($dst$$Register,
12628 $src1$$Register,
12629 zr,
12630 Assembler::LT);
12631 %}
12632 ins_pipe(icond_reg);
12633 %}
12634
12635 // This pattern is automatically generated from aarch64_ad.m4
12636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12637 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12638 %{
12639 effect(DEF dst, USE src1, USE cr);
12640 ins_cost(INSN_COST * 2);
12641 format %{ "csinvw $dst, $src1, zr ge\t" %}
12642
12643 ins_encode %{
12644 __ csinvw($dst$$Register,
12645 $src1$$Register,
12646 zr,
12647 Assembler::GE);
12648 %}
12649 ins_pipe(icond_reg);
12650 %}
12651
12652 // This pattern is automatically generated from aarch64_ad.m4
12653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12654 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12655 %{
12656 match(Set dst (MinI src imm));
12657 ins_cost(INSN_COST * 3);
12658 expand %{
12659 rFlagsReg cr;
12660 compI_reg_imm0(cr, src);
12661 cmovI_reg_imm0_lt(dst, src, cr);
12662 %}
12663 %}
12664
12665 // This pattern is automatically generated from aarch64_ad.m4
12666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12667 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12668 %{
12669 match(Set dst (MinI imm src));
12670 ins_cost(INSN_COST * 3);
12671 expand %{
12672 rFlagsReg cr;
12673 compI_reg_imm0(cr, src);
12674 cmovI_reg_imm0_lt(dst, src, cr);
12675 %}
12676 %}
12677
12678 // This pattern is automatically generated from aarch64_ad.m4
12679 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12680 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12681 %{
12682 match(Set dst (MinI src imm));
12683 ins_cost(INSN_COST * 3);
12684 expand %{
12685 rFlagsReg cr;
12686 compI_reg_imm0(cr, src);
12687 cmovI_reg_imm1_le(dst, src, cr);
12688 %}
12689 %}
12690
12691 // This pattern is automatically generated from aarch64_ad.m4
12692 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12693 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12694 %{
12695 match(Set dst (MinI imm src));
12696 ins_cost(INSN_COST * 3);
12697 expand %{
12698 rFlagsReg cr;
12699 compI_reg_imm0(cr, src);
12700 cmovI_reg_imm1_le(dst, src, cr);
12701 %}
12702 %}
12703
12704 // This pattern is automatically generated from aarch64_ad.m4
12705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12706 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12707 %{
12708 match(Set dst (MinI src imm));
12709 ins_cost(INSN_COST * 3);
12710 expand %{
12711 rFlagsReg cr;
12712 compI_reg_imm0(cr, src);
12713 cmovI_reg_immM1_lt(dst, src, cr);
12714 %}
12715 %}
12716
12717 // This pattern is automatically generated from aarch64_ad.m4
12718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12719 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12720 %{
12721 match(Set dst (MinI imm src));
12722 ins_cost(INSN_COST * 3);
12723 expand %{
12724 rFlagsReg cr;
12725 compI_reg_imm0(cr, src);
12726 cmovI_reg_immM1_lt(dst, src, cr);
12727 %}
12728 %}
12729
12730 // This pattern is automatically generated from aarch64_ad.m4
12731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12732 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12733 %{
12734 match(Set dst (MaxI src imm));
12735 ins_cost(INSN_COST * 3);
12736 expand %{
12737 rFlagsReg cr;
12738 compI_reg_imm0(cr, src);
12739 cmovI_reg_imm0_gt(dst, src, cr);
12740 %}
12741 %}
12742
12743 // This pattern is automatically generated from aarch64_ad.m4
12744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12745 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12746 %{
12747 match(Set dst (MaxI imm src));
12748 ins_cost(INSN_COST * 3);
12749 expand %{
12750 rFlagsReg cr;
12751 compI_reg_imm0(cr, src);
12752 cmovI_reg_imm0_gt(dst, src, cr);
12753 %}
12754 %}
12755
12756 // This pattern is automatically generated from aarch64_ad.m4
12757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12758 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12759 %{
12760 match(Set dst (MaxI src imm));
12761 ins_cost(INSN_COST * 3);
12762 expand %{
12763 rFlagsReg cr;
12764 compI_reg_imm0(cr, src);
12765 cmovI_reg_imm1_gt(dst, src, cr);
12766 %}
12767 %}
12768
12769 // This pattern is automatically generated from aarch64_ad.m4
12770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12771 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12772 %{
12773 match(Set dst (MaxI imm src));
12774 ins_cost(INSN_COST * 3);
12775 expand %{
12776 rFlagsReg cr;
12777 compI_reg_imm0(cr, src);
12778 cmovI_reg_imm1_gt(dst, src, cr);
12779 %}
12780 %}
12781
12782 // This pattern is automatically generated from aarch64_ad.m4
12783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12784 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12785 %{
12786 match(Set dst (MaxI src imm));
12787 ins_cost(INSN_COST * 3);
12788 expand %{
12789 rFlagsReg cr;
12790 compI_reg_imm0(cr, src);
12791 cmovI_reg_immM1_ge(dst, src, cr);
12792 %}
12793 %}
12794
12795 // This pattern is automatically generated from aarch64_ad.m4
12796 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12797 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12798 %{
12799 match(Set dst (MaxI imm src));
12800 ins_cost(INSN_COST * 3);
12801 expand %{
12802 rFlagsReg cr;
12803 compI_reg_imm0(cr, src);
12804 cmovI_reg_immM1_ge(dst, src, cr);
12805 %}
12806 %}
12807
12808 // This pattern is automatically generated from aarch64_ad.m4
12809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12810 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12811 %{
12812 match(Set dst (ReverseI src));
12813 ins_cost(INSN_COST);
12814 format %{ "rbitw $dst, $src" %}
12815 ins_encode %{
12816 __ rbitw($dst$$Register, $src$$Register);
12817 %}
12818 ins_pipe(ialu_reg);
12819 %}
12820
12821 // This pattern is automatically generated from aarch64_ad.m4
12822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12823 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12824 %{
12825 match(Set dst (ReverseL src));
12826 ins_cost(INSN_COST);
12827 format %{ "rbit $dst, $src" %}
12828 ins_encode %{
12829 __ rbit($dst$$Register, $src$$Register);
12830 %}
12831 ins_pipe(ialu_reg);
12832 %}
12833
12834
12835 // END This section of the file is automatically generated. Do not edit --------------
12836
12837
12838 // ============================================================================
12839 // Floating Point Arithmetic Instructions
12840
12841 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12842 match(Set dst (AddHF src1 src2));
12843 format %{ "faddh $dst, $src1, $src2" %}
12844 ins_encode %{
12845 __ faddh($dst$$FloatRegister,
12846 $src1$$FloatRegister,
12847 $src2$$FloatRegister);
12848 %}
12849 ins_pipe(fp_dop_reg_reg_s);
12850 %}
12851
12852 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12853 match(Set dst (AddF src1 src2));
12854
12855 ins_cost(INSN_COST * 5);
12856 format %{ "fadds $dst, $src1, $src2" %}
12857
12858 ins_encode %{
12859 __ fadds(as_FloatRegister($dst$$reg),
12860 as_FloatRegister($src1$$reg),
12861 as_FloatRegister($src2$$reg));
12862 %}
12863
12864 ins_pipe(fp_dop_reg_reg_s);
12865 %}
12866
12867 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12868 match(Set dst (AddD src1 src2));
12869
12870 ins_cost(INSN_COST * 5);
12871 format %{ "faddd $dst, $src1, $src2" %}
12872
12873 ins_encode %{
12874 __ faddd(as_FloatRegister($dst$$reg),
12875 as_FloatRegister($src1$$reg),
12876 as_FloatRegister($src2$$reg));
12877 %}
12878
12879 ins_pipe(fp_dop_reg_reg_d);
12880 %}
12881
12882 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12883 match(Set dst (SubHF src1 src2));
12884 format %{ "fsubh $dst, $src1, $src2" %}
12885 ins_encode %{
12886 __ fsubh($dst$$FloatRegister,
12887 $src1$$FloatRegister,
12888 $src2$$FloatRegister);
12889 %}
12890 ins_pipe(fp_dop_reg_reg_s);
12891 %}
12892
12893 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12894 match(Set dst (SubF src1 src2));
12895
12896 ins_cost(INSN_COST * 5);
12897 format %{ "fsubs $dst, $src1, $src2" %}
12898
12899 ins_encode %{
12900 __ fsubs(as_FloatRegister($dst$$reg),
12901 as_FloatRegister($src1$$reg),
12902 as_FloatRegister($src2$$reg));
12903 %}
12904
12905 ins_pipe(fp_dop_reg_reg_s);
12906 %}
12907
12908 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12909 match(Set dst (SubD src1 src2));
12910
12911 ins_cost(INSN_COST * 5);
12912 format %{ "fsubd $dst, $src1, $src2" %}
12913
12914 ins_encode %{
12915 __ fsubd(as_FloatRegister($dst$$reg),
12916 as_FloatRegister($src1$$reg),
12917 as_FloatRegister($src2$$reg));
12918 %}
12919
12920 ins_pipe(fp_dop_reg_reg_d);
12921 %}
12922
12923 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12924 match(Set dst (MulHF src1 src2));
12925 format %{ "fmulh $dst, $src1, $src2" %}
12926 ins_encode %{
12927 __ fmulh($dst$$FloatRegister,
12928 $src1$$FloatRegister,
12929 $src2$$FloatRegister);
12930 %}
12931 ins_pipe(fp_dop_reg_reg_s);
12932 %}
12933
12934 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12935 match(Set dst (MulF src1 src2));
12936
12937 ins_cost(INSN_COST * 6);
12938 format %{ "fmuls $dst, $src1, $src2" %}
12939
12940 ins_encode %{
12941 __ fmuls(as_FloatRegister($dst$$reg),
12942 as_FloatRegister($src1$$reg),
12943 as_FloatRegister($src2$$reg));
12944 %}
12945
12946 ins_pipe(fp_dop_reg_reg_s);
12947 %}
12948
12949 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12950 match(Set dst (MulD src1 src2));
12951
12952 ins_cost(INSN_COST * 6);
12953 format %{ "fmuld $dst, $src1, $src2" %}
12954
12955 ins_encode %{
12956 __ fmuld(as_FloatRegister($dst$$reg),
12957 as_FloatRegister($src1$$reg),
12958 as_FloatRegister($src2$$reg));
12959 %}
12960
12961 ins_pipe(fp_dop_reg_reg_d);
12962 %}
12963
12964 // src1 * src2 + src3 (half-precision float)
12965 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12966 match(Set dst (FmaHF src3 (Binary src1 src2)));
12967 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12968 ins_encode %{
12969 assert(UseFMA, "Needs FMA instructions support.");
12970 __ fmaddh($dst$$FloatRegister,
12971 $src1$$FloatRegister,
12972 $src2$$FloatRegister,
12973 $src3$$FloatRegister);
12974 %}
12975 ins_pipe(pipe_class_default);
12976 %}
12977
12978 // src1 * src2 + src3
12979 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12980 match(Set dst (FmaF src3 (Binary src1 src2)));
12981
12982 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12983
12984 ins_encode %{
12985 assert(UseFMA, "Needs FMA instructions support.");
12986 __ fmadds(as_FloatRegister($dst$$reg),
12987 as_FloatRegister($src1$$reg),
12988 as_FloatRegister($src2$$reg),
12989 as_FloatRegister($src3$$reg));
12990 %}
12991
12992 ins_pipe(pipe_class_default);
12993 %}
12994
12995 // src1 * src2 + src3
12996 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12997 match(Set dst (FmaD src3 (Binary src1 src2)));
12998
12999 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13000
13001 ins_encode %{
13002 assert(UseFMA, "Needs FMA instructions support.");
13003 __ fmaddd(as_FloatRegister($dst$$reg),
13004 as_FloatRegister($src1$$reg),
13005 as_FloatRegister($src2$$reg),
13006 as_FloatRegister($src3$$reg));
13007 %}
13008
13009 ins_pipe(pipe_class_default);
13010 %}
13011
13012 // src1 * (-src2) + src3
13013 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13014 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13015 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13016
13017 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13018
13019 ins_encode %{
13020 assert(UseFMA, "Needs FMA instructions support.");
13021 __ fmsubs(as_FloatRegister($dst$$reg),
13022 as_FloatRegister($src1$$reg),
13023 as_FloatRegister($src2$$reg),
13024 as_FloatRegister($src3$$reg));
13025 %}
13026
13027 ins_pipe(pipe_class_default);
13028 %}
13029
13030 // src1 * (-src2) + src3
13031 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13032 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13033 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13034
13035 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13036
13037 ins_encode %{
13038 assert(UseFMA, "Needs FMA instructions support.");
13039 __ fmsubd(as_FloatRegister($dst$$reg),
13040 as_FloatRegister($src1$$reg),
13041 as_FloatRegister($src2$$reg),
13042 as_FloatRegister($src3$$reg));
13043 %}
13044
13045 ins_pipe(pipe_class_default);
13046 %}
13047
13048 // src1 * (-src2) - src3
13049 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13050 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13051 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13052
13053 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13054
13055 ins_encode %{
13056 assert(UseFMA, "Needs FMA instructions support.");
13057 __ fnmadds(as_FloatRegister($dst$$reg),
13058 as_FloatRegister($src1$$reg),
13059 as_FloatRegister($src2$$reg),
13060 as_FloatRegister($src3$$reg));
13061 %}
13062
13063 ins_pipe(pipe_class_default);
13064 %}
13065
13066 // src1 * (-src2) - src3
13067 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13068 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13069 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13070
13071 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13072
13073 ins_encode %{
13074 assert(UseFMA, "Needs FMA instructions support.");
13075 __ fnmaddd(as_FloatRegister($dst$$reg),
13076 as_FloatRegister($src1$$reg),
13077 as_FloatRegister($src2$$reg),
13078 as_FloatRegister($src3$$reg));
13079 %}
13080
13081 ins_pipe(pipe_class_default);
13082 %}
13083
13084 // src1 * src2 - src3
13085 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13086 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13087
13088 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13089
13090 ins_encode %{
13091 assert(UseFMA, "Needs FMA instructions support.");
13092 __ fnmsubs(as_FloatRegister($dst$$reg),
13093 as_FloatRegister($src1$$reg),
13094 as_FloatRegister($src2$$reg),
13095 as_FloatRegister($src3$$reg));
13096 %}
13097
13098 ins_pipe(pipe_class_default);
13099 %}
13100
13101 // src1 * src2 - src3
13102 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13103 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13104
13105 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13106
13107 ins_encode %{
13108 assert(UseFMA, "Needs FMA instructions support.");
13109 // n.b. insn name should be fnmsubd
13110 __ fnmsub(as_FloatRegister($dst$$reg),
13111 as_FloatRegister($src1$$reg),
13112 as_FloatRegister($src2$$reg),
13113 as_FloatRegister($src3$$reg));
13114 %}
13115
13116 ins_pipe(pipe_class_default);
13117 %}
13118
13119 // Math.max(HH)H (half-precision float)
13120 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13121 match(Set dst (MaxHF src1 src2));
13122 format %{ "fmaxh $dst, $src1, $src2" %}
13123 ins_encode %{
13124 __ fmaxh($dst$$FloatRegister,
13125 $src1$$FloatRegister,
13126 $src2$$FloatRegister);
13127 %}
13128 ins_pipe(fp_dop_reg_reg_s);
13129 %}
13130
13131 // Math.min(HH)H (half-precision float)
13132 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13133 match(Set dst (MinHF src1 src2));
13134 format %{ "fminh $dst, $src1, $src2" %}
13135 ins_encode %{
13136 __ fminh($dst$$FloatRegister,
13137 $src1$$FloatRegister,
13138 $src2$$FloatRegister);
13139 %}
13140 ins_pipe(fp_dop_reg_reg_s);
13141 %}
13142
13143 // Math.max(FF)F
13144 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13145 match(Set dst (MaxF src1 src2));
13146
13147 format %{ "fmaxs $dst, $src1, $src2" %}
13148 ins_encode %{
13149 __ fmaxs(as_FloatRegister($dst$$reg),
13150 as_FloatRegister($src1$$reg),
13151 as_FloatRegister($src2$$reg));
13152 %}
13153
13154 ins_pipe(fp_dop_reg_reg_s);
13155 %}
13156
13157 // Math.min(FF)F
13158 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13159 match(Set dst (MinF src1 src2));
13160
13161 format %{ "fmins $dst, $src1, $src2" %}
13162 ins_encode %{
13163 __ fmins(as_FloatRegister($dst$$reg),
13164 as_FloatRegister($src1$$reg),
13165 as_FloatRegister($src2$$reg));
13166 %}
13167
13168 ins_pipe(fp_dop_reg_reg_s);
13169 %}
13170
13171 // Math.max(DD)D
13172 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13173 match(Set dst (MaxD src1 src2));
13174
13175 format %{ "fmaxd $dst, $src1, $src2" %}
13176 ins_encode %{
13177 __ fmaxd(as_FloatRegister($dst$$reg),
13178 as_FloatRegister($src1$$reg),
13179 as_FloatRegister($src2$$reg));
13180 %}
13181
13182 ins_pipe(fp_dop_reg_reg_d);
13183 %}
13184
13185 // Math.min(DD)D
13186 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13187 match(Set dst (MinD src1 src2));
13188
13189 format %{ "fmind $dst, $src1, $src2" %}
13190 ins_encode %{
13191 __ fmind(as_FloatRegister($dst$$reg),
13192 as_FloatRegister($src1$$reg),
13193 as_FloatRegister($src2$$reg));
13194 %}
13195
13196 ins_pipe(fp_dop_reg_reg_d);
13197 %}
13198
13199 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13200 match(Set dst (DivHF src1 src2));
13201 format %{ "fdivh $dst, $src1, $src2" %}
13202 ins_encode %{
13203 __ fdivh($dst$$FloatRegister,
13204 $src1$$FloatRegister,
13205 $src2$$FloatRegister);
13206 %}
13207 ins_pipe(fp_div_s);
13208 %}
13209
13210 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13211 match(Set dst (DivF src1 src2));
13212
13213 ins_cost(INSN_COST * 18);
13214 format %{ "fdivs $dst, $src1, $src2" %}
13215
13216 ins_encode %{
13217 __ fdivs(as_FloatRegister($dst$$reg),
13218 as_FloatRegister($src1$$reg),
13219 as_FloatRegister($src2$$reg));
13220 %}
13221
13222 ins_pipe(fp_div_s);
13223 %}
13224
13225 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13226 match(Set dst (DivD src1 src2));
13227
13228 ins_cost(INSN_COST * 32);
13229 format %{ "fdivd $dst, $src1, $src2" %}
13230
13231 ins_encode %{
13232 __ fdivd(as_FloatRegister($dst$$reg),
13233 as_FloatRegister($src1$$reg),
13234 as_FloatRegister($src2$$reg));
13235 %}
13236
13237 ins_pipe(fp_div_d);
13238 %}
13239
13240 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13241 match(Set dst (NegF src));
13242
13243 ins_cost(INSN_COST * 3);
13244 format %{ "fneg $dst, $src" %}
13245
13246 ins_encode %{
13247 __ fnegs(as_FloatRegister($dst$$reg),
13248 as_FloatRegister($src$$reg));
13249 %}
13250
13251 ins_pipe(fp_uop_s);
13252 %}
13253
13254 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13255 match(Set dst (NegD src));
13256
13257 ins_cost(INSN_COST * 3);
13258 format %{ "fnegd $dst, $src" %}
13259
13260 ins_encode %{
13261 __ fnegd(as_FloatRegister($dst$$reg),
13262 as_FloatRegister($src$$reg));
13263 %}
13264
13265 ins_pipe(fp_uop_d);
13266 %}
13267
13268 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13269 %{
13270 match(Set dst (AbsI src));
13271
13272 effect(KILL cr);
13273 ins_cost(INSN_COST * 2);
13274 format %{ "cmpw $src, zr\n\t"
13275 "cnegw $dst, $src, Assembler::LT\t# int abs"
13276 %}
13277
13278 ins_encode %{
13279 __ cmpw(as_Register($src$$reg), zr);
13280 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13281 %}
13282 ins_pipe(pipe_class_default);
13283 %}
13284
13285 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13286 %{
13287 match(Set dst (AbsL src));
13288
13289 effect(KILL cr);
13290 ins_cost(INSN_COST * 2);
13291 format %{ "cmp $src, zr\n\t"
13292 "cneg $dst, $src, Assembler::LT\t# long abs"
13293 %}
13294
13295 ins_encode %{
13296 __ cmp(as_Register($src$$reg), zr);
13297 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13298 %}
13299 ins_pipe(pipe_class_default);
13300 %}
13301
13302 instruct absF_reg(vRegF dst, vRegF src) %{
13303 match(Set dst (AbsF src));
13304
13305 ins_cost(INSN_COST * 3);
13306 format %{ "fabss $dst, $src" %}
13307 ins_encode %{
13308 __ fabss(as_FloatRegister($dst$$reg),
13309 as_FloatRegister($src$$reg));
13310 %}
13311
13312 ins_pipe(fp_uop_s);
13313 %}
13314
13315 instruct absD_reg(vRegD dst, vRegD src) %{
13316 match(Set dst (AbsD src));
13317
13318 ins_cost(INSN_COST * 3);
13319 format %{ "fabsd $dst, $src" %}
13320 ins_encode %{
13321 __ fabsd(as_FloatRegister($dst$$reg),
13322 as_FloatRegister($src$$reg));
13323 %}
13324
13325 ins_pipe(fp_uop_d);
13326 %}
13327
13328 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13329 match(Set dst (AbsF (SubF src1 src2)));
13330
13331 ins_cost(INSN_COST * 3);
13332 format %{ "fabds $dst, $src1, $src2" %}
13333 ins_encode %{
13334 __ fabds(as_FloatRegister($dst$$reg),
13335 as_FloatRegister($src1$$reg),
13336 as_FloatRegister($src2$$reg));
13337 %}
13338
13339 ins_pipe(fp_uop_s);
13340 %}
13341
13342 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13343 match(Set dst (AbsD (SubD src1 src2)));
13344
13345 ins_cost(INSN_COST * 3);
13346 format %{ "fabdd $dst, $src1, $src2" %}
13347 ins_encode %{
13348 __ fabdd(as_FloatRegister($dst$$reg),
13349 as_FloatRegister($src1$$reg),
13350 as_FloatRegister($src2$$reg));
13351 %}
13352
13353 ins_pipe(fp_uop_d);
13354 %}
13355
13356 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13357 match(Set dst (SqrtD src));
13358
13359 ins_cost(INSN_COST * 50);
13360 format %{ "fsqrtd $dst, $src" %}
13361 ins_encode %{
13362 __ fsqrtd(as_FloatRegister($dst$$reg),
13363 as_FloatRegister($src$$reg));
13364 %}
13365
13366 ins_pipe(fp_div_s);
13367 %}
13368
13369 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13370 match(Set dst (SqrtF src));
13371
13372 ins_cost(INSN_COST * 50);
13373 format %{ "fsqrts $dst, $src" %}
13374 ins_encode %{
13375 __ fsqrts(as_FloatRegister($dst$$reg),
13376 as_FloatRegister($src$$reg));
13377 %}
13378
13379 ins_pipe(fp_div_d);
13380 %}
13381
13382 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13383 match(Set dst (SqrtHF src));
13384 format %{ "fsqrth $dst, $src" %}
13385 ins_encode %{
13386 __ fsqrth($dst$$FloatRegister,
13387 $src$$FloatRegister);
13388 %}
13389 ins_pipe(fp_div_s);
13390 %}
13391
13392 // Math.rint, floor, ceil
13393 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13394 match(Set dst (RoundDoubleMode src rmode));
13395 format %{ "frint $dst, $src, $rmode" %}
13396 ins_encode %{
13397 switch ($rmode$$constant) {
13398 case RoundDoubleModeNode::rmode_rint:
13399 __ frintnd(as_FloatRegister($dst$$reg),
13400 as_FloatRegister($src$$reg));
13401 break;
13402 case RoundDoubleModeNode::rmode_floor:
13403 __ frintmd(as_FloatRegister($dst$$reg),
13404 as_FloatRegister($src$$reg));
13405 break;
13406 case RoundDoubleModeNode::rmode_ceil:
13407 __ frintpd(as_FloatRegister($dst$$reg),
13408 as_FloatRegister($src$$reg));
13409 break;
13410 }
13411 %}
13412 ins_pipe(fp_uop_d);
13413 %}
13414
13415 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13416 match(Set dst (CopySignD src1 (Binary src2 zero)));
13417 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13418 format %{ "CopySignD $dst $src1 $src2" %}
13419 ins_encode %{
13420 FloatRegister dst = as_FloatRegister($dst$$reg),
13421 src1 = as_FloatRegister($src1$$reg),
13422 src2 = as_FloatRegister($src2$$reg),
13423 zero = as_FloatRegister($zero$$reg);
13424 __ fnegd(dst, zero);
13425 __ bsl(dst, __ T8B, src2, src1);
13426 %}
13427 ins_pipe(fp_uop_d);
13428 %}
13429
13430 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13431 match(Set dst (CopySignF src1 src2));
13432 effect(TEMP_DEF dst, USE src1, USE src2);
13433 format %{ "CopySignF $dst $src1 $src2" %}
13434 ins_encode %{
13435 FloatRegister dst = as_FloatRegister($dst$$reg),
13436 src1 = as_FloatRegister($src1$$reg),
13437 src2 = as_FloatRegister($src2$$reg);
13438 __ movi(dst, __ T2S, 0x80, 24);
13439 __ bsl(dst, __ T8B, src2, src1);
13440 %}
13441 ins_pipe(fp_uop_d);
13442 %}
13443
13444 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13445 match(Set dst (SignumD src (Binary zero one)));
13446 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13447 format %{ "signumD $dst, $src" %}
13448 ins_encode %{
13449 FloatRegister src = as_FloatRegister($src$$reg),
13450 dst = as_FloatRegister($dst$$reg),
13451 zero = as_FloatRegister($zero$$reg),
13452 one = as_FloatRegister($one$$reg);
13453 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13454 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13455 // Bit selection instruction gets bit from "one" for each enabled bit in
13456 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13457 // NaN the whole "src" will be copied because "dst" is zero. For all other
13458 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13459 // from "src", and all other bits are copied from 1.0.
13460 __ bsl(dst, __ T8B, one, src);
13461 %}
13462 ins_pipe(fp_uop_d);
13463 %}
13464
13465 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13466 match(Set dst (SignumF src (Binary zero one)));
13467 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13468 format %{ "signumF $dst, $src" %}
13469 ins_encode %{
13470 FloatRegister src = as_FloatRegister($src$$reg),
13471 dst = as_FloatRegister($dst$$reg),
13472 zero = as_FloatRegister($zero$$reg),
13473 one = as_FloatRegister($one$$reg);
13474 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13475 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13476 // Bit selection instruction gets bit from "one" for each enabled bit in
13477 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13478 // NaN the whole "src" will be copied because "dst" is zero. For all other
13479 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13480 // from "src", and all other bits are copied from 1.0.
13481 __ bsl(dst, __ T8B, one, src);
13482 %}
13483 ins_pipe(fp_uop_d);
13484 %}
13485
13486 instruct onspinwait() %{
13487 match(OnSpinWait);
13488 ins_cost(INSN_COST);
13489
13490 format %{ "onspinwait" %}
13491
13492 ins_encode %{
13493 __ spin_wait();
13494 %}
13495 ins_pipe(pipe_class_empty);
13496 %}
13497
13498 // ============================================================================
13499 // Logical Instructions
13500
13501 // Integer Logical Instructions
13502
13503 // And Instructions
13504
13505
13506 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13507 match(Set dst (AndI src1 src2));
13508
13509 format %{ "andw $dst, $src1, $src2\t# int" %}
13510
13511 ins_cost(INSN_COST);
13512 ins_encode %{
13513 __ andw(as_Register($dst$$reg),
13514 as_Register($src1$$reg),
13515 as_Register($src2$$reg));
13516 %}
13517
13518 ins_pipe(ialu_reg_reg);
13519 %}
13520
13521 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13522 match(Set dst (AndI src1 src2));
13523
13524 format %{ "andsw $dst, $src1, $src2\t# int" %}
13525
13526 ins_cost(INSN_COST);
13527 ins_encode %{
13528 __ andw(as_Register($dst$$reg),
13529 as_Register($src1$$reg),
13530 (uint64_t)($src2$$constant));
13531 %}
13532
13533 ins_pipe(ialu_reg_imm);
13534 %}
13535
13536 // Or Instructions
13537
13538 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13539 match(Set dst (OrI src1 src2));
13540
13541 format %{ "orrw $dst, $src1, $src2\t# int" %}
13542
13543 ins_cost(INSN_COST);
13544 ins_encode %{
13545 __ orrw(as_Register($dst$$reg),
13546 as_Register($src1$$reg),
13547 as_Register($src2$$reg));
13548 %}
13549
13550 ins_pipe(ialu_reg_reg);
13551 %}
13552
13553 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13554 match(Set dst (OrI src1 src2));
13555
13556 format %{ "orrw $dst, $src1, $src2\t# int" %}
13557
13558 ins_cost(INSN_COST);
13559 ins_encode %{
13560 __ orrw(as_Register($dst$$reg),
13561 as_Register($src1$$reg),
13562 (uint64_t)($src2$$constant));
13563 %}
13564
13565 ins_pipe(ialu_reg_imm);
13566 %}
13567
13568 // Xor Instructions
13569
13570 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13571 match(Set dst (XorI src1 src2));
13572
13573 format %{ "eorw $dst, $src1, $src2\t# int" %}
13574
13575 ins_cost(INSN_COST);
13576 ins_encode %{
13577 __ eorw(as_Register($dst$$reg),
13578 as_Register($src1$$reg),
13579 as_Register($src2$$reg));
13580 %}
13581
13582 ins_pipe(ialu_reg_reg);
13583 %}
13584
13585 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13586 match(Set dst (XorI src1 src2));
13587
13588 format %{ "eorw $dst, $src1, $src2\t# int" %}
13589
13590 ins_cost(INSN_COST);
13591 ins_encode %{
13592 __ eorw(as_Register($dst$$reg),
13593 as_Register($src1$$reg),
13594 (uint64_t)($src2$$constant));
13595 %}
13596
13597 ins_pipe(ialu_reg_imm);
13598 %}
13599
13600 // Long Logical Instructions
13601 // TODO
13602
13603 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13604 match(Set dst (AndL src1 src2));
13605
13606 format %{ "and $dst, $src1, $src2\t# int" %}
13607
13608 ins_cost(INSN_COST);
13609 ins_encode %{
13610 __ andr(as_Register($dst$$reg),
13611 as_Register($src1$$reg),
13612 as_Register($src2$$reg));
13613 %}
13614
13615 ins_pipe(ialu_reg_reg);
13616 %}
13617
13618 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13619 match(Set dst (AndL src1 src2));
13620
13621 format %{ "and $dst, $src1, $src2\t# int" %}
13622
13623 ins_cost(INSN_COST);
13624 ins_encode %{
13625 __ andr(as_Register($dst$$reg),
13626 as_Register($src1$$reg),
13627 (uint64_t)($src2$$constant));
13628 %}
13629
13630 ins_pipe(ialu_reg_imm);
13631 %}
13632
13633 // Or Instructions
13634
13635 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13636 match(Set dst (OrL src1 src2));
13637
13638 format %{ "orr $dst, $src1, $src2\t# int" %}
13639
13640 ins_cost(INSN_COST);
13641 ins_encode %{
13642 __ orr(as_Register($dst$$reg),
13643 as_Register($src1$$reg),
13644 as_Register($src2$$reg));
13645 %}
13646
13647 ins_pipe(ialu_reg_reg);
13648 %}
13649
13650 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13651 match(Set dst (OrL src1 src2));
13652
13653 format %{ "orr $dst, $src1, $src2\t# int" %}
13654
13655 ins_cost(INSN_COST);
13656 ins_encode %{
13657 __ orr(as_Register($dst$$reg),
13658 as_Register($src1$$reg),
13659 (uint64_t)($src2$$constant));
13660 %}
13661
13662 ins_pipe(ialu_reg_imm);
13663 %}
13664
13665 // Xor Instructions
13666
13667 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13668 match(Set dst (XorL src1 src2));
13669
13670 format %{ "eor $dst, $src1, $src2\t# int" %}
13671
13672 ins_cost(INSN_COST);
13673 ins_encode %{
13674 __ eor(as_Register($dst$$reg),
13675 as_Register($src1$$reg),
13676 as_Register($src2$$reg));
13677 %}
13678
13679 ins_pipe(ialu_reg_reg);
13680 %}
13681
13682 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13683 match(Set dst (XorL src1 src2));
13684
13685 ins_cost(INSN_COST);
13686 format %{ "eor $dst, $src1, $src2\t# int" %}
13687
13688 ins_encode %{
13689 __ eor(as_Register($dst$$reg),
13690 as_Register($src1$$reg),
13691 (uint64_t)($src2$$constant));
13692 %}
13693
13694 ins_pipe(ialu_reg_imm);
13695 %}
13696
13697 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13698 %{
13699 match(Set dst (ConvI2L src));
13700
13701 ins_cost(INSN_COST);
13702 format %{ "sxtw $dst, $src\t# i2l" %}
13703 ins_encode %{
13704 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13705 %}
13706 ins_pipe(ialu_reg_shift);
13707 %}
13708
13709 // this pattern occurs in bigmath arithmetic
13710 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13711 %{
13712 match(Set dst (AndL (ConvI2L src) mask));
13713
13714 ins_cost(INSN_COST);
13715 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13716 ins_encode %{
13717 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13718 %}
13719
13720 ins_pipe(ialu_reg_shift);
13721 %}
13722
13723 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13724 match(Set dst (ConvL2I src));
13725
13726 ins_cost(INSN_COST);
13727 format %{ "movw $dst, $src \t// l2i" %}
13728
13729 ins_encode %{
13730 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13731 %}
13732
13733 ins_pipe(ialu_reg);
13734 %}
13735
13736 instruct convD2F_reg(vRegF dst, vRegD src) %{
13737 match(Set dst (ConvD2F src));
13738
13739 ins_cost(INSN_COST * 5);
13740 format %{ "fcvtd $dst, $src \t// d2f" %}
13741
13742 ins_encode %{
13743 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13744 %}
13745
13746 ins_pipe(fp_d2f);
13747 %}
13748
13749 instruct convF2D_reg(vRegD dst, vRegF src) %{
13750 match(Set dst (ConvF2D src));
13751
13752 ins_cost(INSN_COST * 5);
13753 format %{ "fcvts $dst, $src \t// f2d" %}
13754
13755 ins_encode %{
13756 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13757 %}
13758
13759 ins_pipe(fp_f2d);
13760 %}
13761
13762 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13763 match(Set dst (ConvF2I src));
13764
13765 ins_cost(INSN_COST * 5);
13766 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13767
13768 ins_encode %{
13769 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13770 %}
13771
13772 ins_pipe(fp_f2i);
13773 %}
13774
13775 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13776 match(Set dst (ConvF2L src));
13777
13778 ins_cost(INSN_COST * 5);
13779 format %{ "fcvtzs $dst, $src \t// f2l" %}
13780
13781 ins_encode %{
13782 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13783 %}
13784
13785 ins_pipe(fp_f2l);
13786 %}
13787
13788 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13789 match(Set dst (ConvF2HF src));
13790 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13791 "smov $dst, $tmp\t# move result from $tmp to $dst"
13792 %}
13793 effect(TEMP tmp);
13794 ins_encode %{
13795 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13796 %}
13797 ins_pipe(pipe_slow);
13798 %}
13799
13800 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13801 match(Set dst (ConvHF2F src));
13802 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13803 "fcvt $dst, $tmp\t# convert half to single precision"
13804 %}
13805 effect(TEMP tmp);
13806 ins_encode %{
13807 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13808 %}
13809 ins_pipe(pipe_slow);
13810 %}
13811
13812 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13813 match(Set dst (ConvI2F src));
13814
13815 ins_cost(INSN_COST * 5);
13816 format %{ "scvtfws $dst, $src \t// i2f" %}
13817
13818 ins_encode %{
13819 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13820 %}
13821
13822 ins_pipe(fp_i2f);
13823 %}
13824
13825 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13826 match(Set dst (ConvL2F src));
13827
13828 ins_cost(INSN_COST * 5);
13829 format %{ "scvtfs $dst, $src \t// l2f" %}
13830
13831 ins_encode %{
13832 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13833 %}
13834
13835 ins_pipe(fp_l2f);
13836 %}
13837
13838 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13839 match(Set dst (ConvD2I src));
13840
13841 ins_cost(INSN_COST * 5);
13842 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13843
13844 ins_encode %{
13845 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13846 %}
13847
13848 ins_pipe(fp_d2i);
13849 %}
13850
13851 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13852 match(Set dst (ConvD2L src));
13853
13854 ins_cost(INSN_COST * 5);
13855 format %{ "fcvtzd $dst, $src \t// d2l" %}
13856
13857 ins_encode %{
13858 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13859 %}
13860
13861 ins_pipe(fp_d2l);
13862 %}
13863
13864 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13865 match(Set dst (ConvI2D src));
13866
13867 ins_cost(INSN_COST * 5);
13868 format %{ "scvtfwd $dst, $src \t// i2d" %}
13869
13870 ins_encode %{
13871 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13872 %}
13873
13874 ins_pipe(fp_i2d);
13875 %}
13876
13877 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13878 match(Set dst (ConvL2D src));
13879
13880 ins_cost(INSN_COST * 5);
13881 format %{ "scvtfd $dst, $src \t// l2d" %}
13882
13883 ins_encode %{
13884 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13885 %}
13886
13887 ins_pipe(fp_l2d);
13888 %}
13889
13890 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13891 %{
13892 match(Set dst (RoundD src));
13893 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13894 format %{ "java_round_double $dst,$src"%}
13895 ins_encode %{
13896 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13897 as_FloatRegister($ftmp$$reg));
13898 %}
13899 ins_pipe(pipe_slow);
13900 %}
13901
13902 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13903 %{
13904 match(Set dst (RoundF src));
13905 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13906 format %{ "java_round_float $dst,$src"%}
13907 ins_encode %{
13908 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13909 as_FloatRegister($ftmp$$reg));
13910 %}
13911 ins_pipe(pipe_slow);
13912 %}
13913
13914 // stack <-> reg and reg <-> reg shuffles with no conversion
13915
13916 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13917
13918 match(Set dst (MoveF2I src));
13919
13920 effect(DEF dst, USE src);
13921
13922 ins_cost(4 * INSN_COST);
13923
13924 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13925
13926 ins_encode %{
13927 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13928 %}
13929
13930 ins_pipe(iload_reg_reg);
13931
13932 %}
13933
13934 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13935
13936 match(Set dst (MoveI2F src));
13937
13938 effect(DEF dst, USE src);
13939
13940 ins_cost(4 * INSN_COST);
13941
13942 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13943
13944 ins_encode %{
13945 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13946 %}
13947
13948 ins_pipe(pipe_class_memory);
13949
13950 %}
13951
13952 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13953
13954 match(Set dst (MoveD2L src));
13955
13956 effect(DEF dst, USE src);
13957
13958 ins_cost(4 * INSN_COST);
13959
13960 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13961
13962 ins_encode %{
13963 __ ldr($dst$$Register, Address(sp, $src$$disp));
13964 %}
13965
13966 ins_pipe(iload_reg_reg);
13967
13968 %}
13969
13970 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13971
13972 match(Set dst (MoveL2D src));
13973
13974 effect(DEF dst, USE src);
13975
13976 ins_cost(4 * INSN_COST);
13977
13978 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13979
13980 ins_encode %{
13981 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13982 %}
13983
13984 ins_pipe(pipe_class_memory);
13985
13986 %}
13987
13988 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13989
13990 match(Set dst (MoveF2I src));
13991
13992 effect(DEF dst, USE src);
13993
13994 ins_cost(INSN_COST);
13995
13996 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13997
13998 ins_encode %{
13999 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14000 %}
14001
14002 ins_pipe(pipe_class_memory);
14003
14004 %}
14005
14006 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14007
14008 match(Set dst (MoveI2F src));
14009
14010 effect(DEF dst, USE src);
14011
14012 ins_cost(INSN_COST);
14013
14014 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14015
14016 ins_encode %{
14017 __ strw($src$$Register, Address(sp, $dst$$disp));
14018 %}
14019
14020 ins_pipe(istore_reg_reg);
14021
14022 %}
14023
14024 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14025
14026 match(Set dst (MoveD2L src));
14027
14028 effect(DEF dst, USE src);
14029
14030 ins_cost(INSN_COST);
14031
14032 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14033
14034 ins_encode %{
14035 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14036 %}
14037
14038 ins_pipe(pipe_class_memory);
14039
14040 %}
14041
14042 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14043
14044 match(Set dst (MoveL2D src));
14045
14046 effect(DEF dst, USE src);
14047
14048 ins_cost(INSN_COST);
14049
14050 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14051
14052 ins_encode %{
14053 __ str($src$$Register, Address(sp, $dst$$disp));
14054 %}
14055
14056 ins_pipe(istore_reg_reg);
14057
14058 %}
14059
14060 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14061
14062 match(Set dst (MoveF2I src));
14063
14064 effect(DEF dst, USE src);
14065
14066 ins_cost(INSN_COST);
14067
14068 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14069
14070 ins_encode %{
14071 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14072 %}
14073
14074 ins_pipe(fp_f2i);
14075
14076 %}
14077
14078 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14079
14080 match(Set dst (MoveI2F src));
14081
14082 effect(DEF dst, USE src);
14083
14084 ins_cost(INSN_COST);
14085
14086 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14087
14088 ins_encode %{
14089 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14090 %}
14091
14092 ins_pipe(fp_i2f);
14093
14094 %}
14095
14096 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14097
14098 match(Set dst (MoveD2L src));
14099
14100 effect(DEF dst, USE src);
14101
14102 ins_cost(INSN_COST);
14103
14104 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14105
14106 ins_encode %{
14107 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14108 %}
14109
14110 ins_pipe(fp_d2l);
14111
14112 %}
14113
14114 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14115
14116 match(Set dst (MoveL2D src));
14117
14118 effect(DEF dst, USE src);
14119
14120 ins_cost(INSN_COST);
14121
14122 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14123
14124 ins_encode %{
14125 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14126 %}
14127
14128 ins_pipe(fp_l2d);
14129
14130 %}
14131
14132 // ============================================================================
14133 // clearing of an array
14134
14135 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14136 %{
14137 match(Set dummy (ClearArray cnt base));
14138 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14139
14140 ins_cost(4 * INSN_COST);
14141 format %{ "ClearArray $cnt, $base" %}
14142
14143 ins_encode %{
14144 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14145 if (tpc == nullptr) {
14146 ciEnv::current()->record_failure("CodeCache is full");
14147 return;
14148 }
14149 %}
14150
14151 ins_pipe(pipe_class_memory);
14152 %}
14153
14154 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14155 %{
14156 predicate((uint64_t)n->in(2)->get_long()
14157 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14158 match(Set dummy (ClearArray cnt base));
14159 effect(TEMP temp, USE_KILL base, KILL cr);
14160
14161 ins_cost(4 * INSN_COST);
14162 format %{ "ClearArray $cnt, $base" %}
14163
14164 ins_encode %{
14165 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14166 if (tpc == nullptr) {
14167 ciEnv::current()->record_failure("CodeCache is full");
14168 return;
14169 }
14170 %}
14171
14172 ins_pipe(pipe_class_memory);
14173 %}
14174
14175 // ============================================================================
14176 // Overflow Math Instructions
14177
14178 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14179 %{
14180 match(Set cr (OverflowAddI op1 op2));
14181
14182 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14183 ins_cost(INSN_COST);
14184 ins_encode %{
14185 __ cmnw($op1$$Register, $op2$$Register);
14186 %}
14187
14188 ins_pipe(icmp_reg_reg);
14189 %}
14190
14191 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14192 %{
14193 match(Set cr (OverflowAddI op1 op2));
14194
14195 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14196 ins_cost(INSN_COST);
14197 ins_encode %{
14198 __ cmnw($op1$$Register, $op2$$constant);
14199 %}
14200
14201 ins_pipe(icmp_reg_imm);
14202 %}
14203
14204 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14205 %{
14206 match(Set cr (OverflowAddL op1 op2));
14207
14208 format %{ "cmn $op1, $op2\t# overflow check long" %}
14209 ins_cost(INSN_COST);
14210 ins_encode %{
14211 __ cmn($op1$$Register, $op2$$Register);
14212 %}
14213
14214 ins_pipe(icmp_reg_reg);
14215 %}
14216
14217 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14218 %{
14219 match(Set cr (OverflowAddL op1 op2));
14220
14221 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14222 ins_cost(INSN_COST);
14223 ins_encode %{
14224 __ adds(zr, $op1$$Register, $op2$$constant);
14225 %}
14226
14227 ins_pipe(icmp_reg_imm);
14228 %}
14229
14230 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14231 %{
14232 match(Set cr (OverflowSubI op1 op2));
14233
14234 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14235 ins_cost(INSN_COST);
14236 ins_encode %{
14237 __ cmpw($op1$$Register, $op2$$Register);
14238 %}
14239
14240 ins_pipe(icmp_reg_reg);
14241 %}
14242
14243 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14244 %{
14245 match(Set cr (OverflowSubI op1 op2));
14246
14247 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14248 ins_cost(INSN_COST);
14249 ins_encode %{
14250 __ cmpw($op1$$Register, $op2$$constant);
14251 %}
14252
14253 ins_pipe(icmp_reg_imm);
14254 %}
14255
14256 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14257 %{
14258 match(Set cr (OverflowSubL op1 op2));
14259
14260 format %{ "cmp $op1, $op2\t# overflow check long" %}
14261 ins_cost(INSN_COST);
14262 ins_encode %{
14263 __ cmp($op1$$Register, $op2$$Register);
14264 %}
14265
14266 ins_pipe(icmp_reg_reg);
14267 %}
14268
14269 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14270 %{
14271 match(Set cr (OverflowSubL op1 op2));
14272
14273 format %{ "cmp $op1, $op2\t# overflow check long" %}
14274 ins_cost(INSN_COST);
14275 ins_encode %{
14276 __ subs(zr, $op1$$Register, $op2$$constant);
14277 %}
14278
14279 ins_pipe(icmp_reg_imm);
14280 %}
14281
14282 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14283 %{
14284 match(Set cr (OverflowSubI zero op1));
14285
14286 format %{ "cmpw zr, $op1\t# overflow check int" %}
14287 ins_cost(INSN_COST);
14288 ins_encode %{
14289 __ cmpw(zr, $op1$$Register);
14290 %}
14291
14292 ins_pipe(icmp_reg_imm);
14293 %}
14294
14295 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14296 %{
14297 match(Set cr (OverflowSubL zero op1));
14298
14299 format %{ "cmp zr, $op1\t# overflow check long" %}
14300 ins_cost(INSN_COST);
14301 ins_encode %{
14302 __ cmp(zr, $op1$$Register);
14303 %}
14304
14305 ins_pipe(icmp_reg_imm);
14306 %}
14307
14308 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14309 %{
14310 match(Set cr (OverflowMulI op1 op2));
14311
14312 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14313 "cmp rscratch1, rscratch1, sxtw\n\t"
14314 "movw rscratch1, #0x80000000\n\t"
14315 "cselw rscratch1, rscratch1, zr, NE\n\t"
14316 "cmpw rscratch1, #1" %}
14317 ins_cost(5 * INSN_COST);
14318 ins_encode %{
14319 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14320 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14321 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14322 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14323 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14324 %}
14325
14326 ins_pipe(pipe_slow);
14327 %}
14328
14329 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14330 %{
14331 match(If cmp (OverflowMulI op1 op2));
14332 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14333 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14334 effect(USE labl, KILL cr);
14335
14336 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14337 "cmp rscratch1, rscratch1, sxtw\n\t"
14338 "b$cmp $labl" %}
14339 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14340 ins_encode %{
14341 Label* L = $labl$$label;
14342 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14343 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14344 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14345 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14346 %}
14347
14348 ins_pipe(pipe_serial);
14349 %}
14350
14351 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14352 %{
14353 match(Set cr (OverflowMulL op1 op2));
14354
14355 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14356 "smulh rscratch2, $op1, $op2\n\t"
14357 "cmp rscratch2, rscratch1, ASR #63\n\t"
14358 "movw rscratch1, #0x80000000\n\t"
14359 "cselw rscratch1, rscratch1, zr, NE\n\t"
14360 "cmpw rscratch1, #1" %}
14361 ins_cost(6 * INSN_COST);
14362 ins_encode %{
14363 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14364 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14365 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14366 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14367 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14368 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14369 %}
14370
14371 ins_pipe(pipe_slow);
14372 %}
14373
14374 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14375 %{
14376 match(If cmp (OverflowMulL op1 op2));
14377 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14378 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14379 effect(USE labl, KILL cr);
14380
14381 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14382 "smulh rscratch2, $op1, $op2\n\t"
14383 "cmp rscratch2, rscratch1, ASR #63\n\t"
14384 "b$cmp $labl" %}
14385 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14386 ins_encode %{
14387 Label* L = $labl$$label;
14388 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14389 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14390 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14391 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14392 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14393 %}
14394
14395 ins_pipe(pipe_serial);
14396 %}
14397
14398 // ============================================================================
14399 // Compare Instructions
14400
14401 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14402 %{
14403 match(Set cr (CmpI op1 op2));
14404
14405 effect(DEF cr, USE op1, USE op2);
14406
14407 ins_cost(INSN_COST);
14408 format %{ "cmpw $op1, $op2" %}
14409
14410 ins_encode(aarch64_enc_cmpw(op1, op2));
14411
14412 ins_pipe(icmp_reg_reg);
14413 %}
14414
14415 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14416 %{
14417 match(Set cr (CmpI op1 zero));
14418
14419 effect(DEF cr, USE op1);
14420
14421 ins_cost(INSN_COST);
14422 format %{ "cmpw $op1, 0" %}
14423
14424 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14425
14426 ins_pipe(icmp_reg_imm);
14427 %}
14428
14429 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14430 %{
14431 match(Set cr (CmpI op1 op2));
14432
14433 effect(DEF cr, USE op1);
14434
14435 ins_cost(INSN_COST);
14436 format %{ "cmpw $op1, $op2" %}
14437
14438 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14439
14440 ins_pipe(icmp_reg_imm);
14441 %}
14442
14443 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14444 %{
14445 match(Set cr (CmpI op1 op2));
14446
14447 effect(DEF cr, USE op1);
14448
14449 ins_cost(INSN_COST * 2);
14450 format %{ "cmpw $op1, $op2" %}
14451
14452 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14453
14454 ins_pipe(icmp_reg_imm);
14455 %}
14456
14457 // Unsigned compare Instructions; really, same as signed compare
14458 // except it should only be used to feed an If or a CMovI which takes a
14459 // cmpOpU.
14460
14461 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14462 %{
14463 match(Set cr (CmpU op1 op2));
14464
14465 effect(DEF cr, USE op1, USE op2);
14466
14467 ins_cost(INSN_COST);
14468 format %{ "cmpw $op1, $op2\t# unsigned" %}
14469
14470 ins_encode(aarch64_enc_cmpw(op1, op2));
14471
14472 ins_pipe(icmp_reg_reg);
14473 %}
14474
14475 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14476 %{
14477 match(Set cr (CmpU op1 zero));
14478
14479 effect(DEF cr, USE op1);
14480
14481 ins_cost(INSN_COST);
14482 format %{ "cmpw $op1, #0\t# unsigned" %}
14483
14484 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14485
14486 ins_pipe(icmp_reg_imm);
14487 %}
14488
14489 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14490 %{
14491 match(Set cr (CmpU op1 op2));
14492
14493 effect(DEF cr, USE op1);
14494
14495 ins_cost(INSN_COST);
14496 format %{ "cmpw $op1, $op2\t# unsigned" %}
14497
14498 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14499
14500 ins_pipe(icmp_reg_imm);
14501 %}
14502
14503 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14504 %{
14505 match(Set cr (CmpU op1 op2));
14506
14507 effect(DEF cr, USE op1);
14508
14509 ins_cost(INSN_COST * 2);
14510 format %{ "cmpw $op1, $op2\t# unsigned" %}
14511
14512 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14513
14514 ins_pipe(icmp_reg_imm);
14515 %}
14516
14517 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14518 %{
14519 match(Set cr (CmpL op1 op2));
14520
14521 effect(DEF cr, USE op1, USE op2);
14522
14523 ins_cost(INSN_COST);
14524 format %{ "cmp $op1, $op2" %}
14525
14526 ins_encode(aarch64_enc_cmp(op1, op2));
14527
14528 ins_pipe(icmp_reg_reg);
14529 %}
14530
14531 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14532 %{
14533 match(Set cr (CmpL op1 zero));
14534
14535 effect(DEF cr, USE op1);
14536
14537 ins_cost(INSN_COST);
14538 format %{ "tst $op1" %}
14539
14540 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14541
14542 ins_pipe(icmp_reg_imm);
14543 %}
14544
14545 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14546 %{
14547 match(Set cr (CmpL op1 op2));
14548
14549 effect(DEF cr, USE op1);
14550
14551 ins_cost(INSN_COST);
14552 format %{ "cmp $op1, $op2" %}
14553
14554 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14555
14556 ins_pipe(icmp_reg_imm);
14557 %}
14558
14559 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14560 %{
14561 match(Set cr (CmpL op1 op2));
14562
14563 effect(DEF cr, USE op1);
14564
14565 ins_cost(INSN_COST * 2);
14566 format %{ "cmp $op1, $op2" %}
14567
14568 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14569
14570 ins_pipe(icmp_reg_imm);
14571 %}
14572
14573 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14574 %{
14575 match(Set cr (CmpUL op1 op2));
14576
14577 effect(DEF cr, USE op1, USE op2);
14578
14579 ins_cost(INSN_COST);
14580 format %{ "cmp $op1, $op2" %}
14581
14582 ins_encode(aarch64_enc_cmp(op1, op2));
14583
14584 ins_pipe(icmp_reg_reg);
14585 %}
14586
14587 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14588 %{
14589 match(Set cr (CmpUL op1 zero));
14590
14591 effect(DEF cr, USE op1);
14592
14593 ins_cost(INSN_COST);
14594 format %{ "tst $op1" %}
14595
14596 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14597
14598 ins_pipe(icmp_reg_imm);
14599 %}
14600
14601 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14602 %{
14603 match(Set cr (CmpUL op1 op2));
14604
14605 effect(DEF cr, USE op1);
14606
14607 ins_cost(INSN_COST);
14608 format %{ "cmp $op1, $op2" %}
14609
14610 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14611
14612 ins_pipe(icmp_reg_imm);
14613 %}
14614
14615 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14616 %{
14617 match(Set cr (CmpUL op1 op2));
14618
14619 effect(DEF cr, USE op1);
14620
14621 ins_cost(INSN_COST * 2);
14622 format %{ "cmp $op1, $op2" %}
14623
14624 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14625
14626 ins_pipe(icmp_reg_imm);
14627 %}
14628
14629 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14630 %{
14631 match(Set cr (CmpP op1 op2));
14632
14633 effect(DEF cr, USE op1, USE op2);
14634
14635 ins_cost(INSN_COST);
14636 format %{ "cmp $op1, $op2\t // ptr" %}
14637
14638 ins_encode(aarch64_enc_cmpp(op1, op2));
14639
14640 ins_pipe(icmp_reg_reg);
14641 %}
14642
14643 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14644 %{
14645 match(Set cr (CmpN op1 op2));
14646
14647 effect(DEF cr, USE op1, USE op2);
14648
14649 ins_cost(INSN_COST);
14650 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14651
14652 ins_encode(aarch64_enc_cmpn(op1, op2));
14653
14654 ins_pipe(icmp_reg_reg);
14655 %}
14656
14657 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14658 %{
14659 match(Set cr (CmpP op1 zero));
14660
14661 effect(DEF cr, USE op1, USE zero);
14662
14663 ins_cost(INSN_COST);
14664 format %{ "cmp $op1, 0\t // ptr" %}
14665
14666 ins_encode(aarch64_enc_testp(op1));
14667
14668 ins_pipe(icmp_reg_imm);
14669 %}
14670
14671 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14672 %{
14673 match(Set cr (CmpN op1 zero));
14674
14675 effect(DEF cr, USE op1, USE zero);
14676
14677 ins_cost(INSN_COST);
14678 format %{ "cmp $op1, 0\t // compressed ptr" %}
14679
14680 ins_encode(aarch64_enc_testn(op1));
14681
14682 ins_pipe(icmp_reg_imm);
14683 %}
14684
14685 // FP comparisons
14686 //
14687 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14688 // using normal cmpOp. See declaration of rFlagsReg for details.
14689
14690 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14691 %{
14692 match(Set cr (CmpF src1 src2));
14693
14694 ins_cost(3 * INSN_COST);
14695 format %{ "fcmps $src1, $src2" %}
14696
14697 ins_encode %{
14698 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14699 %}
14700
14701 ins_pipe(pipe_class_compare);
14702 %}
14703
14704 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14705 %{
14706 match(Set cr (CmpF src1 src2));
14707
14708 ins_cost(3 * INSN_COST);
14709 format %{ "fcmps $src1, 0.0" %}
14710
14711 ins_encode %{
14712 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14713 %}
14714
14715 ins_pipe(pipe_class_compare);
14716 %}
14717 // FROM HERE
14718
14719 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14720 %{
14721 match(Set cr (CmpD src1 src2));
14722
14723 ins_cost(3 * INSN_COST);
14724 format %{ "fcmpd $src1, $src2" %}
14725
14726 ins_encode %{
14727 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14728 %}
14729
14730 ins_pipe(pipe_class_compare);
14731 %}
14732
14733 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14734 %{
14735 match(Set cr (CmpD src1 src2));
14736
14737 ins_cost(3 * INSN_COST);
14738 format %{ "fcmpd $src1, 0.0" %}
14739
14740 ins_encode %{
14741 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14742 %}
14743
14744 ins_pipe(pipe_class_compare);
14745 %}
14746
14747 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14748 %{
14749 match(Set dst (CmpF3 src1 src2));
14750 effect(KILL cr);
14751
14752 ins_cost(5 * INSN_COST);
14753 format %{ "fcmps $src1, $src2\n\t"
14754 "csinvw($dst, zr, zr, eq\n\t"
14755 "csnegw($dst, $dst, $dst, lt)"
14756 %}
14757
14758 ins_encode %{
14759 Label done;
14760 FloatRegister s1 = as_FloatRegister($src1$$reg);
14761 FloatRegister s2 = as_FloatRegister($src2$$reg);
14762 Register d = as_Register($dst$$reg);
14763 __ fcmps(s1, s2);
14764 // installs 0 if EQ else -1
14765 __ csinvw(d, zr, zr, Assembler::EQ);
14766 // keeps -1 if less or unordered else installs 1
14767 __ csnegw(d, d, d, Assembler::LT);
14768 __ bind(done);
14769 %}
14770
14771 ins_pipe(pipe_class_default);
14772
14773 %}
14774
14775 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14776 %{
14777 match(Set dst (CmpD3 src1 src2));
14778 effect(KILL cr);
14779
14780 ins_cost(5 * INSN_COST);
14781 format %{ "fcmpd $src1, $src2\n\t"
14782 "csinvw($dst, zr, zr, eq\n\t"
14783 "csnegw($dst, $dst, $dst, lt)"
14784 %}
14785
14786 ins_encode %{
14787 Label done;
14788 FloatRegister s1 = as_FloatRegister($src1$$reg);
14789 FloatRegister s2 = as_FloatRegister($src2$$reg);
14790 Register d = as_Register($dst$$reg);
14791 __ fcmpd(s1, s2);
14792 // installs 0 if EQ else -1
14793 __ csinvw(d, zr, zr, Assembler::EQ);
14794 // keeps -1 if less or unordered else installs 1
14795 __ csnegw(d, d, d, Assembler::LT);
14796 __ bind(done);
14797 %}
14798 ins_pipe(pipe_class_default);
14799
14800 %}
14801
14802 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14803 %{
14804 match(Set dst (CmpF3 src1 zero));
14805 effect(KILL cr);
14806
14807 ins_cost(5 * INSN_COST);
14808 format %{ "fcmps $src1, 0.0\n\t"
14809 "csinvw($dst, zr, zr, eq\n\t"
14810 "csnegw($dst, $dst, $dst, lt)"
14811 %}
14812
14813 ins_encode %{
14814 Label done;
14815 FloatRegister s1 = as_FloatRegister($src1$$reg);
14816 Register d = as_Register($dst$$reg);
14817 __ fcmps(s1, 0.0);
14818 // installs 0 if EQ else -1
14819 __ csinvw(d, zr, zr, Assembler::EQ);
14820 // keeps -1 if less or unordered else installs 1
14821 __ csnegw(d, d, d, Assembler::LT);
14822 __ bind(done);
14823 %}
14824
14825 ins_pipe(pipe_class_default);
14826
14827 %}
14828
14829 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14830 %{
14831 match(Set dst (CmpD3 src1 zero));
14832 effect(KILL cr);
14833
14834 ins_cost(5 * INSN_COST);
14835 format %{ "fcmpd $src1, 0.0\n\t"
14836 "csinvw($dst, zr, zr, eq\n\t"
14837 "csnegw($dst, $dst, $dst, lt)"
14838 %}
14839
14840 ins_encode %{
14841 Label done;
14842 FloatRegister s1 = as_FloatRegister($src1$$reg);
14843 Register d = as_Register($dst$$reg);
14844 __ fcmpd(s1, 0.0);
14845 // installs 0 if EQ else -1
14846 __ csinvw(d, zr, zr, Assembler::EQ);
14847 // keeps -1 if less or unordered else installs 1
14848 __ csnegw(d, d, d, Assembler::LT);
14849 __ bind(done);
14850 %}
14851 ins_pipe(pipe_class_default);
14852
14853 %}
14854
14855 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14856 %{
14857 match(Set dst (CmpLTMask p q));
14858 effect(KILL cr);
14859
14860 ins_cost(3 * INSN_COST);
14861
14862 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14863 "csetw $dst, lt\n\t"
14864 "subw $dst, zr, $dst"
14865 %}
14866
14867 ins_encode %{
14868 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14869 __ csetw(as_Register($dst$$reg), Assembler::LT);
14870 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14871 %}
14872
14873 ins_pipe(ialu_reg_reg);
14874 %}
14875
14876 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14877 %{
14878 match(Set dst (CmpLTMask src zero));
14879 effect(KILL cr);
14880
14881 ins_cost(INSN_COST);
14882
14883 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14884
14885 ins_encode %{
14886 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14887 %}
14888
14889 ins_pipe(ialu_reg_shift);
14890 %}
14891
14892 // ============================================================================
14893 // Max and Min
14894
14895 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14896
14897 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14898 %{
14899 effect(DEF cr, USE src);
14900 ins_cost(INSN_COST);
14901 format %{ "cmpw $src, 0" %}
14902
14903 ins_encode %{
14904 __ cmpw($src$$Register, 0);
14905 %}
14906 ins_pipe(icmp_reg_imm);
14907 %}
14908
14909 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14910 %{
14911 match(Set dst (MinI src1 src2));
14912 ins_cost(INSN_COST * 3);
14913
14914 expand %{
14915 rFlagsReg cr;
14916 compI_reg_reg(cr, src1, src2);
14917 cmovI_reg_reg_lt(dst, src1, src2, cr);
14918 %}
14919 %}
14920
14921 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14922 %{
14923 match(Set dst (MaxI src1 src2));
14924 ins_cost(INSN_COST * 3);
14925
14926 expand %{
14927 rFlagsReg cr;
14928 compI_reg_reg(cr, src1, src2);
14929 cmovI_reg_reg_gt(dst, src1, src2, cr);
14930 %}
14931 %}
14932
14933
14934 // ============================================================================
14935 // Branch Instructions
14936
14937 // Direct Branch.
14938 instruct branch(label lbl)
14939 %{
14940 match(Goto);
14941
14942 effect(USE lbl);
14943
14944 ins_cost(BRANCH_COST);
14945 format %{ "b $lbl" %}
14946
14947 ins_encode(aarch64_enc_b(lbl));
14948
14949 ins_pipe(pipe_branch);
14950 %}
14951
14952 // Conditional Near Branch
14953 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14954 %{
14955 // Same match rule as `branchConFar'.
14956 match(If cmp cr);
14957
14958 effect(USE lbl);
14959
14960 ins_cost(BRANCH_COST);
14961 // If set to 1 this indicates that the current instruction is a
14962 // short variant of a long branch. This avoids using this
14963 // instruction in first-pass matching. It will then only be used in
14964 // the `Shorten_branches' pass.
14965 // ins_short_branch(1);
14966 format %{ "b$cmp $lbl" %}
14967
14968 ins_encode(aarch64_enc_br_con(cmp, lbl));
14969
14970 ins_pipe(pipe_branch_cond);
14971 %}
14972
14973 // Conditional Near Branch Unsigned
14974 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14975 %{
14976 // Same match rule as `branchConFar'.
14977 match(If cmp cr);
14978
14979 effect(USE lbl);
14980
14981 ins_cost(BRANCH_COST);
14982 // If set to 1 this indicates that the current instruction is a
14983 // short variant of a long branch. This avoids using this
14984 // instruction in first-pass matching. It will then only be used in
14985 // the `Shorten_branches' pass.
14986 // ins_short_branch(1);
14987 format %{ "b$cmp $lbl\t# unsigned" %}
14988
14989 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14990
14991 ins_pipe(pipe_branch_cond);
14992 %}
14993
14994 // Make use of CBZ and CBNZ. These instructions, as well as being
14995 // shorter than (cmp; branch), have the additional benefit of not
14996 // killing the flags.
14997
14998 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14999 match(If cmp (CmpI op1 op2));
15000 effect(USE labl);
15001
15002 ins_cost(BRANCH_COST);
15003 format %{ "cbw$cmp $op1, $labl" %}
15004 ins_encode %{
15005 Label* L = $labl$$label;
15006 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15007 if (cond == Assembler::EQ)
15008 __ cbzw($op1$$Register, *L);
15009 else
15010 __ cbnzw($op1$$Register, *L);
15011 %}
15012 ins_pipe(pipe_cmp_branch);
15013 %}
15014
15015 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15016 match(If cmp (CmpL op1 op2));
15017 effect(USE labl);
15018
15019 ins_cost(BRANCH_COST);
15020 format %{ "cb$cmp $op1, $labl" %}
15021 ins_encode %{
15022 Label* L = $labl$$label;
15023 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15024 if (cond == Assembler::EQ)
15025 __ cbz($op1$$Register, *L);
15026 else
15027 __ cbnz($op1$$Register, *L);
15028 %}
15029 ins_pipe(pipe_cmp_branch);
15030 %}
15031
15032 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15033 match(If cmp (CmpP op1 op2));
15034 effect(USE labl);
15035
15036 ins_cost(BRANCH_COST);
15037 format %{ "cb$cmp $op1, $labl" %}
15038 ins_encode %{
15039 Label* L = $labl$$label;
15040 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15041 if (cond == Assembler::EQ)
15042 __ cbz($op1$$Register, *L);
15043 else
15044 __ cbnz($op1$$Register, *L);
15045 %}
15046 ins_pipe(pipe_cmp_branch);
15047 %}
15048
15049 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15050 match(If cmp (CmpN op1 op2));
15051 effect(USE labl);
15052
15053 ins_cost(BRANCH_COST);
15054 format %{ "cbw$cmp $op1, $labl" %}
15055 ins_encode %{
15056 Label* L = $labl$$label;
15057 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15058 if (cond == Assembler::EQ)
15059 __ cbzw($op1$$Register, *L);
15060 else
15061 __ cbnzw($op1$$Register, *L);
15062 %}
15063 ins_pipe(pipe_cmp_branch);
15064 %}
15065
15066 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15067 match(If cmp (CmpP (DecodeN oop) zero));
15068 effect(USE labl);
15069
15070 ins_cost(BRANCH_COST);
15071 format %{ "cb$cmp $oop, $labl" %}
15072 ins_encode %{
15073 Label* L = $labl$$label;
15074 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15075 if (cond == Assembler::EQ)
15076 __ cbzw($oop$$Register, *L);
15077 else
15078 __ cbnzw($oop$$Register, *L);
15079 %}
15080 ins_pipe(pipe_cmp_branch);
15081 %}
15082
15083 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15084 match(If cmp (CmpU op1 op2));
15085 effect(USE labl);
15086
15087 ins_cost(BRANCH_COST);
15088 format %{ "cbw$cmp $op1, $labl" %}
15089 ins_encode %{
15090 Label* L = $labl$$label;
15091 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15092 if (cond == Assembler::EQ || cond == Assembler::LS) {
15093 __ cbzw($op1$$Register, *L);
15094 } else {
15095 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15096 __ cbnzw($op1$$Register, *L);
15097 }
15098 %}
15099 ins_pipe(pipe_cmp_branch);
15100 %}
15101
15102 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15103 match(If cmp (CmpUL op1 op2));
15104 effect(USE labl);
15105
15106 ins_cost(BRANCH_COST);
15107 format %{ "cb$cmp $op1, $labl" %}
15108 ins_encode %{
15109 Label* L = $labl$$label;
15110 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15111 if (cond == Assembler::EQ || cond == Assembler::LS) {
15112 __ cbz($op1$$Register, *L);
15113 } else {
15114 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15115 __ cbnz($op1$$Register, *L);
15116 }
15117 %}
15118 ins_pipe(pipe_cmp_branch);
15119 %}
15120
15121 // Test bit and Branch
15122
15123 // Patterns for short (< 32KiB) variants
15124 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15125 match(If cmp (CmpL op1 op2));
15126 effect(USE labl);
15127
15128 ins_cost(BRANCH_COST);
15129 format %{ "cb$cmp $op1, $labl # long" %}
15130 ins_encode %{
15131 Label* L = $labl$$label;
15132 Assembler::Condition cond =
15133 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15134 __ tbr(cond, $op1$$Register, 63, *L);
15135 %}
15136 ins_pipe(pipe_cmp_branch);
15137 ins_short_branch(1);
15138 %}
15139
15140 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15141 match(If cmp (CmpI op1 op2));
15142 effect(USE labl);
15143
15144 ins_cost(BRANCH_COST);
15145 format %{ "cb$cmp $op1, $labl # int" %}
15146 ins_encode %{
15147 Label* L = $labl$$label;
15148 Assembler::Condition cond =
15149 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15150 __ tbr(cond, $op1$$Register, 31, *L);
15151 %}
15152 ins_pipe(pipe_cmp_branch);
15153 ins_short_branch(1);
15154 %}
15155
15156 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15157 match(If cmp (CmpL (AndL op1 op2) op3));
15158 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15159 effect(USE labl);
15160
15161 ins_cost(BRANCH_COST);
15162 format %{ "tb$cmp $op1, $op2, $labl" %}
15163 ins_encode %{
15164 Label* L = $labl$$label;
15165 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15166 int bit = exact_log2_long($op2$$constant);
15167 __ tbr(cond, $op1$$Register, bit, *L);
15168 %}
15169 ins_pipe(pipe_cmp_branch);
15170 ins_short_branch(1);
15171 %}
15172
15173 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15174 match(If cmp (CmpI (AndI op1 op2) op3));
15175 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15176 effect(USE labl);
15177
15178 ins_cost(BRANCH_COST);
15179 format %{ "tb$cmp $op1, $op2, $labl" %}
15180 ins_encode %{
15181 Label* L = $labl$$label;
15182 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15183 int bit = exact_log2((juint)$op2$$constant);
15184 __ tbr(cond, $op1$$Register, bit, *L);
15185 %}
15186 ins_pipe(pipe_cmp_branch);
15187 ins_short_branch(1);
15188 %}
15189
15190 // And far variants
15191 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15192 match(If cmp (CmpL op1 op2));
15193 effect(USE labl);
15194
15195 ins_cost(BRANCH_COST);
15196 format %{ "cb$cmp $op1, $labl # long" %}
15197 ins_encode %{
15198 Label* L = $labl$$label;
15199 Assembler::Condition cond =
15200 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15201 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15202 %}
15203 ins_pipe(pipe_cmp_branch);
15204 %}
15205
15206 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15207 match(If cmp (CmpI op1 op2));
15208 effect(USE labl);
15209
15210 ins_cost(BRANCH_COST);
15211 format %{ "cb$cmp $op1, $labl # int" %}
15212 ins_encode %{
15213 Label* L = $labl$$label;
15214 Assembler::Condition cond =
15215 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15216 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15217 %}
15218 ins_pipe(pipe_cmp_branch);
15219 %}
15220
15221 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15222 match(If cmp (CmpL (AndL op1 op2) op3));
15223 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15224 effect(USE labl);
15225
15226 ins_cost(BRANCH_COST);
15227 format %{ "tb$cmp $op1, $op2, $labl" %}
15228 ins_encode %{
15229 Label* L = $labl$$label;
15230 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15231 int bit = exact_log2_long($op2$$constant);
15232 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15233 %}
15234 ins_pipe(pipe_cmp_branch);
15235 %}
15236
15237 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15238 match(If cmp (CmpI (AndI op1 op2) op3));
15239 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15240 effect(USE labl);
15241
15242 ins_cost(BRANCH_COST);
15243 format %{ "tb$cmp $op1, $op2, $labl" %}
15244 ins_encode %{
15245 Label* L = $labl$$label;
15246 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15247 int bit = exact_log2((juint)$op2$$constant);
15248 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15249 %}
15250 ins_pipe(pipe_cmp_branch);
15251 %}
15252
15253 // Test bits
15254
15255 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15256 match(Set cr (CmpL (AndL op1 op2) op3));
15257 predicate(Assembler::operand_valid_for_logical_immediate
15258 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15259
15260 ins_cost(INSN_COST);
15261 format %{ "tst $op1, $op2 # long" %}
15262 ins_encode %{
15263 __ tst($op1$$Register, $op2$$constant);
15264 %}
15265 ins_pipe(ialu_reg_reg);
15266 %}
15267
15268 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15269 match(Set cr (CmpI (AndI op1 op2) op3));
15270 predicate(Assembler::operand_valid_for_logical_immediate
15271 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15272
15273 ins_cost(INSN_COST);
15274 format %{ "tst $op1, $op2 # int" %}
15275 ins_encode %{
15276 __ tstw($op1$$Register, $op2$$constant);
15277 %}
15278 ins_pipe(ialu_reg_reg);
15279 %}
15280
15281 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15282 match(Set cr (CmpL (AndL op1 op2) op3));
15283
15284 ins_cost(INSN_COST);
15285 format %{ "tst $op1, $op2 # long" %}
15286 ins_encode %{
15287 __ tst($op1$$Register, $op2$$Register);
15288 %}
15289 ins_pipe(ialu_reg_reg);
15290 %}
15291
15292 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15293 match(Set cr (CmpI (AndI op1 op2) op3));
15294
15295 ins_cost(INSN_COST);
15296 format %{ "tstw $op1, $op2 # int" %}
15297 ins_encode %{
15298 __ tstw($op1$$Register, $op2$$Register);
15299 %}
15300 ins_pipe(ialu_reg_reg);
15301 %}
15302
15303
15304 // Conditional Far Branch
15305 // Conditional Far Branch Unsigned
15306 // TODO: fixme
15307
15308 // counted loop end branch near
15309 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15310 %{
15311 match(CountedLoopEnd cmp cr);
15312
15313 effect(USE lbl);
15314
15315 ins_cost(BRANCH_COST);
15316 // short variant.
15317 // ins_short_branch(1);
15318 format %{ "b$cmp $lbl \t// counted loop end" %}
15319
15320 ins_encode(aarch64_enc_br_con(cmp, lbl));
15321
15322 ins_pipe(pipe_branch);
15323 %}
15324
15325 // counted loop end branch far
15326 // TODO: fixme
15327
15328 // ============================================================================
15329 // inlined locking and unlocking
15330
15331 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15332 %{
15333 match(Set cr (FastLock object box));
15334 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15335
15336 ins_cost(5 * INSN_COST);
15337 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15338
15339 ins_encode %{
15340 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15341 %}
15342
15343 ins_pipe(pipe_serial);
15344 %}
15345
15346 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15347 %{
15348 match(Set cr (FastUnlock object box));
15349 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15350
15351 ins_cost(5 * INSN_COST);
15352 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15353
15354 ins_encode %{
15355 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15356 %}
15357
15358 ins_pipe(pipe_serial);
15359 %}
15360
15361 // ============================================================================
15362 // Safepoint Instructions
15363
15364 // TODO
15365 // provide a near and far version of this code
15366
15367 instruct safePoint(rFlagsReg cr, iRegP poll)
15368 %{
15369 match(SafePoint poll);
15370 effect(KILL cr);
15371
15372 format %{
15373 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15374 %}
15375 ins_encode %{
15376 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15377 %}
15378 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15379 %}
15380
15381
15382 // ============================================================================
15383 // Procedure Call/Return Instructions
15384
15385 // Call Java Static Instruction
15386
15387 instruct CallStaticJavaDirect(method meth)
15388 %{
15389 match(CallStaticJava);
15390
15391 effect(USE meth);
15392
15393 ins_cost(CALL_COST);
15394
15395 format %{ "call,static $meth \t// ==> " %}
15396
15397 ins_encode(aarch64_enc_java_static_call(meth),
15398 aarch64_enc_call_epilog);
15399
15400 ins_pipe(pipe_class_call);
15401 %}
15402
15403 // TO HERE
15404
15405 // Call Java Dynamic Instruction
15406 instruct CallDynamicJavaDirect(method meth)
15407 %{
15408 match(CallDynamicJava);
15409
15410 effect(USE meth);
15411
15412 ins_cost(CALL_COST);
15413
15414 format %{ "CALL,dynamic $meth \t// ==> " %}
15415
15416 ins_encode(aarch64_enc_java_dynamic_call(meth),
15417 aarch64_enc_call_epilog);
15418
15419 ins_pipe(pipe_class_call);
15420 %}
15421
15422 // Call Runtime Instruction
15423
15424 instruct CallRuntimeDirect(method meth)
15425 %{
15426 match(CallRuntime);
15427
15428 effect(USE meth);
15429
15430 ins_cost(CALL_COST);
15431
15432 format %{ "CALL, runtime $meth" %}
15433
15434 ins_encode( aarch64_enc_java_to_runtime(meth) );
15435
15436 ins_pipe(pipe_class_call);
15437 %}
15438
15439 // Call Runtime Instruction
15440
15441 instruct CallLeafDirect(method meth)
15442 %{
15443 match(CallLeaf);
15444
15445 effect(USE meth);
15446
15447 ins_cost(CALL_COST);
15448
15449 format %{ "CALL, runtime leaf $meth" %}
15450
15451 ins_encode( aarch64_enc_java_to_runtime(meth) );
15452
15453 ins_pipe(pipe_class_call);
15454 %}
15455
15456 // Call Runtime Instruction without safepoint and with vector arguments
15457 instruct CallLeafDirectVector(method meth)
15458 %{
15459 match(CallLeafVector);
15460
15461 effect(USE meth);
15462
15463 ins_cost(CALL_COST);
15464
15465 format %{ "CALL, runtime leaf vector $meth" %}
15466
15467 ins_encode(aarch64_enc_java_to_runtime(meth));
15468
15469 ins_pipe(pipe_class_call);
15470 %}
15471
15472 // Call Runtime Instruction
15473
15474 instruct CallLeafNoFPDirect(method meth)
15475 %{
15476 match(CallLeafNoFP);
15477
15478 effect(USE meth);
15479
15480 ins_cost(CALL_COST);
15481
15482 format %{ "CALL, runtime leaf nofp $meth" %}
15483
15484 ins_encode( aarch64_enc_java_to_runtime(meth) );
15485
15486 ins_pipe(pipe_class_call);
15487 %}
15488
15489 // Tail Call; Jump from runtime stub to Java code.
15490 // Also known as an 'interprocedural jump'.
15491 // Target of jump will eventually return to caller.
15492 // TailJump below removes the return address.
15493 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15494 // emitted just above the TailCall which has reset rfp to the caller state.
15495 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15496 %{
15497 match(TailCall jump_target method_ptr);
15498
15499 ins_cost(CALL_COST);
15500
15501 format %{ "br $jump_target\t# $method_ptr holds method" %}
15502
15503 ins_encode(aarch64_enc_tail_call(jump_target));
15504
15505 ins_pipe(pipe_class_call);
15506 %}
15507
15508 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15509 %{
15510 match(TailJump jump_target ex_oop);
15511
15512 ins_cost(CALL_COST);
15513
15514 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15515
15516 ins_encode(aarch64_enc_tail_jmp(jump_target));
15517
15518 ins_pipe(pipe_class_call);
15519 %}
15520
15521 // Forward exception.
15522 instruct ForwardExceptionjmp()
15523 %{
15524 match(ForwardException);
15525 ins_cost(CALL_COST);
15526
15527 format %{ "b forward_exception_stub" %}
15528 ins_encode %{
15529 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15530 %}
15531 ins_pipe(pipe_class_call);
15532 %}
15533
15534 // Create exception oop: created by stack-crawling runtime code.
15535 // Created exception is now available to this handler, and is setup
15536 // just prior to jumping to this handler. No code emitted.
15537 // TODO check
15538 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15539 instruct CreateException(iRegP_R0 ex_oop)
15540 %{
15541 match(Set ex_oop (CreateEx));
15542
15543 format %{ " -- \t// exception oop; no code emitted" %}
15544
15545 size(0);
15546
15547 ins_encode( /*empty*/ );
15548
15549 ins_pipe(pipe_class_empty);
15550 %}
15551
15552 // Rethrow exception: The exception oop will come in the first
15553 // argument position. Then JUMP (not call) to the rethrow stub code.
15554 instruct RethrowException() %{
15555 match(Rethrow);
15556 ins_cost(CALL_COST);
15557
15558 format %{ "b rethrow_stub" %}
15559
15560 ins_encode( aarch64_enc_rethrow() );
15561
15562 ins_pipe(pipe_class_call);
15563 %}
15564
15565
15566 // Return Instruction
15567 // epilog node loads ret address into lr as part of frame pop
15568 instruct Ret()
15569 %{
15570 match(Return);
15571
15572 format %{ "ret\t// return register" %}
15573
15574 ins_encode( aarch64_enc_ret() );
15575
15576 ins_pipe(pipe_branch);
15577 %}
15578
15579 // Die now.
15580 instruct ShouldNotReachHere() %{
15581 match(Halt);
15582
15583 ins_cost(CALL_COST);
15584 format %{ "ShouldNotReachHere" %}
15585
15586 ins_encode %{
15587 if (is_reachable()) {
15588 const char* str = __ code_string(_halt_reason);
15589 __ stop(str);
15590 }
15591 %}
15592
15593 ins_pipe(pipe_class_default);
15594 %}
15595
15596 // ============================================================================
15597 // Partial Subtype Check
15598 //
15599 // superklass array for an instance of the superklass. Set a hidden
15600 // internal cache on a hit (cache is checked with exposed code in
15601 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15602 // encoding ALSO sets flags.
15603
15604 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15605 %{
15606 match(Set result (PartialSubtypeCheck sub super));
15607 predicate(!UseSecondarySupersTable);
15608 effect(KILL cr, KILL temp);
15609
15610 ins_cost(20 * INSN_COST); // slightly larger than the next version
15611 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15612
15613 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15614
15615 opcode(0x1); // Force zero of result reg on hit
15616
15617 ins_pipe(pipe_class_memory);
15618 %}
15619
15620 // Two versions of partialSubtypeCheck, both used when we need to
15621 // search for a super class in the secondary supers array. The first
15622 // is used when we don't know _a priori_ the class being searched
15623 // for. The second, far more common, is used when we do know: this is
15624 // used for instanceof, checkcast, and any case where C2 can determine
15625 // it by constant propagation.
15626
15627 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15628 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15629 rFlagsReg cr)
15630 %{
15631 match(Set result (PartialSubtypeCheck sub super));
15632 predicate(UseSecondarySupersTable);
15633 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15634
15635 ins_cost(10 * INSN_COST); // slightly larger than the next version
15636 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15637
15638 ins_encode %{
15639 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15640 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15641 $vtemp$$FloatRegister,
15642 $result$$Register, /*L_success*/nullptr);
15643 %}
15644
15645 ins_pipe(pipe_class_memory);
15646 %}
15647
15648 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15649 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15650 rFlagsReg cr)
15651 %{
15652 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15653 predicate(UseSecondarySupersTable);
15654 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15655
15656 ins_cost(5 * INSN_COST); // smaller than the next version
15657 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15658
15659 ins_encode %{
15660 bool success = false;
15661 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15662 if (InlineSecondarySupersTest) {
15663 success =
15664 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15665 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15666 $vtemp$$FloatRegister,
15667 $result$$Register,
15668 super_klass_slot);
15669 } else {
15670 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15671 success = (call != nullptr);
15672 }
15673 if (!success) {
15674 ciEnv::current()->record_failure("CodeCache is full");
15675 return;
15676 }
15677 %}
15678
15679 ins_pipe(pipe_class_memory);
15680 %}
15681
15682 // Intrisics for String.compareTo()
15683
15684 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15685 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15686 %{
15687 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15688 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15689 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15690
15691 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15692 ins_encode %{
15693 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15694 __ string_compare($str1$$Register, $str2$$Register,
15695 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15696 $tmp1$$Register, $tmp2$$Register,
15697 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15698 %}
15699 ins_pipe(pipe_class_memory);
15700 %}
15701
15702 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15703 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15704 %{
15705 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15706 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15707 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15708
15709 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15710 ins_encode %{
15711 __ string_compare($str1$$Register, $str2$$Register,
15712 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15713 $tmp1$$Register, $tmp2$$Register,
15714 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15715 %}
15716 ins_pipe(pipe_class_memory);
15717 %}
15718
15719 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15720 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15721 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15722 %{
15723 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15724 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15725 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15726 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15727
15728 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15729 ins_encode %{
15730 __ string_compare($str1$$Register, $str2$$Register,
15731 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15732 $tmp1$$Register, $tmp2$$Register,
15733 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15734 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15735 %}
15736 ins_pipe(pipe_class_memory);
15737 %}
15738
15739 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15740 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15741 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15742 %{
15743 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15744 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15745 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15746 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15747
15748 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15749 ins_encode %{
15750 __ string_compare($str1$$Register, $str2$$Register,
15751 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15752 $tmp1$$Register, $tmp2$$Register,
15753 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15754 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15755 %}
15756 ins_pipe(pipe_class_memory);
15757 %}
15758
15759 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15760 // these string_compare variants as NEON register type for convenience so that the prototype of
15761 // string_compare can be shared with all variants.
15762
15763 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15764 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15765 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15766 pRegGov_P1 pgtmp2, rFlagsReg cr)
15767 %{
15768 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15769 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15770 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15771 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15772
15773 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15774 ins_encode %{
15775 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15776 __ string_compare($str1$$Register, $str2$$Register,
15777 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15778 $tmp1$$Register, $tmp2$$Register,
15779 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15780 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15781 StrIntrinsicNode::LL);
15782 %}
15783 ins_pipe(pipe_class_memory);
15784 %}
15785
15786 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15787 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15788 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15789 pRegGov_P1 pgtmp2, rFlagsReg cr)
15790 %{
15791 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15792 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15793 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15794 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15795
15796 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15797 ins_encode %{
15798 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15799 __ string_compare($str1$$Register, $str2$$Register,
15800 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15801 $tmp1$$Register, $tmp2$$Register,
15802 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15803 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15804 StrIntrinsicNode::LU);
15805 %}
15806 ins_pipe(pipe_class_memory);
15807 %}
15808
15809 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15810 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15811 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15812 pRegGov_P1 pgtmp2, rFlagsReg cr)
15813 %{
15814 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15815 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15816 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15817 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15818
15819 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15820 ins_encode %{
15821 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15822 __ string_compare($str1$$Register, $str2$$Register,
15823 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15824 $tmp1$$Register, $tmp2$$Register,
15825 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15826 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15827 StrIntrinsicNode::UL);
15828 %}
15829 ins_pipe(pipe_class_memory);
15830 %}
15831
15832 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15833 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15834 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15835 pRegGov_P1 pgtmp2, rFlagsReg cr)
15836 %{
15837 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15838 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15839 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15840 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15841
15842 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15843 ins_encode %{
15844 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15845 __ string_compare($str1$$Register, $str2$$Register,
15846 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15847 $tmp1$$Register, $tmp2$$Register,
15848 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15849 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15850 StrIntrinsicNode::UU);
15851 %}
15852 ins_pipe(pipe_class_memory);
15853 %}
15854
15855 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15856 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15857 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15858 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15859 %{
15860 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15861 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15862 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15863 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15864 TEMP vtmp0, TEMP vtmp1, KILL cr);
15865 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15866 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15867
15868 ins_encode %{
15869 __ string_indexof($str1$$Register, $str2$$Register,
15870 $cnt1$$Register, $cnt2$$Register,
15871 $tmp1$$Register, $tmp2$$Register,
15872 $tmp3$$Register, $tmp4$$Register,
15873 $tmp5$$Register, $tmp6$$Register,
15874 -1, $result$$Register, StrIntrinsicNode::UU);
15875 %}
15876 ins_pipe(pipe_class_memory);
15877 %}
15878
15879 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15880 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15881 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15882 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15883 %{
15884 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15885 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15886 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15887 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15888 TEMP vtmp0, TEMP vtmp1, KILL cr);
15889 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15890 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15891
15892 ins_encode %{
15893 __ string_indexof($str1$$Register, $str2$$Register,
15894 $cnt1$$Register, $cnt2$$Register,
15895 $tmp1$$Register, $tmp2$$Register,
15896 $tmp3$$Register, $tmp4$$Register,
15897 $tmp5$$Register, $tmp6$$Register,
15898 -1, $result$$Register, StrIntrinsicNode::LL);
15899 %}
15900 ins_pipe(pipe_class_memory);
15901 %}
15902
15903 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15904 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15905 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15906 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15907 %{
15908 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15909 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15910 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15911 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15912 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15913 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15914 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15915
15916 ins_encode %{
15917 __ string_indexof($str1$$Register, $str2$$Register,
15918 $cnt1$$Register, $cnt2$$Register,
15919 $tmp1$$Register, $tmp2$$Register,
15920 $tmp3$$Register, $tmp4$$Register,
15921 $tmp5$$Register, $tmp6$$Register,
15922 -1, $result$$Register, StrIntrinsicNode::UL);
15923 %}
15924 ins_pipe(pipe_class_memory);
15925 %}
15926
15927 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15928 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15929 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15930 %{
15931 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15932 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15933 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15934 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15935 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15936 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15937
15938 ins_encode %{
15939 int icnt2 = (int)$int_cnt2$$constant;
15940 __ string_indexof($str1$$Register, $str2$$Register,
15941 $cnt1$$Register, zr,
15942 $tmp1$$Register, $tmp2$$Register,
15943 $tmp3$$Register, $tmp4$$Register, zr, zr,
15944 icnt2, $result$$Register, StrIntrinsicNode::UU);
15945 %}
15946 ins_pipe(pipe_class_memory);
15947 %}
15948
15949 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15950 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15951 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15952 %{
15953 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15954 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15955 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15956 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15957 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15958 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15959
15960 ins_encode %{
15961 int icnt2 = (int)$int_cnt2$$constant;
15962 __ string_indexof($str1$$Register, $str2$$Register,
15963 $cnt1$$Register, zr,
15964 $tmp1$$Register, $tmp2$$Register,
15965 $tmp3$$Register, $tmp4$$Register, zr, zr,
15966 icnt2, $result$$Register, StrIntrinsicNode::LL);
15967 %}
15968 ins_pipe(pipe_class_memory);
15969 %}
15970
15971 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15972 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15973 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15974 %{
15975 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15976 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15977 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15978 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15979 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15980 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15981
15982 ins_encode %{
15983 int icnt2 = (int)$int_cnt2$$constant;
15984 __ string_indexof($str1$$Register, $str2$$Register,
15985 $cnt1$$Register, zr,
15986 $tmp1$$Register, $tmp2$$Register,
15987 $tmp3$$Register, $tmp4$$Register, zr, zr,
15988 icnt2, $result$$Register, StrIntrinsicNode::UL);
15989 %}
15990 ins_pipe(pipe_class_memory);
15991 %}
15992
15993 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15994 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15995 iRegINoSp tmp3, rFlagsReg cr)
15996 %{
15997 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15998 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15999 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16000 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16001
16002 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16003
16004 ins_encode %{
16005 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16006 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16007 $tmp3$$Register);
16008 %}
16009 ins_pipe(pipe_class_memory);
16010 %}
16011
16012 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16013 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16014 iRegINoSp tmp3, rFlagsReg cr)
16015 %{
16016 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16017 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16018 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16019 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16020
16021 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16022
16023 ins_encode %{
16024 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16025 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16026 $tmp3$$Register);
16027 %}
16028 ins_pipe(pipe_class_memory);
16029 %}
16030
16031 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16032 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16033 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16034 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16035 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16036 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16037 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16038 ins_encode %{
16039 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16040 $result$$Register, $ztmp1$$FloatRegister,
16041 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16042 $ptmp$$PRegister, true /* isL */);
16043 %}
16044 ins_pipe(pipe_class_memory);
16045 %}
16046
16047 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16048 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16049 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16050 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16051 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16052 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16053 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16054 ins_encode %{
16055 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16056 $result$$Register, $ztmp1$$FloatRegister,
16057 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16058 $ptmp$$PRegister, false /* isL */);
16059 %}
16060 ins_pipe(pipe_class_memory);
16061 %}
16062
16063 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16064 iRegI_R0 result, rFlagsReg cr)
16065 %{
16066 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16067 match(Set result (StrEquals (Binary str1 str2) cnt));
16068 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16069
16070 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16071 ins_encode %{
16072 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16073 __ string_equals($str1$$Register, $str2$$Register,
16074 $result$$Register, $cnt$$Register);
16075 %}
16076 ins_pipe(pipe_class_memory);
16077 %}
16078
16079 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16080 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16081 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16082 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16083 iRegP_R10 tmp, rFlagsReg cr)
16084 %{
16085 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16086 match(Set result (AryEq ary1 ary2));
16087 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16088 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16089 TEMP vtmp6, TEMP vtmp7, KILL cr);
16090
16091 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16092 ins_encode %{
16093 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16094 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16095 $result$$Register, $tmp$$Register, 1);
16096 if (tpc == nullptr) {
16097 ciEnv::current()->record_failure("CodeCache is full");
16098 return;
16099 }
16100 %}
16101 ins_pipe(pipe_class_memory);
16102 %}
16103
16104 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16105 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16106 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16107 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16108 iRegP_R10 tmp, rFlagsReg cr)
16109 %{
16110 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16111 match(Set result (AryEq ary1 ary2));
16112 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16113 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16114 TEMP vtmp6, TEMP vtmp7, KILL cr);
16115
16116 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16117 ins_encode %{
16118 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16119 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16120 $result$$Register, $tmp$$Register, 2);
16121 if (tpc == nullptr) {
16122 ciEnv::current()->record_failure("CodeCache is full");
16123 return;
16124 }
16125 %}
16126 ins_pipe(pipe_class_memory);
16127 %}
16128
16129 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16130 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16131 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16132 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16133 %{
16134 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16135 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16136 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16137
16138 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16139 ins_encode %{
16140 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16141 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16142 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16143 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16144 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16145 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16146 (BasicType)$basic_type$$constant);
16147 if (tpc == nullptr) {
16148 ciEnv::current()->record_failure("CodeCache is full");
16149 return;
16150 }
16151 %}
16152 ins_pipe(pipe_class_memory);
16153 %}
16154
16155 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16156 %{
16157 match(Set result (CountPositives ary1 len));
16158 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16159 format %{ "count positives byte[] $ary1,$len -> $result" %}
16160 ins_encode %{
16161 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16162 if (tpc == nullptr) {
16163 ciEnv::current()->record_failure("CodeCache is full");
16164 return;
16165 }
16166 %}
16167 ins_pipe( pipe_slow );
16168 %}
16169
16170 // fast char[] to byte[] compression
16171 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16172 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16173 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16174 iRegI_R0 result, rFlagsReg cr)
16175 %{
16176 match(Set result (StrCompressedCopy src (Binary dst len)));
16177 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16178 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16179
16180 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16181 ins_encode %{
16182 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16183 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16184 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16185 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16186 %}
16187 ins_pipe(pipe_slow);
16188 %}
16189
16190 // fast byte[] to char[] inflation
16191 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16192 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16193 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16194 %{
16195 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16196 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16197 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16198 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16199
16200 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16201 ins_encode %{
16202 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16203 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16204 $vtmp2$$FloatRegister, $tmp$$Register);
16205 if (tpc == nullptr) {
16206 ciEnv::current()->record_failure("CodeCache is full");
16207 return;
16208 }
16209 %}
16210 ins_pipe(pipe_class_memory);
16211 %}
16212
16213 // encode char[] to byte[] in ISO_8859_1
16214 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16215 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16216 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16217 iRegI_R0 result, rFlagsReg cr)
16218 %{
16219 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16220 match(Set result (EncodeISOArray src (Binary dst len)));
16221 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16222 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16223
16224 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16225 ins_encode %{
16226 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16227 $result$$Register, false,
16228 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16229 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16230 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16231 %}
16232 ins_pipe(pipe_class_memory);
16233 %}
16234
16235 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16236 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16237 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16238 iRegI_R0 result, rFlagsReg cr)
16239 %{
16240 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16241 match(Set result (EncodeISOArray src (Binary dst len)));
16242 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16243 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16244
16245 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16246 ins_encode %{
16247 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16248 $result$$Register, true,
16249 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16250 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16251 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16252 %}
16253 ins_pipe(pipe_class_memory);
16254 %}
16255
16256 //----------------------------- CompressBits/ExpandBits ------------------------
16257
16258 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16259 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16260 match(Set dst (CompressBits src mask));
16261 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16262 format %{ "mov $tsrc, $src\n\t"
16263 "mov $tmask, $mask\n\t"
16264 "bext $tdst, $tsrc, $tmask\n\t"
16265 "mov $dst, $tdst"
16266 %}
16267 ins_encode %{
16268 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16269 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16270 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16271 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16272 %}
16273 ins_pipe(pipe_slow);
16274 %}
16275
16276 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16277 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16278 match(Set dst (CompressBits (LoadI mem) mask));
16279 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16280 format %{ "ldrs $tsrc, $mem\n\t"
16281 "ldrs $tmask, $mask\n\t"
16282 "bext $tdst, $tsrc, $tmask\n\t"
16283 "mov $dst, $tdst"
16284 %}
16285 ins_encode %{
16286 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16287 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16288 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16289 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16290 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16291 %}
16292 ins_pipe(pipe_slow);
16293 %}
16294
16295 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16296 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16297 match(Set dst (CompressBits src mask));
16298 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16299 format %{ "mov $tsrc, $src\n\t"
16300 "mov $tmask, $mask\n\t"
16301 "bext $tdst, $tsrc, $tmask\n\t"
16302 "mov $dst, $tdst"
16303 %}
16304 ins_encode %{
16305 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16306 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16307 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16308 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16309 %}
16310 ins_pipe(pipe_slow);
16311 %}
16312
16313 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16314 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16315 match(Set dst (CompressBits (LoadL mem) mask));
16316 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16317 format %{ "ldrd $tsrc, $mem\n\t"
16318 "ldrd $tmask, $mask\n\t"
16319 "bext $tdst, $tsrc, $tmask\n\t"
16320 "mov $dst, $tdst"
16321 %}
16322 ins_encode %{
16323 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16324 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16325 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16326 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16327 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16328 %}
16329 ins_pipe(pipe_slow);
16330 %}
16331
16332 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16333 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16334 match(Set dst (ExpandBits src mask));
16335 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16336 format %{ "mov $tsrc, $src\n\t"
16337 "mov $tmask, $mask\n\t"
16338 "bdep $tdst, $tsrc, $tmask\n\t"
16339 "mov $dst, $tdst"
16340 %}
16341 ins_encode %{
16342 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16343 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16344 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16345 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16346 %}
16347 ins_pipe(pipe_slow);
16348 %}
16349
16350 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16351 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16352 match(Set dst (ExpandBits (LoadI mem) mask));
16353 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16354 format %{ "ldrs $tsrc, $mem\n\t"
16355 "ldrs $tmask, $mask\n\t"
16356 "bdep $tdst, $tsrc, $tmask\n\t"
16357 "mov $dst, $tdst"
16358 %}
16359 ins_encode %{
16360 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16361 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16362 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16363 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16364 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16365 %}
16366 ins_pipe(pipe_slow);
16367 %}
16368
16369 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16370 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16371 match(Set dst (ExpandBits src mask));
16372 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16373 format %{ "mov $tsrc, $src\n\t"
16374 "mov $tmask, $mask\n\t"
16375 "bdep $tdst, $tsrc, $tmask\n\t"
16376 "mov $dst, $tdst"
16377 %}
16378 ins_encode %{
16379 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16380 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16381 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16382 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16383 %}
16384 ins_pipe(pipe_slow);
16385 %}
16386
16387
16388 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16389 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16390 match(Set dst (ExpandBits (LoadL mem) mask));
16391 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16392 format %{ "ldrd $tsrc, $mem\n\t"
16393 "ldrd $tmask, $mask\n\t"
16394 "bdep $tdst, $tsrc, $tmask\n\t"
16395 "mov $dst, $tdst"
16396 %}
16397 ins_encode %{
16398 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16399 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16400 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16401 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16402 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16403 %}
16404 ins_pipe(pipe_slow);
16405 %}
16406
16407 //----------------------------- Reinterpret ----------------------------------
16408 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16409 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16410 match(Set dst (ReinterpretHF2S src));
16411 format %{ "reinterpretHF2S $dst, $src" %}
16412 ins_encode %{
16413 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16414 %}
16415 ins_pipe(pipe_slow);
16416 %}
16417
16418 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16419 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16420 match(Set dst (ReinterpretS2HF src));
16421 format %{ "reinterpretS2HF $dst, $src" %}
16422 ins_encode %{
16423 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16424 %}
16425 ins_pipe(pipe_slow);
16426 %}
16427
16428 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16429 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16430 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16431 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16432 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16433 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16434 // can be omitted in this pattern, resulting in -
16435 // fcvt $dst, $src // Convert float to half-precision float
16436 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16437 %{
16438 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16439 format %{ "convF2HFAndS2HF $dst, $src" %}
16440 ins_encode %{
16441 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16442 %}
16443 ins_pipe(pipe_slow);
16444 %}
16445
16446 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16447 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16448 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16449 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16450 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16451 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16452 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16453 // resulting in -
16454 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16455 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16456 %{
16457 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16458 format %{ "convHF2SAndHF2F $dst, $src" %}
16459 ins_encode %{
16460 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16461 %}
16462 ins_pipe(pipe_slow);
16463 %}
16464
16465 // ============================================================================
16466 // This name is KNOWN by the ADLC and cannot be changed.
16467 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16468 // for this guy.
16469 instruct tlsLoadP(thread_RegP dst)
16470 %{
16471 match(Set dst (ThreadLocal));
16472
16473 ins_cost(0);
16474
16475 format %{ " -- \t// $dst=Thread::current(), empty" %}
16476
16477 size(0);
16478
16479 ins_encode( /*empty*/ );
16480
16481 ins_pipe(pipe_class_empty);
16482 %}
16483
16484 //----------PEEPHOLE RULES-----------------------------------------------------
16485 // These must follow all instruction definitions as they use the names
16486 // defined in the instructions definitions.
16487 //
16488 // peepmatch ( root_instr_name [preceding_instruction]* );
16489 //
16490 // peepconstraint %{
16491 // (instruction_number.operand_name relational_op instruction_number.operand_name
16492 // [, ...] );
16493 // // instruction numbers are zero-based using left to right order in peepmatch
16494 //
16495 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16496 // // provide an instruction_number.operand_name for each operand that appears
16497 // // in the replacement instruction's match rule
16498 //
16499 // ---------VM FLAGS---------------------------------------------------------
16500 //
16501 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16502 //
16503 // Each peephole rule is given an identifying number starting with zero and
16504 // increasing by one in the order seen by the parser. An individual peephole
16505 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16506 // on the command-line.
16507 //
16508 // ---------CURRENT LIMITATIONS----------------------------------------------
16509 //
16510 // Only match adjacent instructions in same basic block
16511 // Only equality constraints
16512 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16513 // Only one replacement instruction
16514 //
16515 // ---------EXAMPLE----------------------------------------------------------
16516 //
16517 // // pertinent parts of existing instructions in architecture description
16518 // instruct movI(iRegINoSp dst, iRegI src)
16519 // %{
16520 // match(Set dst (CopyI src));
16521 // %}
16522 //
16523 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16524 // %{
16525 // match(Set dst (AddI dst src));
16526 // effect(KILL cr);
16527 // %}
16528 //
16529 // // Change (inc mov) to lea
16530 // peephole %{
16531 // // increment preceded by register-register move
16532 // peepmatch ( incI_iReg movI );
16533 // // require that the destination register of the increment
16534 // // match the destination register of the move
16535 // peepconstraint ( 0.dst == 1.dst );
16536 // // construct a replacement instruction that sets
16537 // // the destination to ( move's source register + one )
16538 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16539 // %}
16540 //
16541
16542 // Implementation no longer uses movX instructions since
16543 // machine-independent system no longer uses CopyX nodes.
16544 //
16545 // peephole
16546 // %{
16547 // peepmatch (incI_iReg movI);
16548 // peepconstraint (0.dst == 1.dst);
16549 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16550 // %}
16551
16552 // peephole
16553 // %{
16554 // peepmatch (decI_iReg movI);
16555 // peepconstraint (0.dst == 1.dst);
16556 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16557 // %}
16558
16559 // peephole
16560 // %{
16561 // peepmatch (addI_iReg_imm movI);
16562 // peepconstraint (0.dst == 1.dst);
16563 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16564 // %}
16565
16566 // peephole
16567 // %{
16568 // peepmatch (incL_iReg movL);
16569 // peepconstraint (0.dst == 1.dst);
16570 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16571 // %}
16572
16573 // peephole
16574 // %{
16575 // peepmatch (decL_iReg movL);
16576 // peepconstraint (0.dst == 1.dst);
16577 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16578 // %}
16579
16580 // peephole
16581 // %{
16582 // peepmatch (addL_iReg_imm movL);
16583 // peepconstraint (0.dst == 1.dst);
16584 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16585 // %}
16586
16587 // peephole
16588 // %{
16589 // peepmatch (addP_iReg_imm movP);
16590 // peepconstraint (0.dst == 1.dst);
16591 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16592 // %}
16593
16594 // // Change load of spilled value to only a spill
16595 // instruct storeI(memory mem, iRegI src)
16596 // %{
16597 // match(Set mem (StoreI mem src));
16598 // %}
16599 //
16600 // instruct loadI(iRegINoSp dst, memory mem)
16601 // %{
16602 // match(Set dst (LoadI mem));
16603 // %}
16604 //
16605
16606 //----------SMARTSPILL RULES---------------------------------------------------
16607 // These must follow all instruction definitions as they use the names
16608 // defined in the instructions definitions.
16609
16610 // Local Variables:
16611 // mode: c++
16612 // End: