1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BootTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else if (_entry_point == nullptr) {
1694 // See CallLeafNoFPIndirect
1695 return 1 * NativeInstruction::instruction_size;
1696 } else {
1697 return 6 * NativeInstruction::instruction_size;
1698 }
1699 }
1700
1701 //=============================================================================
1702
1703 #ifndef PRODUCT
1704 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1705 st->print("BREAKPOINT");
1706 }
1707 #endif
1708
1709 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1710 __ brk(0);
1711 }
1712
1713 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1714 return MachNode::size(ra_);
1715 }
1716
1717 //=============================================================================
1718
1719 #ifndef PRODUCT
1720 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1721 st->print("nop \t# %d bytes pad for loops and calls", _count);
1722 }
1723 #endif
1724
1725 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1726 for (int i = 0; i < _count; i++) {
1727 __ nop();
1728 }
1729 }
1730
1731 uint MachNopNode::size(PhaseRegAlloc*) const {
1732 return _count * NativeInstruction::instruction_size;
1733 }
1734
1735 //=============================================================================
1736 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1737
1738 int ConstantTable::calculate_table_base_offset() const {
1739 return 0; // absolute addressing, no offset
1740 }
1741
1742 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1743 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1744 ShouldNotReachHere();
1745 }
1746
1747 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1748 // Empty encoding
1749 }
1750
1751 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1752 return 0;
1753 }
1754
1755 #ifndef PRODUCT
1756 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1757 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1758 }
1759 #endif
1760
1761 #ifndef PRODUCT
1762 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1763 Compile* C = ra_->C;
1764
1765 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1766
1767 if (C->output()->need_stack_bang(framesize))
1768 st->print("# stack bang size=%d\n\t", framesize);
1769
1770 if (VM_Version::use_rop_protection()) {
1771 st->print("ldr zr, [lr]\n\t");
1772 st->print("paciaz\n\t");
1773 }
1774 if (framesize < ((1 << 9) + 2 * wordSize)) {
1775 st->print("sub sp, sp, #%d\n\t", framesize);
1776 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1777 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1778 } else {
1779 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1780 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1781 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1782 st->print("sub sp, sp, rscratch1");
1783 }
1784 if (C->stub_function() == nullptr) {
1785 st->print("\n\t");
1786 st->print("ldr rscratch1, [guard]\n\t");
1787 st->print("dmb ishld\n\t");
1788 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1789 st->print("cmp rscratch1, rscratch2\n\t");
1790 st->print("b.eq skip");
1791 st->print("\n\t");
1792 st->print("blr #nmethod_entry_barrier_stub\n\t");
1793 st->print("b skip\n\t");
1794 st->print("guard: int\n\t");
1795 st->print("\n\t");
1796 st->print("skip:\n\t");
1797 }
1798 }
1799 #endif
1800
1801 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1802 Compile* C = ra_->C;
1803
1804
1805 __ verified_entry(C, 0);
1806
1807 if (C->stub_function() == nullptr) {
1808 __ entry_barrier();
1809 }
1810
1811 if (!Compile::current()->output()->in_scratch_emit_size()) {
1812 __ bind(*_verified_entry);
1813 }
1814
1815 if (VerifyStackAtCalls) {
1816 Unimplemented();
1817 }
1818
1819 C->output()->set_frame_complete(__ offset());
1820
1821 if (C->has_mach_constant_base_node()) {
1822 // NOTE: We set the table base offset here because users might be
1823 // emitted before MachConstantBaseNode.
1824 ConstantTable& constant_table = C->output()->constant_table();
1825 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1826 }
1827 }
1828
1829 int MachPrologNode::reloc() const
1830 {
1831 return 0;
1832 }
1833
1834 //=============================================================================
1835
1836 #ifndef PRODUCT
1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1838 Compile* C = ra_->C;
1839 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1840
1841 st->print("# pop frame %d\n\t",framesize);
1842
1843 if (framesize == 0) {
1844 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1845 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1846 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1847 st->print("add sp, sp, #%d\n\t", framesize);
1848 } else {
1849 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1850 st->print("add sp, sp, rscratch1\n\t");
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 }
1853 if (VM_Version::use_rop_protection()) {
1854 st->print("autiaz\n\t");
1855 st->print("ldr zr, [lr]\n\t");
1856 }
1857
1858 if (do_polling() && C->is_method_compilation()) {
1859 st->print("# test polling word\n\t");
1860 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1861 st->print("cmp sp, rscratch1\n\t");
1862 st->print("bhi #slow_path");
1863 }
1864 }
1865 #endif
1866
1867 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1868 Compile* C = ra_->C;
1869 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1870
1871 __ remove_frame(framesize, C->needs_stack_repair());
1872
1873 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1874 __ reserved_stack_check();
1875 }
1876
1877 if (do_polling() && C->is_method_compilation()) {
1878 Label dummy_label;
1879 Label* code_stub = &dummy_label;
1880 if (!C->output()->in_scratch_emit_size()) {
1881 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1882 C->output()->add_stub(stub);
1883 code_stub = &stub->entry();
1884 }
1885 __ relocate(relocInfo::poll_return_type);
1886 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1887 }
1888 }
1889
1890 int MachEpilogNode::reloc() const {
1891 // Return number of relocatable values contained in this instruction.
1892 return 1; // 1 for polling page.
1893 }
1894
1895 const Pipeline * MachEpilogNode::pipeline() const {
1896 return MachNode::pipeline_class();
1897 }
1898
1899 //=============================================================================
1900
1901 static enum RC rc_class(OptoReg::Name reg) {
1902
1903 if (reg == OptoReg::Bad) {
1904 return rc_bad;
1905 }
1906
1907 // we have 32 int registers * 2 halves
1908 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1909
1910 if (reg < slots_of_int_registers) {
1911 return rc_int;
1912 }
1913
1914 // we have 32 float register * 8 halves
1915 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1916 if (reg < slots_of_int_registers + slots_of_float_registers) {
1917 return rc_float;
1918 }
1919
1920 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1921 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1922 return rc_predicate;
1923 }
1924
1925 // Between predicate regs & stack is the flags.
1926 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1927
1928 return rc_stack;
1929 }
1930
1931 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1932 Compile* C = ra_->C;
1933
1934 // Get registers to move.
1935 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1936 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1937 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1938 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1939
1940 enum RC src_hi_rc = rc_class(src_hi);
1941 enum RC src_lo_rc = rc_class(src_lo);
1942 enum RC dst_hi_rc = rc_class(dst_hi);
1943 enum RC dst_lo_rc = rc_class(dst_lo);
1944
1945 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1946
1947 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1948 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1949 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1950 "expected aligned-adjacent pairs");
1951 }
1952
1953 if (src_lo == dst_lo && src_hi == dst_hi) {
1954 return 0; // Self copy, no move.
1955 }
1956
1957 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1958 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1959 int src_offset = ra_->reg2offset(src_lo);
1960 int dst_offset = ra_->reg2offset(dst_lo);
1961
1962 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1963 uint ireg = ideal_reg();
1964 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
1965 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
1966 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
1967 if (ireg == Op_VecA && masm) {
1968 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1969 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1970 // stack->stack
1971 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1972 sve_vector_reg_size_in_bytes);
1973 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1974 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1975 sve_vector_reg_size_in_bytes);
1976 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1977 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1978 sve_vector_reg_size_in_bytes);
1979 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1980 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1981 as_FloatRegister(Matcher::_regEncode[src_lo]),
1982 as_FloatRegister(Matcher::_regEncode[src_lo]));
1983 } else {
1984 ShouldNotReachHere();
1985 }
1986 } else if (masm) {
1987 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1988 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1989 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1990 // stack->stack
1991 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1992 if (ireg == Op_VecD) {
1993 __ unspill(rscratch1, true, src_offset);
1994 __ spill(rscratch1, true, dst_offset);
1995 } else {
1996 __ spill_copy128(src_offset, dst_offset);
1997 }
1998 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1999 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2000 ireg == Op_VecD ? __ T8B : __ T16B,
2001 as_FloatRegister(Matcher::_regEncode[src_lo]));
2002 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2003 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2004 ireg == Op_VecD ? __ D : __ Q,
2005 ra_->reg2offset(dst_lo));
2006 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2007 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2008 ireg == Op_VecD ? __ D : __ Q,
2009 ra_->reg2offset(src_lo));
2010 } else {
2011 ShouldNotReachHere();
2012 }
2013 }
2014 } else if (masm) {
2015 switch (src_lo_rc) {
2016 case rc_int:
2017 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2018 if (is64) {
2019 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2020 as_Register(Matcher::_regEncode[src_lo]));
2021 } else {
2022 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2023 as_Register(Matcher::_regEncode[src_lo]));
2024 }
2025 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2026 if (is64) {
2027 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2028 as_Register(Matcher::_regEncode[src_lo]));
2029 } else {
2030 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2031 as_Register(Matcher::_regEncode[src_lo]));
2032 }
2033 } else { // gpr --> stack spill
2034 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2035 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2036 }
2037 break;
2038 case rc_float:
2039 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2040 if (is64) {
2041 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2042 as_FloatRegister(Matcher::_regEncode[src_lo]));
2043 } else {
2044 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2045 as_FloatRegister(Matcher::_regEncode[src_lo]));
2046 }
2047 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2048 if (is64) {
2049 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 as_FloatRegister(Matcher::_regEncode[src_lo]));
2051 } else {
2052 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2053 as_FloatRegister(Matcher::_regEncode[src_lo]));
2054 }
2055 } else { // fpr --> stack spill
2056 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2057 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2058 is64 ? __ D : __ S, dst_offset);
2059 }
2060 break;
2061 case rc_stack:
2062 if (dst_lo_rc == rc_int) { // stack --> gpr load
2063 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2064 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2065 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2066 is64 ? __ D : __ S, src_offset);
2067 } else if (dst_lo_rc == rc_predicate) {
2068 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2069 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2070 } else { // stack --> stack copy
2071 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2072 if (ideal_reg() == Op_RegVectMask) {
2073 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2074 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2075 } else {
2076 __ unspill(rscratch1, is64, src_offset);
2077 __ spill(rscratch1, is64, dst_offset);
2078 }
2079 }
2080 break;
2081 case rc_predicate:
2082 if (dst_lo_rc == rc_predicate) {
2083 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2084 } else if (dst_lo_rc == rc_stack) {
2085 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2086 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2087 } else {
2088 assert(false, "bad src and dst rc_class combination.");
2089 ShouldNotReachHere();
2090 }
2091 break;
2092 default:
2093 assert(false, "bad rc_class for spill");
2094 ShouldNotReachHere();
2095 }
2096 }
2097
2098 if (st) {
2099 st->print("spill ");
2100 if (src_lo_rc == rc_stack) {
2101 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2102 } else {
2103 st->print("%s -> ", Matcher::regName[src_lo]);
2104 }
2105 if (dst_lo_rc == rc_stack) {
2106 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2107 } else {
2108 st->print("%s", Matcher::regName[dst_lo]);
2109 }
2110 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2111 int vsize = 0;
2112 switch (ideal_reg()) {
2113 case Op_VecD:
2114 vsize = 64;
2115 break;
2116 case Op_VecX:
2117 vsize = 128;
2118 break;
2119 case Op_VecA:
2120 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2121 break;
2122 default:
2123 assert(false, "bad register type for spill");
2124 ShouldNotReachHere();
2125 }
2126 st->print("\t# vector spill size = %d", vsize);
2127 } else if (ideal_reg() == Op_RegVectMask) {
2128 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2129 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2130 st->print("\t# predicate spill size = %d", vsize);
2131 } else {
2132 st->print("\t# spill size = %d", is64 ? 64 : 32);
2133 }
2134 }
2135
2136 return 0;
2137
2138 }
2139
2140 #ifndef PRODUCT
2141 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2142 if (!ra_)
2143 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2144 else
2145 implementation(nullptr, ra_, false, st);
2146 }
2147 #endif
2148
2149 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2150 implementation(masm, ra_, false, nullptr);
2151 }
2152
2153 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2154 return MachNode::size(ra_);
2155 }
2156
2157 //=============================================================================
2158
2159 #ifndef PRODUCT
2160 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2161 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2162 int reg = ra_->get_reg_first(this);
2163 st->print("add %s, rsp, #%d]\t# box lock",
2164 Matcher::regName[reg], offset);
2165 }
2166 #endif
2167
2168 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2169 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2170 int reg = ra_->get_encode(this);
2171
2172 // This add will handle any 24-bit signed offset. 24 bits allows an
2173 // 8 megabyte stack frame.
2174 __ add(as_Register(reg), sp, offset);
2175 }
2176
2177 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2178 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2179 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2180
2181 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2182 return NativeInstruction::instruction_size;
2183 } else {
2184 return 2 * NativeInstruction::instruction_size;
2185 }
2186 }
2187
2188 ///=============================================================================
2189 #ifndef PRODUCT
2190 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2191 {
2192 st->print_cr("# MachVEPNode");
2193 if (!_verified) {
2194 st->print_cr("\t load_class");
2195 } else {
2196 st->print_cr("\t unpack_inline_arg");
2197 }
2198 }
2199 #endif
2200
2201 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2202 {
2203 if (!_verified) {
2204 __ ic_check(1);
2205 } else {
2206 // TODO 8284443 Avoid creation of temporary frame
2207 if (ra_->C->stub_function() == nullptr) {
2208 __ verified_entry(ra_->C, 0);
2209 __ entry_barrier();
2210 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2211 __ remove_frame(framesize, false);
2212 }
2213 // Unpack inline type args passed as oop and then jump to
2214 // the verified entry point (skipping the unverified entry).
2215 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2216 // Emit code for verified entry and save increment for stack repair on return
2217 __ verified_entry(ra_->C, sp_inc);
2218 if (Compile::current()->output()->in_scratch_emit_size()) {
2219 Label dummy_verified_entry;
2220 __ b(dummy_verified_entry);
2221 } else {
2222 __ b(*_verified_entry);
2223 }
2224 }
2225 }
2226
2227 //=============================================================================
2228 #ifndef PRODUCT
2229 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2230 {
2231 st->print_cr("# MachUEPNode");
2232 if (UseCompressedClassPointers) {
2233 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2234 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2235 st->print_cr("\tcmpw rscratch1, r10");
2236 } else {
2237 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2238 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2239 st->print_cr("\tcmp rscratch1, r10");
2240 }
2241 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2242 }
2243 #endif
2244
2245 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2246 {
2247 __ ic_check(InteriorEntryAlignment);
2248 }
2249
2250 // REQUIRED EMIT CODE
2251
2252 //=============================================================================
2253
2254 // Emit deopt handler code.
2255 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2256 {
2257 // Note that the code buffer's insts_mark is always relative to insts.
2258 // That's why we must use the macroassembler to generate a handler.
2259 address base = __ start_a_stub(size_deopt_handler());
2260 if (base == nullptr) {
2261 ciEnv::current()->record_failure("CodeCache is full");
2262 return 0; // CodeBuffer::expand failed
2263 }
2264
2265 int offset = __ offset();
2266 Label start;
2267 __ bind(start);
2268 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2269
2270 int entry_offset = __ offset();
2271 __ b(start);
2272
2273 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2274 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2275 "out of bounds read in post-call NOP check");
2276 __ end_a_stub();
2277 return entry_offset;
2278 }
2279
2280 // REQUIRED MATCHER CODE
2281
2282 //=============================================================================
2283
2284 bool Matcher::match_rule_supported(int opcode) {
2285 if (!has_match_rule(opcode))
2286 return false;
2287
2288 switch (opcode) {
2289 case Op_OnSpinWait:
2290 return VM_Version::supports_on_spin_wait();
2291 case Op_CacheWB:
2292 case Op_CacheWBPreSync:
2293 case Op_CacheWBPostSync:
2294 if (!VM_Version::supports_data_cache_line_flush()) {
2295 return false;
2296 }
2297 break;
2298 case Op_ExpandBits:
2299 case Op_CompressBits:
2300 if (!VM_Version::supports_svebitperm()) {
2301 return false;
2302 }
2303 break;
2304 case Op_FmaF:
2305 case Op_FmaD:
2306 case Op_FmaVF:
2307 case Op_FmaVD:
2308 if (!UseFMA) {
2309 return false;
2310 }
2311 break;
2312 case Op_FmaHF:
2313 // UseFMA flag also needs to be checked along with FEAT_FP16
2314 if (!UseFMA || !is_feat_fp16_supported()) {
2315 return false;
2316 }
2317 break;
2318 case Op_AddHF:
2319 case Op_SubHF:
2320 case Op_MulHF:
2321 case Op_DivHF:
2322 case Op_MinHF:
2323 case Op_MaxHF:
2324 case Op_SqrtHF:
2325 // Half-precision floating point scalar operations require FEAT_FP16
2326 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2327 // features are supported.
2328 if (!is_feat_fp16_supported()) {
2329 return false;
2330 }
2331 break;
2332 }
2333
2334 return true; // Per default match rules are supported.
2335 }
2336
2337 const RegMask* Matcher::predicate_reg_mask(void) {
2338 return &_PR_REG_mask;
2339 }
2340
2341 bool Matcher::supports_vector_calling_convention(void) {
2342 return EnableVectorSupport;
2343 }
2344
2345 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2346 assert(EnableVectorSupport, "sanity");
2347 int lo = V0_num;
2348 int hi = V0_H_num;
2349 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2350 hi = V0_K_num;
2351 }
2352 return OptoRegPair(hi, lo);
2353 }
2354
2355 // Is this branch offset short enough that a short branch can be used?
2356 //
2357 // NOTE: If the platform does not provide any short branch variants, then
2358 // this method should return false for offset 0.
2359 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2360 // The passed offset is relative to address of the branch.
2361
2362 return (-32768 <= offset && offset < 32768);
2363 }
2364
2365 // Vector width in bytes.
2366 int Matcher::vector_width_in_bytes(BasicType bt) {
2367 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2368 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2369 // Minimum 2 values in vector
2370 if (size < 2*type2aelembytes(bt)) size = 0;
2371 // But never < 4
2372 if (size < 4) size = 0;
2373 return size;
2374 }
2375
2376 // Limits on vector size (number of elements) loaded into vector.
2377 int Matcher::max_vector_size(const BasicType bt) {
2378 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2379 }
2380
2381 int Matcher::min_vector_size(const BasicType bt) {
2382 // Usually, the shortest vector length supported by AArch64 ISA and
2383 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2384 // vectors in a few special cases.
2385 int size;
2386 switch(bt) {
2387 case T_BOOLEAN:
2388 // Load/store a vector mask with only 2 elements for vector types
2389 // such as "2I/2F/2L/2D".
2390 size = 2;
2391 break;
2392 case T_BYTE:
2393 // Generate a "4B" vector, to support vector cast between "8B/16B"
2394 // and "4S/4I/4L/4F/4D".
2395 size = 4;
2396 break;
2397 case T_SHORT:
2398 // Generate a "2S" vector, to support vector cast between "4S/8S"
2399 // and "2I/2L/2F/2D".
2400 size = 2;
2401 break;
2402 default:
2403 // Limit the min vector length to 64-bit.
2404 size = 8 / type2aelembytes(bt);
2405 // The number of elements in a vector should be at least 2.
2406 size = MAX2(size, 2);
2407 }
2408
2409 int max_size = max_vector_size(bt);
2410 return MIN2(size, max_size);
2411 }
2412
2413 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2414 return Matcher::max_vector_size(bt);
2415 }
2416
2417 // Actual max scalable vector register length.
2418 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2419 return Matcher::max_vector_size(bt);
2420 }
2421
2422 // Vector ideal reg.
2423 uint Matcher::vector_ideal_reg(int len) {
2424 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2425 return Op_VecA;
2426 }
2427 switch(len) {
2428 // For 16-bit/32-bit mask vector, reuse VecD.
2429 case 2:
2430 case 4:
2431 case 8: return Op_VecD;
2432 case 16: return Op_VecX;
2433 }
2434 ShouldNotReachHere();
2435 return 0;
2436 }
2437
2438 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2439 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2440 switch (ideal_reg) {
2441 case Op_VecA: return new vecAOper();
2442 case Op_VecD: return new vecDOper();
2443 case Op_VecX: return new vecXOper();
2444 }
2445 ShouldNotReachHere();
2446 return nullptr;
2447 }
2448
2449 bool Matcher::is_reg2reg_move(MachNode* m) {
2450 return false;
2451 }
2452
2453 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2454 return false;
2455 }
2456
2457 bool Matcher::is_generic_vector(MachOper* opnd) {
2458 return opnd->opcode() == VREG;
2459 }
2460
2461 // Return whether or not this register is ever used as an argument.
2462 // This function is used on startup to build the trampoline stubs in
2463 // generateOptoStub. Registers not mentioned will be killed by the VM
2464 // call in the trampoline, and arguments in those registers not be
2465 // available to the callee.
2466 bool Matcher::can_be_java_arg(int reg)
2467 {
2468 return
2469 reg == R0_num || reg == R0_H_num ||
2470 reg == R1_num || reg == R1_H_num ||
2471 reg == R2_num || reg == R2_H_num ||
2472 reg == R3_num || reg == R3_H_num ||
2473 reg == R4_num || reg == R4_H_num ||
2474 reg == R5_num || reg == R5_H_num ||
2475 reg == R6_num || reg == R6_H_num ||
2476 reg == R7_num || reg == R7_H_num ||
2477 reg == V0_num || reg == V0_H_num ||
2478 reg == V1_num || reg == V1_H_num ||
2479 reg == V2_num || reg == V2_H_num ||
2480 reg == V3_num || reg == V3_H_num ||
2481 reg == V4_num || reg == V4_H_num ||
2482 reg == V5_num || reg == V5_H_num ||
2483 reg == V6_num || reg == V6_H_num ||
2484 reg == V7_num || reg == V7_H_num;
2485 }
2486
2487 bool Matcher::is_spillable_arg(int reg)
2488 {
2489 return can_be_java_arg(reg);
2490 }
2491
2492 uint Matcher::int_pressure_limit()
2493 {
2494 // JDK-8183543: When taking the number of available registers as int
2495 // register pressure threshold, the jtreg test:
2496 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2497 // failed due to C2 compilation failure with
2498 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2499 //
2500 // A derived pointer is live at CallNode and then is flagged by RA
2501 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2502 // derived pointers and lastly fail to spill after reaching maximum
2503 // number of iterations. Lowering the default pressure threshold to
2504 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2505 // a high register pressure area of the code so that split_DEF can
2506 // generate DefinitionSpillCopy for the derived pointer.
2507 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2508 if (!PreserveFramePointer) {
2509 // When PreserveFramePointer is off, frame pointer is allocatable,
2510 // but different from other SOC registers, it is excluded from
2511 // fatproj's mask because its save type is No-Save. Decrease 1 to
2512 // ensure high pressure at fatproj when PreserveFramePointer is off.
2513 // See check_pressure_at_fatproj().
2514 default_int_pressure_threshold--;
2515 }
2516 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2517 }
2518
2519 uint Matcher::float_pressure_limit()
2520 {
2521 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2522 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2523 }
2524
2525 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2526 return false;
2527 }
2528
2529 const RegMask& Matcher::divI_proj_mask() {
2530 ShouldNotReachHere();
2531 return RegMask::EMPTY;
2532 }
2533
2534 // Register for MODI projection of divmodI.
2535 const RegMask& Matcher::modI_proj_mask() {
2536 ShouldNotReachHere();
2537 return RegMask::EMPTY;
2538 }
2539
2540 // Register for DIVL projection of divmodL.
2541 const RegMask& Matcher::divL_proj_mask() {
2542 ShouldNotReachHere();
2543 return RegMask::EMPTY;
2544 }
2545
2546 // Register for MODL projection of divmodL.
2547 const RegMask& Matcher::modL_proj_mask() {
2548 ShouldNotReachHere();
2549 return RegMask::EMPTY;
2550 }
2551
2552 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2553 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2554 Node* u = addp->fast_out(i);
2555 if (u->is_LoadStore()) {
2556 // On AArch64, LoadStoreNodes (i.e. compare and swap
2557 // instructions) only take register indirect as an operand, so
2558 // any attempt to use an AddPNode as an input to a LoadStoreNode
2559 // must fail.
2560 return false;
2561 }
2562 if (u->is_Mem()) {
2563 int opsize = u->as_Mem()->memory_size();
2564 assert(opsize > 0, "unexpected memory operand size");
2565 if (u->as_Mem()->memory_size() != (1<<shift)) {
2566 return false;
2567 }
2568 }
2569 }
2570 return true;
2571 }
2572
2573 // Convert BootTest condition to Assembler condition.
2574 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2575 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2576 Assembler::Condition result;
2577 switch(cond) {
2578 case BoolTest::eq:
2579 result = Assembler::EQ; break;
2580 case BoolTest::ne:
2581 result = Assembler::NE; break;
2582 case BoolTest::le:
2583 result = Assembler::LE; break;
2584 case BoolTest::ge:
2585 result = Assembler::GE; break;
2586 case BoolTest::lt:
2587 result = Assembler::LT; break;
2588 case BoolTest::gt:
2589 result = Assembler::GT; break;
2590 case BoolTest::ule:
2591 result = Assembler::LS; break;
2592 case BoolTest::uge:
2593 result = Assembler::HS; break;
2594 case BoolTest::ult:
2595 result = Assembler::LO; break;
2596 case BoolTest::ugt:
2597 result = Assembler::HI; break;
2598 case BoolTest::overflow:
2599 result = Assembler::VS; break;
2600 case BoolTest::no_overflow:
2601 result = Assembler::VC; break;
2602 default:
2603 ShouldNotReachHere();
2604 return Assembler::Condition(-1);
2605 }
2606
2607 // Check conversion
2608 if (cond & BoolTest::unsigned_compare) {
2609 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2610 } else {
2611 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2612 }
2613
2614 return result;
2615 }
2616
2617 // Binary src (Replicate con)
2618 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2619 if (n == nullptr || m == nullptr) {
2620 return false;
2621 }
2622
2623 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2624 return false;
2625 }
2626
2627 Node* imm_node = m->in(1);
2628 if (!imm_node->is_Con()) {
2629 return false;
2630 }
2631
2632 const Type* t = imm_node->bottom_type();
2633 if (!(t->isa_int() || t->isa_long())) {
2634 return false;
2635 }
2636
2637 switch (n->Opcode()) {
2638 case Op_AndV:
2639 case Op_OrV:
2640 case Op_XorV: {
2641 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2642 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2643 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2644 }
2645 case Op_AddVB:
2646 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2647 case Op_AddVS:
2648 case Op_AddVI:
2649 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2650 case Op_AddVL:
2651 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2652 default:
2653 return false;
2654 }
2655 }
2656
2657 // (XorV src (Replicate m1))
2658 // (XorVMask src (MaskAll m1))
2659 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2660 if (n != nullptr && m != nullptr) {
2661 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2662 VectorNode::is_all_ones_vector(m);
2663 }
2664 return false;
2665 }
2666
2667 // Should the matcher clone input 'm' of node 'n'?
2668 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2669 if (is_vshift_con_pattern(n, m) ||
2670 is_vector_bitwise_not_pattern(n, m) ||
2671 is_valid_sve_arith_imm_pattern(n, m) ||
2672 is_encode_and_store_pattern(n, m)) {
2673 mstack.push(m, Visit);
2674 return true;
2675 }
2676 return false;
2677 }
2678
2679 // Should the Matcher clone shifts on addressing modes, expecting them
2680 // to be subsumed into complex addressing expressions or compute them
2681 // into registers?
2682 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2683
2684 // Loads and stores with indirect memory input (e.g., volatile loads and
2685 // stores) do not subsume the input into complex addressing expressions. If
2686 // the addressing expression is input to at least one such load or store, do
2687 // not clone the addressing expression. Query needs_acquiring_load and
2688 // needs_releasing_store as a proxy for indirect memory input, as it is not
2689 // possible to directly query for indirect memory input at this stage.
2690 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2691 Node* n = m->fast_out(i);
2692 if (n->is_Load() && needs_acquiring_load(n)) {
2693 return false;
2694 }
2695 if (n->is_Store() && needs_releasing_store(n)) {
2696 return false;
2697 }
2698 }
2699
2700 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2701 return true;
2702 }
2703
2704 Node *off = m->in(AddPNode::Offset);
2705 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2706 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2707 // Are there other uses besides address expressions?
2708 !is_visited(off)) {
2709 address_visited.set(off->_idx); // Flag as address_visited
2710 mstack.push(off->in(2), Visit);
2711 Node *conv = off->in(1);
2712 if (conv->Opcode() == Op_ConvI2L &&
2713 // Are there other uses besides address expressions?
2714 !is_visited(conv)) {
2715 address_visited.set(conv->_idx); // Flag as address_visited
2716 mstack.push(conv->in(1), Pre_Visit);
2717 } else {
2718 mstack.push(conv, Pre_Visit);
2719 }
2720 address_visited.test_set(m->_idx); // Flag as address_visited
2721 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2722 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2723 return true;
2724 } else if (off->Opcode() == Op_ConvI2L &&
2725 // Are there other uses besides address expressions?
2726 !is_visited(off)) {
2727 address_visited.test_set(m->_idx); // Flag as address_visited
2728 address_visited.set(off->_idx); // Flag as address_visited
2729 mstack.push(off->in(1), Pre_Visit);
2730 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2731 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2732 return true;
2733 }
2734 return false;
2735 }
2736
2737 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2738 { \
2739 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2740 guarantee(DISP == 0, "mode not permitted for volatile"); \
2741 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2742 __ INSN(REG, as_Register(BASE)); \
2743 }
2744
2745
2746 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2747 {
2748 Address::extend scale;
2749
2750 // Hooboy, this is fugly. We need a way to communicate to the
2751 // encoder that the index needs to be sign extended, so we have to
2752 // enumerate all the cases.
2753 switch (opcode) {
2754 case INDINDEXSCALEDI2L:
2755 case INDINDEXSCALEDI2LN:
2756 case INDINDEXI2L:
2757 case INDINDEXI2LN:
2758 scale = Address::sxtw(size);
2759 break;
2760 default:
2761 scale = Address::lsl(size);
2762 }
2763
2764 if (index == -1) {
2765 return Address(base, disp);
2766 } else {
2767 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2768 return Address(base, as_Register(index), scale);
2769 }
2770 }
2771
2772
2773 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2774 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2775 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2776 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2777 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2778
2779 // Used for all non-volatile memory accesses. The use of
2780 // $mem->opcode() to discover whether this pattern uses sign-extended
2781 // offsets is something of a kludge.
2782 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2783 Register reg, int opcode,
2784 Register base, int index, int scale, int disp,
2785 int size_in_memory)
2786 {
2787 Address addr = mem2address(opcode, base, index, scale, disp);
2788 if (addr.getMode() == Address::base_plus_offset) {
2789 /* Fix up any out-of-range offsets. */
2790 assert_different_registers(rscratch1, base);
2791 assert_different_registers(rscratch1, reg);
2792 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2793 }
2794 (masm->*insn)(reg, addr);
2795 }
2796
2797 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2798 FloatRegister reg, int opcode,
2799 Register base, int index, int size, int disp,
2800 int size_in_memory)
2801 {
2802 Address::extend scale;
2803
2804 switch (opcode) {
2805 case INDINDEXSCALEDI2L:
2806 case INDINDEXSCALEDI2LN:
2807 scale = Address::sxtw(size);
2808 break;
2809 default:
2810 scale = Address::lsl(size);
2811 }
2812
2813 if (index == -1) {
2814 // Fix up any out-of-range offsets.
2815 assert_different_registers(rscratch1, base);
2816 Address addr = Address(base, disp);
2817 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2818 (masm->*insn)(reg, addr);
2819 } else {
2820 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2821 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2822 }
2823 }
2824
2825 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2826 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2827 int opcode, Register base, int index, int size, int disp)
2828 {
2829 if (index == -1) {
2830 (masm->*insn)(reg, T, Address(base, disp));
2831 } else {
2832 assert(disp == 0, "unsupported address mode");
2833 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2834 }
2835 }
2836
2837 %}
2838
2839
2840
2841 //----------ENCODING BLOCK-----------------------------------------------------
2842 // This block specifies the encoding classes used by the compiler to
2843 // output byte streams. Encoding classes are parameterized macros
2844 // used by Machine Instruction Nodes in order to generate the bit
2845 // encoding of the instruction. Operands specify their base encoding
2846 // interface with the interface keyword. There are currently
2847 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2848 // COND_INTER. REG_INTER causes an operand to generate a function
2849 // which returns its register number when queried. CONST_INTER causes
2850 // an operand to generate a function which returns the value of the
2851 // constant when queried. MEMORY_INTER causes an operand to generate
2852 // four functions which return the Base Register, the Index Register,
2853 // the Scale Value, and the Offset Value of the operand when queried.
2854 // COND_INTER causes an operand to generate six functions which return
2855 // the encoding code (ie - encoding bits for the instruction)
2856 // associated with each basic boolean condition for a conditional
2857 // instruction.
2858 //
2859 // Instructions specify two basic values for encoding. Again, a
2860 // function is available to check if the constant displacement is an
2861 // oop. They use the ins_encode keyword to specify their encoding
2862 // classes (which must be a sequence of enc_class names, and their
2863 // parameters, specified in the encoding block), and they use the
2864 // opcode keyword to specify, in order, their primary, secondary, and
2865 // tertiary opcode. Only the opcode sections which a particular
2866 // instruction needs for encoding need to be specified.
2867 encode %{
2868 // Build emit functions for each basic byte or larger field in the
2869 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2870 // from C++ code in the enc_class source block. Emit functions will
2871 // live in the main source block for now. In future, we can
2872 // generalize this by adding a syntax that specifies the sizes of
2873 // fields in an order, so that the adlc can build the emit functions
2874 // automagically
2875
2876 // catch all for unimplemented encodings
2877 enc_class enc_unimplemented %{
2878 __ unimplemented("C2 catch all");
2879 %}
2880
2881 // BEGIN Non-volatile memory access
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2934 Register dst_reg = as_Register($dst$$reg);
2935 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2942 Register dst_reg = as_Register($dst$$reg);
2943 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2950 Register dst_reg = as_Register($dst$$reg);
2951 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2958 Register dst_reg = as_Register($dst$$reg);
2959 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2960 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2961 %}
2962
2963 // This encoding class is generated automatically from ad_encode.m4.
2964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2965 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2966 Register dst_reg = as_Register($dst$$reg);
2967 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2968 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2969 %}
2970
2971 // This encoding class is generated automatically from ad_encode.m4.
2972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2973 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2974 Register dst_reg = as_Register($dst$$reg);
2975 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2976 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2977 %}
2978
2979 // This encoding class is generated automatically from ad_encode.m4.
2980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2981 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2982 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2983 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2984 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2985 %}
2986
2987 // This encoding class is generated automatically from ad_encode.m4.
2988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2989 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2990 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2991 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2998 Register src_reg = as_Register($src$$reg);
2999 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3000 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3001 %}
3002
3003 // This encoding class is generated automatically from ad_encode.m4.
3004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3005 enc_class aarch64_enc_strb0(memory1 mem) %{
3006 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3013 Register src_reg = as_Register($src$$reg);
3014 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3015 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3016 %}
3017
3018 // This encoding class is generated automatically from ad_encode.m4.
3019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3020 enc_class aarch64_enc_strh0(memory2 mem) %{
3021 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3022 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3023 %}
3024
3025 // This encoding class is generated automatically from ad_encode.m4.
3026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3027 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3028 Register src_reg = as_Register($src$$reg);
3029 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3030 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3031 %}
3032
3033 // This encoding class is generated automatically from ad_encode.m4.
3034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3035 enc_class aarch64_enc_strw0(memory4 mem) %{
3036 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3037 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3038 %}
3039
3040 // This encoding class is generated automatically from ad_encode.m4.
3041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3042 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3043 Register src_reg = as_Register($src$$reg);
3044 // we sometimes get asked to store the stack pointer into the
3045 // current thread -- we cannot do that directly on AArch64
3046 if (src_reg == r31_sp) {
3047 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3048 __ mov(rscratch2, sp);
3049 src_reg = rscratch2;
3050 }
3051 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3052 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3053 %}
3054
3055 // This encoding class is generated automatically from ad_encode.m4.
3056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3057 enc_class aarch64_enc_str0(memory8 mem) %{
3058 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3059 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3060 %}
3061
3062 // This encoding class is generated automatically from ad_encode.m4.
3063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3064 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3065 FloatRegister src_reg = as_FloatRegister($src$$reg);
3066 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3067 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3068 %}
3069
3070 // This encoding class is generated automatically from ad_encode.m4.
3071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3072 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3073 FloatRegister src_reg = as_FloatRegister($src$$reg);
3074 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3075 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3076 %}
3077
3078 // This encoding class is generated automatically from ad_encode.m4.
3079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3080 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3081 __ membar(Assembler::StoreStore);
3082 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3083 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3084 %}
3085
3086 // END Non-volatile memory access
3087
3088 // Vector loads and stores
3089 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3090 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3091 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3092 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3093 %}
3094
3095 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3096 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3097 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3098 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3099 %}
3100
3101 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3102 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3103 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3104 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3105 %}
3106
3107 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3108 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3109 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3110 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3111 %}
3112
3113 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3114 FloatRegister src_reg = as_FloatRegister($src$$reg);
3115 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3116 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3117 %}
3118
3119 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3120 FloatRegister src_reg = as_FloatRegister($src$$reg);
3121 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3122 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3123 %}
3124
3125 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3126 FloatRegister src_reg = as_FloatRegister($src$$reg);
3127 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3128 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3129 %}
3130
3131 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3132 FloatRegister src_reg = as_FloatRegister($src$$reg);
3133 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3134 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3135 %}
3136
3137 // volatile loads and stores
3138
3139 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3140 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3141 rscratch1, stlrb);
3142 %}
3143
3144 enc_class aarch64_enc_stlrb0(memory mem) %{
3145 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3146 rscratch1, stlrb);
3147 %}
3148
3149 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3150 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3151 rscratch1, stlrh);
3152 %}
3153
3154 enc_class aarch64_enc_stlrh0(memory mem) %{
3155 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3156 rscratch1, stlrh);
3157 %}
3158
3159 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3160 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3161 rscratch1, stlrw);
3162 %}
3163
3164 enc_class aarch64_enc_stlrw0(memory mem) %{
3165 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, stlrw);
3167 %}
3168
3169 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3170 Register dst_reg = as_Register($dst$$reg);
3171 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3172 rscratch1, ldarb);
3173 __ sxtbw(dst_reg, dst_reg);
3174 %}
3175
3176 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3177 Register dst_reg = as_Register($dst$$reg);
3178 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3179 rscratch1, ldarb);
3180 __ sxtb(dst_reg, dst_reg);
3181 %}
3182
3183 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3184 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3185 rscratch1, ldarb);
3186 %}
3187
3188 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3189 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3190 rscratch1, ldarb);
3191 %}
3192
3193 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3194 Register dst_reg = as_Register($dst$$reg);
3195 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3196 rscratch1, ldarh);
3197 __ sxthw(dst_reg, dst_reg);
3198 %}
3199
3200 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3201 Register dst_reg = as_Register($dst$$reg);
3202 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3203 rscratch1, ldarh);
3204 __ sxth(dst_reg, dst_reg);
3205 %}
3206
3207 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3208 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3209 rscratch1, ldarh);
3210 %}
3211
3212 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3213 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3214 rscratch1, ldarh);
3215 %}
3216
3217 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3218 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3219 rscratch1, ldarw);
3220 %}
3221
3222 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3223 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3224 rscratch1, ldarw);
3225 %}
3226
3227 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3228 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3229 rscratch1, ldar);
3230 %}
3231
3232 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3233 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3234 rscratch1, ldarw);
3235 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3236 %}
3237
3238 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3239 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3240 rscratch1, ldar);
3241 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3242 %}
3243
3244 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3245 Register src_reg = as_Register($src$$reg);
3246 // we sometimes get asked to store the stack pointer into the
3247 // current thread -- we cannot do that directly on AArch64
3248 if (src_reg == r31_sp) {
3249 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3250 __ mov(rscratch2, sp);
3251 src_reg = rscratch2;
3252 }
3253 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3254 rscratch1, stlr);
3255 %}
3256
3257 enc_class aarch64_enc_stlr0(memory mem) %{
3258 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3259 rscratch1, stlr);
3260 %}
3261
3262 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3263 {
3264 FloatRegister src_reg = as_FloatRegister($src$$reg);
3265 __ fmovs(rscratch2, src_reg);
3266 }
3267 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3268 rscratch1, stlrw);
3269 %}
3270
3271 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3272 {
3273 FloatRegister src_reg = as_FloatRegister($src$$reg);
3274 __ fmovd(rscratch2, src_reg);
3275 }
3276 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3277 rscratch1, stlr);
3278 %}
3279
3280 // synchronized read/update encodings
3281
3282 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3283 Register dst_reg = as_Register($dst$$reg);
3284 Register base = as_Register($mem$$base);
3285 int index = $mem$$index;
3286 int scale = $mem$$scale;
3287 int disp = $mem$$disp;
3288 if (index == -1) {
3289 if (disp != 0) {
3290 __ lea(rscratch1, Address(base, disp));
3291 __ ldaxr(dst_reg, rscratch1);
3292 } else {
3293 // TODO
3294 // should we ever get anything other than this case?
3295 __ ldaxr(dst_reg, base);
3296 }
3297 } else {
3298 Register index_reg = as_Register(index);
3299 if (disp == 0) {
3300 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3301 __ ldaxr(dst_reg, rscratch1);
3302 } else {
3303 __ lea(rscratch1, Address(base, disp));
3304 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3305 __ ldaxr(dst_reg, rscratch1);
3306 }
3307 }
3308 %}
3309
3310 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3311 Register src_reg = as_Register($src$$reg);
3312 Register base = as_Register($mem$$base);
3313 int index = $mem$$index;
3314 int scale = $mem$$scale;
3315 int disp = $mem$$disp;
3316 if (index == -1) {
3317 if (disp != 0) {
3318 __ lea(rscratch2, Address(base, disp));
3319 __ stlxr(rscratch1, src_reg, rscratch2);
3320 } else {
3321 // TODO
3322 // should we ever get anything other than this case?
3323 __ stlxr(rscratch1, src_reg, base);
3324 }
3325 } else {
3326 Register index_reg = as_Register(index);
3327 if (disp == 0) {
3328 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3329 __ stlxr(rscratch1, src_reg, rscratch2);
3330 } else {
3331 __ lea(rscratch2, Address(base, disp));
3332 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3333 __ stlxr(rscratch1, src_reg, rscratch2);
3334 }
3335 }
3336 __ cmpw(rscratch1, zr);
3337 %}
3338
3339 // prefetch encodings
3340
3341 enc_class aarch64_enc_prefetchw(memory mem) %{
3342 Register base = as_Register($mem$$base);
3343 int index = $mem$$index;
3344 int scale = $mem$$scale;
3345 int disp = $mem$$disp;
3346 if (index == -1) {
3347 // Fix up any out-of-range offsets.
3348 assert_different_registers(rscratch1, base);
3349 Address addr = Address(base, disp);
3350 addr = __ legitimize_address(addr, 8, rscratch1);
3351 __ prfm(addr, PSTL1KEEP);
3352 } else {
3353 Register index_reg = as_Register(index);
3354 if (disp == 0) {
3355 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3356 } else {
3357 __ lea(rscratch1, Address(base, disp));
3358 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3359 }
3360 }
3361 %}
3362
3363 // mov encodings
3364
3365 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3366 uint32_t con = (uint32_t)$src$$constant;
3367 Register dst_reg = as_Register($dst$$reg);
3368 if (con == 0) {
3369 __ movw(dst_reg, zr);
3370 } else {
3371 __ movw(dst_reg, con);
3372 }
3373 %}
3374
3375 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3376 Register dst_reg = as_Register($dst$$reg);
3377 uint64_t con = (uint64_t)$src$$constant;
3378 if (con == 0) {
3379 __ mov(dst_reg, zr);
3380 } else {
3381 __ mov(dst_reg, con);
3382 }
3383 %}
3384
3385 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3386 Register dst_reg = as_Register($dst$$reg);
3387 address con = (address)$src$$constant;
3388 if (con == nullptr || con == (address)1) {
3389 ShouldNotReachHere();
3390 } else {
3391 relocInfo::relocType rtype = $src->constant_reloc();
3392 if (rtype == relocInfo::oop_type) {
3393 __ movoop(dst_reg, (jobject)con);
3394 } else if (rtype == relocInfo::metadata_type) {
3395 __ mov_metadata(dst_reg, (Metadata*)con);
3396 } else {
3397 assert(rtype == relocInfo::none, "unexpected reloc type");
3398 if (! __ is_valid_AArch64_address(con) ||
3399 con < (address)(uintptr_t)os::vm_page_size()) {
3400 __ mov(dst_reg, con);
3401 } else {
3402 uint64_t offset;
3403 __ adrp(dst_reg, con, offset);
3404 __ add(dst_reg, dst_reg, offset);
3405 }
3406 }
3407 }
3408 %}
3409
3410 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3411 Register dst_reg = as_Register($dst$$reg);
3412 __ mov(dst_reg, zr);
3413 %}
3414
3415 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3416 Register dst_reg = as_Register($dst$$reg);
3417 __ mov(dst_reg, (uint64_t)1);
3418 %}
3419
3420 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3421 Register dst_reg = as_Register($dst$$reg);
3422 address con = (address)$src$$constant;
3423 if (con == nullptr) {
3424 ShouldNotReachHere();
3425 } else {
3426 relocInfo::relocType rtype = $src->constant_reloc();
3427 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3428 __ set_narrow_oop(dst_reg, (jobject)con);
3429 }
3430 %}
3431
3432 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3433 Register dst_reg = as_Register($dst$$reg);
3434 __ mov(dst_reg, zr);
3435 %}
3436
3437 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3438 Register dst_reg = as_Register($dst$$reg);
3439 address con = (address)$src$$constant;
3440 if (con == nullptr) {
3441 ShouldNotReachHere();
3442 } else {
3443 relocInfo::relocType rtype = $src->constant_reloc();
3444 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3445 __ set_narrow_klass(dst_reg, (Klass *)con);
3446 }
3447 %}
3448
3449 // arithmetic encodings
3450
3451 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3452 Register dst_reg = as_Register($dst$$reg);
3453 Register src_reg = as_Register($src1$$reg);
3454 int32_t con = (int32_t)$src2$$constant;
3455 // add has primary == 0, subtract has primary == 1
3456 if ($primary) { con = -con; }
3457 if (con < 0) {
3458 __ subw(dst_reg, src_reg, -con);
3459 } else {
3460 __ addw(dst_reg, src_reg, con);
3461 }
3462 %}
3463
3464 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3465 Register dst_reg = as_Register($dst$$reg);
3466 Register src_reg = as_Register($src1$$reg);
3467 int32_t con = (int32_t)$src2$$constant;
3468 // add has primary == 0, subtract has primary == 1
3469 if ($primary) { con = -con; }
3470 if (con < 0) {
3471 __ sub(dst_reg, src_reg, -con);
3472 } else {
3473 __ add(dst_reg, src_reg, con);
3474 }
3475 %}
3476
3477 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3478 Register dst_reg = as_Register($dst$$reg);
3479 Register src1_reg = as_Register($src1$$reg);
3480 Register src2_reg = as_Register($src2$$reg);
3481 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3482 %}
3483
3484 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3485 Register dst_reg = as_Register($dst$$reg);
3486 Register src1_reg = as_Register($src1$$reg);
3487 Register src2_reg = as_Register($src2$$reg);
3488 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3489 %}
3490
3491 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3492 Register dst_reg = as_Register($dst$$reg);
3493 Register src1_reg = as_Register($src1$$reg);
3494 Register src2_reg = as_Register($src2$$reg);
3495 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3496 %}
3497
3498 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3499 Register dst_reg = as_Register($dst$$reg);
3500 Register src1_reg = as_Register($src1$$reg);
3501 Register src2_reg = as_Register($src2$$reg);
3502 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3503 %}
3504
3505 // compare instruction encodings
3506
3507 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3508 Register reg1 = as_Register($src1$$reg);
3509 Register reg2 = as_Register($src2$$reg);
3510 __ cmpw(reg1, reg2);
3511 %}
3512
3513 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3514 Register reg = as_Register($src1$$reg);
3515 int32_t val = $src2$$constant;
3516 if (val >= 0) {
3517 __ subsw(zr, reg, val);
3518 } else {
3519 __ addsw(zr, reg, -val);
3520 }
3521 %}
3522
3523 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3524 Register reg1 = as_Register($src1$$reg);
3525 uint32_t val = (uint32_t)$src2$$constant;
3526 __ movw(rscratch1, val);
3527 __ cmpw(reg1, rscratch1);
3528 %}
3529
3530 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3531 Register reg1 = as_Register($src1$$reg);
3532 Register reg2 = as_Register($src2$$reg);
3533 __ cmp(reg1, reg2);
3534 %}
3535
3536 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3537 Register reg = as_Register($src1$$reg);
3538 int64_t val = $src2$$constant;
3539 if (val >= 0) {
3540 __ subs(zr, reg, val);
3541 } else if (val != -val) {
3542 __ adds(zr, reg, -val);
3543 } else {
3544 // aargh, Long.MIN_VALUE is a special case
3545 __ orr(rscratch1, zr, (uint64_t)val);
3546 __ subs(zr, reg, rscratch1);
3547 }
3548 %}
3549
3550 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3551 Register reg1 = as_Register($src1$$reg);
3552 uint64_t val = (uint64_t)$src2$$constant;
3553 __ mov(rscratch1, val);
3554 __ cmp(reg1, rscratch1);
3555 %}
3556
3557 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3558 Register reg1 = as_Register($src1$$reg);
3559 Register reg2 = as_Register($src2$$reg);
3560 __ cmp(reg1, reg2);
3561 %}
3562
3563 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3564 Register reg1 = as_Register($src1$$reg);
3565 Register reg2 = as_Register($src2$$reg);
3566 __ cmpw(reg1, reg2);
3567 %}
3568
3569 enc_class aarch64_enc_testp(iRegP src) %{
3570 Register reg = as_Register($src$$reg);
3571 __ cmp(reg, zr);
3572 %}
3573
3574 enc_class aarch64_enc_testn(iRegN src) %{
3575 Register reg = as_Register($src$$reg);
3576 __ cmpw(reg, zr);
3577 %}
3578
3579 enc_class aarch64_enc_b(label lbl) %{
3580 Label *L = $lbl$$label;
3581 __ b(*L);
3582 %}
3583
3584 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3585 Label *L = $lbl$$label;
3586 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3587 %}
3588
3589 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3590 Label *L = $lbl$$label;
3591 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3592 %}
3593
3594 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3595 %{
3596 Register sub_reg = as_Register($sub$$reg);
3597 Register super_reg = as_Register($super$$reg);
3598 Register temp_reg = as_Register($temp$$reg);
3599 Register result_reg = as_Register($result$$reg);
3600
3601 Label miss;
3602 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3603 nullptr, &miss,
3604 /*set_cond_codes:*/ true);
3605 if ($primary) {
3606 __ mov(result_reg, zr);
3607 }
3608 __ bind(miss);
3609 %}
3610
3611 enc_class aarch64_enc_java_static_call(method meth) %{
3612 address addr = (address)$meth$$method;
3613 address call;
3614 if (!_method) {
3615 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3616 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3617 if (call == nullptr) {
3618 ciEnv::current()->record_failure("CodeCache is full");
3619 return;
3620 }
3621 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3622 // The NOP here is purely to ensure that eliding a call to
3623 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3624 __ nop();
3625 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3626 } else {
3627 int method_index = resolved_method_index(masm);
3628 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3629 : static_call_Relocation::spec(method_index);
3630 call = __ trampoline_call(Address(addr, rspec));
3631 if (call == nullptr) {
3632 ciEnv::current()->record_failure("CodeCache is full");
3633 return;
3634 }
3635 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3636 // Calls of the same statically bound method can share
3637 // a stub to the interpreter.
3638 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3639 } else {
3640 // Emit stub for static call
3641 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3642 if (stub == nullptr) {
3643 ciEnv::current()->record_failure("CodeCache is full");
3644 return;
3645 }
3646 }
3647 }
3648
3649 __ post_call_nop();
3650
3651 // Only non uncommon_trap calls need to reinitialize ptrue.
3652 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3653 __ reinitialize_ptrue();
3654 }
3655 %}
3656
3657 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3658 int method_index = resolved_method_index(masm);
3659 address call = __ ic_call((address)$meth$$method, method_index);
3660 if (call == nullptr) {
3661 ciEnv::current()->record_failure("CodeCache is full");
3662 return;
3663 }
3664 __ post_call_nop();
3665 if (Compile::current()->max_vector_size() > 0) {
3666 __ reinitialize_ptrue();
3667 }
3668 %}
3669
3670 enc_class aarch64_enc_call_epilog() %{
3671 if (VerifyStackAtCalls) {
3672 // Check that stack depth is unchanged: find majik cookie on stack
3673 __ call_Unimplemented();
3674 }
3675 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
3676 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3677 // Search for the corresponding projection, get the register and emit code that initialized it.
3678 uint con = (tf()->range_cc()->cnt() - 1);
3679 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3680 ProjNode* proj = fast_out(i)->as_Proj();
3681 if (proj->_con == con) {
3682 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3683 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3684 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3685 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3686 __ cmp(r0, zr);
3687 __ cset(toReg, Assembler::NE);
3688 if (reg->is_stack()) {
3689 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3690 __ str(toReg, Address(sp, st_off));
3691 }
3692 break;
3693 }
3694 }
3695 if (return_value_is_used()) {
3696 // An inline type is returned as fields in multiple registers.
3697 // R0 either contains an oop if the inline type is buffered or a pointer
3698 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3699 // if the lowest bit is set to allow C2 to use the oop after null checking.
3700 // r0 &= (r0 & 1) - 1
3701 __ andr(rscratch1, r0, 0x1);
3702 __ sub(rscratch1, rscratch1, 0x1);
3703 __ andr(r0, r0, rscratch1);
3704 }
3705 }
3706 %}
3707
3708 enc_class aarch64_enc_java_to_runtime(method meth) %{
3709 // some calls to generated routines (arraycopy code) are scheduled
3710 // by C2 as runtime calls. if so we can call them using a br (they
3711 // will be in a reachable segment) otherwise we have to use a blr
3712 // which loads the absolute address into a register.
3713 address entry = (address)$meth$$method;
3714 CodeBlob *cb = CodeCache::find_blob(entry);
3715 if (cb) {
3716 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3717 if (call == nullptr) {
3718 ciEnv::current()->record_failure("CodeCache is full");
3719 return;
3720 }
3721 __ post_call_nop();
3722 } else {
3723 Label retaddr;
3724 // Make the anchor frame walkable
3725 __ adr(rscratch2, retaddr);
3726 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3727 __ lea(rscratch1, RuntimeAddress(entry));
3728 __ blr(rscratch1);
3729 __ bind(retaddr);
3730 __ post_call_nop();
3731 }
3732 if (Compile::current()->max_vector_size() > 0) {
3733 __ reinitialize_ptrue();
3734 }
3735 %}
3736
3737 enc_class aarch64_enc_rethrow() %{
3738 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3739 %}
3740
3741 enc_class aarch64_enc_ret() %{
3742 #ifdef ASSERT
3743 if (Compile::current()->max_vector_size() > 0) {
3744 __ verify_ptrue();
3745 }
3746 #endif
3747 __ ret(lr);
3748 %}
3749
3750 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3751 Register target_reg = as_Register($jump_target$$reg);
3752 __ br(target_reg);
3753 %}
3754
3755 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3756 Register target_reg = as_Register($jump_target$$reg);
3757 // exception oop should be in r0
3758 // ret addr has been popped into lr
3759 // callee expects it in r3
3760 __ mov(r3, lr);
3761 __ br(target_reg);
3762 %}
3763
3764 %}
3765
3766 //----------FRAME--------------------------------------------------------------
3767 // Definition of frame structure and management information.
3768 //
3769 // S T A C K L A Y O U T Allocators stack-slot number
3770 // | (to get allocators register number
3771 // G Owned by | | v add OptoReg::stack0())
3772 // r CALLER | |
3773 // o | +--------+ pad to even-align allocators stack-slot
3774 // w V | pad0 | numbers; owned by CALLER
3775 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3776 // h ^ | in | 5
3777 // | | args | 4 Holes in incoming args owned by SELF
3778 // | | | | 3
3779 // | | +--------+
3780 // V | | old out| Empty on Intel, window on Sparc
3781 // | old |preserve| Must be even aligned.
3782 // | SP-+--------+----> Matcher::_old_SP, even aligned
3783 // | | in | 3 area for Intel ret address
3784 // Owned by |preserve| Empty on Sparc.
3785 // SELF +--------+
3786 // | | pad2 | 2 pad to align old SP
3787 // | +--------+ 1
3788 // | | locks | 0
3789 // | +--------+----> OptoReg::stack0(), even aligned
3790 // | | pad1 | 11 pad to align new SP
3791 // | +--------+
3792 // | | | 10
3793 // | | spills | 9 spills
3794 // V | | 8 (pad0 slot for callee)
3795 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3796 // ^ | out | 7
3797 // | | args | 6 Holes in outgoing args owned by CALLEE
3798 // Owned by +--------+
3799 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3800 // | new |preserve| Must be even-aligned.
3801 // | SP-+--------+----> Matcher::_new_SP, even aligned
3802 // | | |
3803 //
3804 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3805 // known from SELF's arguments and the Java calling convention.
3806 // Region 6-7 is determined per call site.
3807 // Note 2: If the calling convention leaves holes in the incoming argument
3808 // area, those holes are owned by SELF. Holes in the outgoing area
3809 // are owned by the CALLEE. Holes should not be necessary in the
3810 // incoming area, as the Java calling convention is completely under
3811 // the control of the AD file. Doubles can be sorted and packed to
3812 // avoid holes. Holes in the outgoing arguments may be necessary for
3813 // varargs C calling conventions.
3814 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3815 // even aligned with pad0 as needed.
3816 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3817 // (the latter is true on Intel but is it false on AArch64?)
3818 // region 6-11 is even aligned; it may be padded out more so that
3819 // the region from SP to FP meets the minimum stack alignment.
3820 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3821 // alignment. Region 11, pad1, may be dynamically extended so that
3822 // SP meets the minimum alignment.
3823
3824 frame %{
3825 // These three registers define part of the calling convention
3826 // between compiled code and the interpreter.
3827
3828 // Inline Cache Register or Method for I2C.
3829 inline_cache_reg(R12);
3830
3831 // Number of stack slots consumed by locking an object
3832 sync_stack_slots(2);
3833
3834 // Compiled code's Frame Pointer
3835 frame_pointer(R31);
3836
3837 // Interpreter stores its frame pointer in a register which is
3838 // stored to the stack by I2CAdaptors.
3839 // I2CAdaptors convert from interpreted java to compiled java.
3840 interpreter_frame_pointer(R29);
3841
3842 // Stack alignment requirement
3843 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3844
3845 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3846 // for calls to C. Supports the var-args backing area for register parms.
3847 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3848
3849 // The after-PROLOG location of the return address. Location of
3850 // return address specifies a type (REG or STACK) and a number
3851 // representing the register number (i.e. - use a register name) or
3852 // stack slot.
3853 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3854 // Otherwise, it is above the locks and verification slot and alignment word
3855 // TODO this may well be correct but need to check why that - 2 is there
3856 // ppc port uses 0 but we definitely need to allow for fixed_slots
3857 // which folds in the space used for monitors
3858 return_addr(STACK - 2 +
3859 align_up((Compile::current()->in_preserve_stack_slots() +
3860 Compile::current()->fixed_slots()),
3861 stack_alignment_in_slots()));
3862
3863 // Location of compiled Java return values. Same as C for now.
3864 return_value
3865 %{
3866 // TODO do we allow ideal_reg == Op_RegN???
3867 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3868 "only return normal values");
3869
3870 static const int lo[Op_RegL + 1] = { // enum name
3871 0, // Op_Node
3872 0, // Op_Set
3873 R0_num, // Op_RegN
3874 R0_num, // Op_RegI
3875 R0_num, // Op_RegP
3876 V0_num, // Op_RegF
3877 V0_num, // Op_RegD
3878 R0_num // Op_RegL
3879 };
3880
3881 static const int hi[Op_RegL + 1] = { // enum name
3882 0, // Op_Node
3883 0, // Op_Set
3884 OptoReg::Bad, // Op_RegN
3885 OptoReg::Bad, // Op_RegI
3886 R0_H_num, // Op_RegP
3887 OptoReg::Bad, // Op_RegF
3888 V0_H_num, // Op_RegD
3889 R0_H_num // Op_RegL
3890 };
3891
3892 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3893 %}
3894 %}
3895
3896 //----------ATTRIBUTES---------------------------------------------------------
3897 //----------Operand Attributes-------------------------------------------------
3898 op_attrib op_cost(1); // Required cost attribute
3899
3900 //----------Instruction Attributes---------------------------------------------
3901 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3902 ins_attrib ins_size(32); // Required size attribute (in bits)
3903 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3904 // a non-matching short branch variant
3905 // of some long branch?
3906 ins_attrib ins_alignment(4); // Required alignment attribute (must
3907 // be a power of 2) specifies the
3908 // alignment that some part of the
3909 // instruction (not necessarily the
3910 // start) requires. If > 1, a
3911 // compute_padding() function must be
3912 // provided for the instruction
3913
3914 // Whether this node is expanded during code emission into a sequence of
3915 // instructions and the first instruction can perform an implicit null check.
3916 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3917
3918 //----------OPERANDS-----------------------------------------------------------
3919 // Operand definitions must precede instruction definitions for correct parsing
3920 // in the ADLC because operands constitute user defined types which are used in
3921 // instruction definitions.
3922
3923 //----------Simple Operands----------------------------------------------------
3924
3925 // Integer operands 32 bit
3926 // 32 bit immediate
3927 operand immI()
3928 %{
3929 match(ConI);
3930
3931 op_cost(0);
3932 format %{ %}
3933 interface(CONST_INTER);
3934 %}
3935
3936 // 32 bit zero
3937 operand immI0()
3938 %{
3939 predicate(n->get_int() == 0);
3940 match(ConI);
3941
3942 op_cost(0);
3943 format %{ %}
3944 interface(CONST_INTER);
3945 %}
3946
3947 // 32 bit unit increment
3948 operand immI_1()
3949 %{
3950 predicate(n->get_int() == 1);
3951 match(ConI);
3952
3953 op_cost(0);
3954 format %{ %}
3955 interface(CONST_INTER);
3956 %}
3957
3958 // 32 bit unit decrement
3959 operand immI_M1()
3960 %{
3961 predicate(n->get_int() == -1);
3962 match(ConI);
3963
3964 op_cost(0);
3965 format %{ %}
3966 interface(CONST_INTER);
3967 %}
3968
3969 // Shift values for add/sub extension shift
3970 operand immIExt()
3971 %{
3972 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3973 match(ConI);
3974
3975 op_cost(0);
3976 format %{ %}
3977 interface(CONST_INTER);
3978 %}
3979
3980 operand immI_gt_1()
3981 %{
3982 predicate(n->get_int() > 1);
3983 match(ConI);
3984
3985 op_cost(0);
3986 format %{ %}
3987 interface(CONST_INTER);
3988 %}
3989
3990 operand immI_le_4()
3991 %{
3992 predicate(n->get_int() <= 4);
3993 match(ConI);
3994
3995 op_cost(0);
3996 format %{ %}
3997 interface(CONST_INTER);
3998 %}
3999
4000 operand immI_4()
4001 %{
4002 predicate(n->get_int() == 4);
4003 match(ConI);
4004
4005 op_cost(0);
4006 format %{ %}
4007 interface(CONST_INTER);
4008 %}
4009
4010 operand immI_16()
4011 %{
4012 predicate(n->get_int() == 16);
4013 match(ConI);
4014
4015 op_cost(0);
4016 format %{ %}
4017 interface(CONST_INTER);
4018 %}
4019
4020 operand immI_24()
4021 %{
4022 predicate(n->get_int() == 24);
4023 match(ConI);
4024
4025 op_cost(0);
4026 format %{ %}
4027 interface(CONST_INTER);
4028 %}
4029
4030 operand immI_32()
4031 %{
4032 predicate(n->get_int() == 32);
4033 match(ConI);
4034
4035 op_cost(0);
4036 format %{ %}
4037 interface(CONST_INTER);
4038 %}
4039
4040 operand immI_48()
4041 %{
4042 predicate(n->get_int() == 48);
4043 match(ConI);
4044
4045 op_cost(0);
4046 format %{ %}
4047 interface(CONST_INTER);
4048 %}
4049
4050 operand immI_56()
4051 %{
4052 predicate(n->get_int() == 56);
4053 match(ConI);
4054
4055 op_cost(0);
4056 format %{ %}
4057 interface(CONST_INTER);
4058 %}
4059
4060 operand immI_255()
4061 %{
4062 predicate(n->get_int() == 255);
4063 match(ConI);
4064
4065 op_cost(0);
4066 format %{ %}
4067 interface(CONST_INTER);
4068 %}
4069
4070 operand immI_65535()
4071 %{
4072 predicate(n->get_int() == 65535);
4073 match(ConI);
4074
4075 op_cost(0);
4076 format %{ %}
4077 interface(CONST_INTER);
4078 %}
4079
4080 operand immI_positive()
4081 %{
4082 predicate(n->get_int() > 0);
4083 match(ConI);
4084
4085 op_cost(0);
4086 format %{ %}
4087 interface(CONST_INTER);
4088 %}
4089
4090 // BoolTest condition for signed compare
4091 operand immI_cmp_cond()
4092 %{
4093 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4094 match(ConI);
4095
4096 op_cost(0);
4097 format %{ %}
4098 interface(CONST_INTER);
4099 %}
4100
4101 // BoolTest condition for unsigned compare
4102 operand immI_cmpU_cond()
4103 %{
4104 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4105 match(ConI);
4106
4107 op_cost(0);
4108 format %{ %}
4109 interface(CONST_INTER);
4110 %}
4111
4112 operand immL_255()
4113 %{
4114 predicate(n->get_long() == 255L);
4115 match(ConL);
4116
4117 op_cost(0);
4118 format %{ %}
4119 interface(CONST_INTER);
4120 %}
4121
4122 operand immL_65535()
4123 %{
4124 predicate(n->get_long() == 65535L);
4125 match(ConL);
4126
4127 op_cost(0);
4128 format %{ %}
4129 interface(CONST_INTER);
4130 %}
4131
4132 operand immL_4294967295()
4133 %{
4134 predicate(n->get_long() == 4294967295L);
4135 match(ConL);
4136
4137 op_cost(0);
4138 format %{ %}
4139 interface(CONST_INTER);
4140 %}
4141
4142 operand immL_bitmask()
4143 %{
4144 predicate((n->get_long() != 0)
4145 && ((n->get_long() & 0xc000000000000000l) == 0)
4146 && is_power_of_2(n->get_long() + 1));
4147 match(ConL);
4148
4149 op_cost(0);
4150 format %{ %}
4151 interface(CONST_INTER);
4152 %}
4153
4154 operand immI_bitmask()
4155 %{
4156 predicate((n->get_int() != 0)
4157 && ((n->get_int() & 0xc0000000) == 0)
4158 && is_power_of_2(n->get_int() + 1));
4159 match(ConI);
4160
4161 op_cost(0);
4162 format %{ %}
4163 interface(CONST_INTER);
4164 %}
4165
4166 operand immL_positive_bitmaskI()
4167 %{
4168 predicate((n->get_long() != 0)
4169 && ((julong)n->get_long() < 0x80000000ULL)
4170 && is_power_of_2(n->get_long() + 1));
4171 match(ConL);
4172
4173 op_cost(0);
4174 format %{ %}
4175 interface(CONST_INTER);
4176 %}
4177
4178 // Scale values for scaled offset addressing modes (up to long but not quad)
4179 operand immIScale()
4180 %{
4181 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4182 match(ConI);
4183
4184 op_cost(0);
4185 format %{ %}
4186 interface(CONST_INTER);
4187 %}
4188
4189 // 5 bit signed integer
4190 operand immI5()
4191 %{
4192 predicate(Assembler::is_simm(n->get_int(), 5));
4193 match(ConI);
4194
4195 op_cost(0);
4196 format %{ %}
4197 interface(CONST_INTER);
4198 %}
4199
4200 // 7 bit unsigned integer
4201 operand immIU7()
4202 %{
4203 predicate(Assembler::is_uimm(n->get_int(), 7));
4204 match(ConI);
4205
4206 op_cost(0);
4207 format %{ %}
4208 interface(CONST_INTER);
4209 %}
4210
4211 // Offset for scaled or unscaled immediate loads and stores
4212 operand immIOffset()
4213 %{
4214 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4215 match(ConI);
4216
4217 op_cost(0);
4218 format %{ %}
4219 interface(CONST_INTER);
4220 %}
4221
4222 operand immIOffset1()
4223 %{
4224 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4225 match(ConI);
4226
4227 op_cost(0);
4228 format %{ %}
4229 interface(CONST_INTER);
4230 %}
4231
4232 operand immIOffset2()
4233 %{
4234 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4235 match(ConI);
4236
4237 op_cost(0);
4238 format %{ %}
4239 interface(CONST_INTER);
4240 %}
4241
4242 operand immIOffset4()
4243 %{
4244 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4245 match(ConI);
4246
4247 op_cost(0);
4248 format %{ %}
4249 interface(CONST_INTER);
4250 %}
4251
4252 operand immIOffset8()
4253 %{
4254 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4255 match(ConI);
4256
4257 op_cost(0);
4258 format %{ %}
4259 interface(CONST_INTER);
4260 %}
4261
4262 operand immIOffset16()
4263 %{
4264 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4265 match(ConI);
4266
4267 op_cost(0);
4268 format %{ %}
4269 interface(CONST_INTER);
4270 %}
4271
4272 operand immLOffset()
4273 %{
4274 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4275 match(ConL);
4276
4277 op_cost(0);
4278 format %{ %}
4279 interface(CONST_INTER);
4280 %}
4281
4282 operand immLoffset1()
4283 %{
4284 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4285 match(ConL);
4286
4287 op_cost(0);
4288 format %{ %}
4289 interface(CONST_INTER);
4290 %}
4291
4292 operand immLoffset2()
4293 %{
4294 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4295 match(ConL);
4296
4297 op_cost(0);
4298 format %{ %}
4299 interface(CONST_INTER);
4300 %}
4301
4302 operand immLoffset4()
4303 %{
4304 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4305 match(ConL);
4306
4307 op_cost(0);
4308 format %{ %}
4309 interface(CONST_INTER);
4310 %}
4311
4312 operand immLoffset8()
4313 %{
4314 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4315 match(ConL);
4316
4317 op_cost(0);
4318 format %{ %}
4319 interface(CONST_INTER);
4320 %}
4321
4322 operand immLoffset16()
4323 %{
4324 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4325 match(ConL);
4326
4327 op_cost(0);
4328 format %{ %}
4329 interface(CONST_INTER);
4330 %}
4331
4332 // 5 bit signed long integer
4333 operand immL5()
4334 %{
4335 predicate(Assembler::is_simm(n->get_long(), 5));
4336 match(ConL);
4337
4338 op_cost(0);
4339 format %{ %}
4340 interface(CONST_INTER);
4341 %}
4342
4343 // 7 bit unsigned long integer
4344 operand immLU7()
4345 %{
4346 predicate(Assembler::is_uimm(n->get_long(), 7));
4347 match(ConL);
4348
4349 op_cost(0);
4350 format %{ %}
4351 interface(CONST_INTER);
4352 %}
4353
4354 // 8 bit signed value.
4355 operand immI8()
4356 %{
4357 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4358 match(ConI);
4359
4360 op_cost(0);
4361 format %{ %}
4362 interface(CONST_INTER);
4363 %}
4364
4365 // 8 bit signed value (simm8), or #simm8 LSL 8.
4366 operand immIDupV()
4367 %{
4368 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4369 match(ConI);
4370
4371 op_cost(0);
4372 format %{ %}
4373 interface(CONST_INTER);
4374 %}
4375
4376 // 8 bit signed value (simm8), or #simm8 LSL 8.
4377 operand immLDupV()
4378 %{
4379 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4380 match(ConL);
4381
4382 op_cost(0);
4383 format %{ %}
4384 interface(CONST_INTER);
4385 %}
4386
4387 // 8 bit signed value (simm8), or #simm8 LSL 8.
4388 operand immHDupV()
4389 %{
4390 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4391 match(ConH);
4392
4393 op_cost(0);
4394 format %{ %}
4395 interface(CONST_INTER);
4396 %}
4397
4398 // 8 bit integer valid for vector add sub immediate
4399 operand immBAddSubV()
4400 %{
4401 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4402 match(ConI);
4403
4404 op_cost(0);
4405 format %{ %}
4406 interface(CONST_INTER);
4407 %}
4408
4409 // 32 bit integer valid for add sub immediate
4410 operand immIAddSub()
4411 %{
4412 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4413 match(ConI);
4414 op_cost(0);
4415 format %{ %}
4416 interface(CONST_INTER);
4417 %}
4418
4419 // 32 bit integer valid for vector add sub immediate
4420 operand immIAddSubV()
4421 %{
4422 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4423 match(ConI);
4424
4425 op_cost(0);
4426 format %{ %}
4427 interface(CONST_INTER);
4428 %}
4429
4430 // 32 bit unsigned integer valid for logical immediate
4431
4432 operand immBLog()
4433 %{
4434 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4435 match(ConI);
4436
4437 op_cost(0);
4438 format %{ %}
4439 interface(CONST_INTER);
4440 %}
4441
4442 operand immSLog()
4443 %{
4444 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4445 match(ConI);
4446
4447 op_cost(0);
4448 format %{ %}
4449 interface(CONST_INTER);
4450 %}
4451
4452 operand immILog()
4453 %{
4454 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4455 match(ConI);
4456
4457 op_cost(0);
4458 format %{ %}
4459 interface(CONST_INTER);
4460 %}
4461
4462 // Integer operands 64 bit
4463 // 64 bit immediate
4464 operand immL()
4465 %{
4466 match(ConL);
4467
4468 op_cost(0);
4469 format %{ %}
4470 interface(CONST_INTER);
4471 %}
4472
4473 // 64 bit zero
4474 operand immL0()
4475 %{
4476 predicate(n->get_long() == 0);
4477 match(ConL);
4478
4479 op_cost(0);
4480 format %{ %}
4481 interface(CONST_INTER);
4482 %}
4483
4484 // 64 bit unit decrement
4485 operand immL_M1()
4486 %{
4487 predicate(n->get_long() == -1);
4488 match(ConL);
4489
4490 op_cost(0);
4491 format %{ %}
4492 interface(CONST_INTER);
4493 %}
4494
4495 // 64 bit integer valid for add sub immediate
4496 operand immLAddSub()
4497 %{
4498 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4499 match(ConL);
4500 op_cost(0);
4501 format %{ %}
4502 interface(CONST_INTER);
4503 %}
4504
4505 // 64 bit integer valid for addv subv immediate
4506 operand immLAddSubV()
4507 %{
4508 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4509 match(ConL);
4510
4511 op_cost(0);
4512 format %{ %}
4513 interface(CONST_INTER);
4514 %}
4515
4516 // 64 bit integer valid for logical immediate
4517 operand immLLog()
4518 %{
4519 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4520 match(ConL);
4521 op_cost(0);
4522 format %{ %}
4523 interface(CONST_INTER);
4524 %}
4525
4526 // Long Immediate: low 32-bit mask
4527 operand immL_32bits()
4528 %{
4529 predicate(n->get_long() == 0xFFFFFFFFL);
4530 match(ConL);
4531 op_cost(0);
4532 format %{ %}
4533 interface(CONST_INTER);
4534 %}
4535
4536 // Pointer operands
4537 // Pointer Immediate
4538 operand immP()
4539 %{
4540 match(ConP);
4541
4542 op_cost(0);
4543 format %{ %}
4544 interface(CONST_INTER);
4545 %}
4546
4547 // nullptr Pointer Immediate
4548 operand immP0()
4549 %{
4550 predicate(n->get_ptr() == 0);
4551 match(ConP);
4552
4553 op_cost(0);
4554 format %{ %}
4555 interface(CONST_INTER);
4556 %}
4557
4558 // Pointer Immediate One
4559 // this is used in object initialization (initial object header)
4560 operand immP_1()
4561 %{
4562 predicate(n->get_ptr() == 1);
4563 match(ConP);
4564
4565 op_cost(0);
4566 format %{ %}
4567 interface(CONST_INTER);
4568 %}
4569
4570 // Float and Double operands
4571 // Double Immediate
4572 operand immD()
4573 %{
4574 match(ConD);
4575 op_cost(0);
4576 format %{ %}
4577 interface(CONST_INTER);
4578 %}
4579
4580 // Double Immediate: +0.0d
4581 operand immD0()
4582 %{
4583 predicate(jlong_cast(n->getd()) == 0);
4584 match(ConD);
4585
4586 op_cost(0);
4587 format %{ %}
4588 interface(CONST_INTER);
4589 %}
4590
4591 // constant 'double +0.0'.
4592 operand immDPacked()
4593 %{
4594 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4595 match(ConD);
4596 op_cost(0);
4597 format %{ %}
4598 interface(CONST_INTER);
4599 %}
4600
4601 // Float Immediate
4602 operand immF()
4603 %{
4604 match(ConF);
4605 op_cost(0);
4606 format %{ %}
4607 interface(CONST_INTER);
4608 %}
4609
4610 // Float Immediate: +0.0f.
4611 operand immF0()
4612 %{
4613 predicate(jint_cast(n->getf()) == 0);
4614 match(ConF);
4615
4616 op_cost(0);
4617 format %{ %}
4618 interface(CONST_INTER);
4619 %}
4620
4621 // Half Float (FP16) Immediate
4622 operand immH()
4623 %{
4624 match(ConH);
4625 op_cost(0);
4626 format %{ %}
4627 interface(CONST_INTER);
4628 %}
4629
4630 //
4631 operand immFPacked()
4632 %{
4633 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4634 match(ConF);
4635 op_cost(0);
4636 format %{ %}
4637 interface(CONST_INTER);
4638 %}
4639
4640 // Narrow pointer operands
4641 // Narrow Pointer Immediate
4642 operand immN()
4643 %{
4644 match(ConN);
4645
4646 op_cost(0);
4647 format %{ %}
4648 interface(CONST_INTER);
4649 %}
4650
4651 // Narrow nullptr Pointer Immediate
4652 operand immN0()
4653 %{
4654 predicate(n->get_narrowcon() == 0);
4655 match(ConN);
4656
4657 op_cost(0);
4658 format %{ %}
4659 interface(CONST_INTER);
4660 %}
4661
4662 operand immNKlass()
4663 %{
4664 match(ConNKlass);
4665
4666 op_cost(0);
4667 format %{ %}
4668 interface(CONST_INTER);
4669 %}
4670
4671 // Integer 32 bit Register Operands
4672 // Integer 32 bitRegister (excludes SP)
4673 operand iRegI()
4674 %{
4675 constraint(ALLOC_IN_RC(any_reg32));
4676 match(RegI);
4677 match(iRegINoSp);
4678 op_cost(0);
4679 format %{ %}
4680 interface(REG_INTER);
4681 %}
4682
4683 // Integer 32 bit Register not Special
4684 operand iRegINoSp()
4685 %{
4686 constraint(ALLOC_IN_RC(no_special_reg32));
4687 match(RegI);
4688 op_cost(0);
4689 format %{ %}
4690 interface(REG_INTER);
4691 %}
4692
4693 // Integer 64 bit Register Operands
4694 // Integer 64 bit Register (includes SP)
4695 operand iRegL()
4696 %{
4697 constraint(ALLOC_IN_RC(any_reg));
4698 match(RegL);
4699 match(iRegLNoSp);
4700 op_cost(0);
4701 format %{ %}
4702 interface(REG_INTER);
4703 %}
4704
4705 // Integer 64 bit Register not Special
4706 operand iRegLNoSp()
4707 %{
4708 constraint(ALLOC_IN_RC(no_special_reg));
4709 match(RegL);
4710 match(iRegL_R0);
4711 format %{ %}
4712 interface(REG_INTER);
4713 %}
4714
4715 // Pointer Register Operands
4716 // Pointer Register
4717 operand iRegP()
4718 %{
4719 constraint(ALLOC_IN_RC(ptr_reg));
4720 match(RegP);
4721 match(iRegPNoSp);
4722 match(iRegP_R0);
4723 //match(iRegP_R2);
4724 //match(iRegP_R4);
4725 match(iRegP_R5);
4726 match(thread_RegP);
4727 op_cost(0);
4728 format %{ %}
4729 interface(REG_INTER);
4730 %}
4731
4732 // Pointer 64 bit Register not Special
4733 operand iRegPNoSp()
4734 %{
4735 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4736 match(RegP);
4737 // match(iRegP);
4738 // match(iRegP_R0);
4739 // match(iRegP_R2);
4740 // match(iRegP_R4);
4741 // match(iRegP_R5);
4742 // match(thread_RegP);
4743 op_cost(0);
4744 format %{ %}
4745 interface(REG_INTER);
4746 %}
4747
4748 // This operand is not allowed to use rfp even if
4749 // rfp is not used to hold the frame pointer.
4750 operand iRegPNoSpNoRfp()
4751 %{
4752 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4753 match(RegP);
4754 match(iRegPNoSp);
4755 op_cost(0);
4756 format %{ %}
4757 interface(REG_INTER);
4758 %}
4759
4760 // Pointer 64 bit Register R0 only
4761 operand iRegP_R0()
4762 %{
4763 constraint(ALLOC_IN_RC(r0_reg));
4764 match(RegP);
4765 // match(iRegP);
4766 match(iRegPNoSp);
4767 op_cost(0);
4768 format %{ %}
4769 interface(REG_INTER);
4770 %}
4771
4772 // Pointer 64 bit Register R1 only
4773 operand iRegP_R1()
4774 %{
4775 constraint(ALLOC_IN_RC(r1_reg));
4776 match(RegP);
4777 // match(iRegP);
4778 match(iRegPNoSp);
4779 op_cost(0);
4780 format %{ %}
4781 interface(REG_INTER);
4782 %}
4783
4784 // Pointer 64 bit Register R2 only
4785 operand iRegP_R2()
4786 %{
4787 constraint(ALLOC_IN_RC(r2_reg));
4788 match(RegP);
4789 // match(iRegP);
4790 match(iRegPNoSp);
4791 op_cost(0);
4792 format %{ %}
4793 interface(REG_INTER);
4794 %}
4795
4796 // Pointer 64 bit Register R3 only
4797 operand iRegP_R3()
4798 %{
4799 constraint(ALLOC_IN_RC(r3_reg));
4800 match(RegP);
4801 // match(iRegP);
4802 match(iRegPNoSp);
4803 op_cost(0);
4804 format %{ %}
4805 interface(REG_INTER);
4806 %}
4807
4808 // Pointer 64 bit Register R4 only
4809 operand iRegP_R4()
4810 %{
4811 constraint(ALLOC_IN_RC(r4_reg));
4812 match(RegP);
4813 // match(iRegP);
4814 match(iRegPNoSp);
4815 op_cost(0);
4816 format %{ %}
4817 interface(REG_INTER);
4818 %}
4819
4820 // Pointer 64 bit Register R5 only
4821 operand iRegP_R5()
4822 %{
4823 constraint(ALLOC_IN_RC(r5_reg));
4824 match(RegP);
4825 // match(iRegP);
4826 match(iRegPNoSp);
4827 op_cost(0);
4828 format %{ %}
4829 interface(REG_INTER);
4830 %}
4831
4832 // Pointer 64 bit Register R10 only
4833 operand iRegP_R10()
4834 %{
4835 constraint(ALLOC_IN_RC(r10_reg));
4836 match(RegP);
4837 // match(iRegP);
4838 match(iRegPNoSp);
4839 op_cost(0);
4840 format %{ %}
4841 interface(REG_INTER);
4842 %}
4843
4844 // Long 64 bit Register R0 only
4845 operand iRegL_R0()
4846 %{
4847 constraint(ALLOC_IN_RC(r0_reg));
4848 match(RegL);
4849 match(iRegLNoSp);
4850 op_cost(0);
4851 format %{ %}
4852 interface(REG_INTER);
4853 %}
4854
4855 // Long 64 bit Register R11 only
4856 operand iRegL_R11()
4857 %{
4858 constraint(ALLOC_IN_RC(r11_reg));
4859 match(RegL);
4860 match(iRegLNoSp);
4861 op_cost(0);
4862 format %{ %}
4863 interface(REG_INTER);
4864 %}
4865
4866 // Register R0 only
4867 operand iRegI_R0()
4868 %{
4869 constraint(ALLOC_IN_RC(int_r0_reg));
4870 match(RegI);
4871 match(iRegINoSp);
4872 op_cost(0);
4873 format %{ %}
4874 interface(REG_INTER);
4875 %}
4876
4877 // Register R2 only
4878 operand iRegI_R2()
4879 %{
4880 constraint(ALLOC_IN_RC(int_r2_reg));
4881 match(RegI);
4882 match(iRegINoSp);
4883 op_cost(0);
4884 format %{ %}
4885 interface(REG_INTER);
4886 %}
4887
4888 // Register R3 only
4889 operand iRegI_R3()
4890 %{
4891 constraint(ALLOC_IN_RC(int_r3_reg));
4892 match(RegI);
4893 match(iRegINoSp);
4894 op_cost(0);
4895 format %{ %}
4896 interface(REG_INTER);
4897 %}
4898
4899
4900 // Register R4 only
4901 operand iRegI_R4()
4902 %{
4903 constraint(ALLOC_IN_RC(int_r4_reg));
4904 match(RegI);
4905 match(iRegINoSp);
4906 op_cost(0);
4907 format %{ %}
4908 interface(REG_INTER);
4909 %}
4910
4911
4912 // Pointer Register Operands
4913 // Narrow Pointer Register
4914 operand iRegN()
4915 %{
4916 constraint(ALLOC_IN_RC(any_reg32));
4917 match(RegN);
4918 match(iRegNNoSp);
4919 op_cost(0);
4920 format %{ %}
4921 interface(REG_INTER);
4922 %}
4923
4924 // Integer 64 bit Register not Special
4925 operand iRegNNoSp()
4926 %{
4927 constraint(ALLOC_IN_RC(no_special_reg32));
4928 match(RegN);
4929 op_cost(0);
4930 format %{ %}
4931 interface(REG_INTER);
4932 %}
4933
4934 // Float Register
4935 // Float register operands
4936 operand vRegF()
4937 %{
4938 constraint(ALLOC_IN_RC(float_reg));
4939 match(RegF);
4940
4941 op_cost(0);
4942 format %{ %}
4943 interface(REG_INTER);
4944 %}
4945
4946 // Double Register
4947 // Double register operands
4948 operand vRegD()
4949 %{
4950 constraint(ALLOC_IN_RC(double_reg));
4951 match(RegD);
4952
4953 op_cost(0);
4954 format %{ %}
4955 interface(REG_INTER);
4956 %}
4957
4958 // Generic vector class. This will be used for
4959 // all vector operands, including NEON and SVE.
4960 operand vReg()
4961 %{
4962 constraint(ALLOC_IN_RC(dynamic));
4963 match(VecA);
4964 match(VecD);
4965 match(VecX);
4966
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 operand vReg_V10()
4973 %{
4974 constraint(ALLOC_IN_RC(v10_veca_reg));
4975 match(vReg);
4976
4977 op_cost(0);
4978 format %{ %}
4979 interface(REG_INTER);
4980 %}
4981
4982 operand vReg_V11()
4983 %{
4984 constraint(ALLOC_IN_RC(v11_veca_reg));
4985 match(vReg);
4986
4987 op_cost(0);
4988 format %{ %}
4989 interface(REG_INTER);
4990 %}
4991
4992 operand vReg_V12()
4993 %{
4994 constraint(ALLOC_IN_RC(v12_veca_reg));
4995 match(vReg);
4996
4997 op_cost(0);
4998 format %{ %}
4999 interface(REG_INTER);
5000 %}
5001
5002 operand vReg_V13()
5003 %{
5004 constraint(ALLOC_IN_RC(v13_veca_reg));
5005 match(vReg);
5006
5007 op_cost(0);
5008 format %{ %}
5009 interface(REG_INTER);
5010 %}
5011
5012 operand vReg_V17()
5013 %{
5014 constraint(ALLOC_IN_RC(v17_veca_reg));
5015 match(vReg);
5016
5017 op_cost(0);
5018 format %{ %}
5019 interface(REG_INTER);
5020 %}
5021
5022 operand vReg_V18()
5023 %{
5024 constraint(ALLOC_IN_RC(v18_veca_reg));
5025 match(vReg);
5026
5027 op_cost(0);
5028 format %{ %}
5029 interface(REG_INTER);
5030 %}
5031
5032 operand vReg_V23()
5033 %{
5034 constraint(ALLOC_IN_RC(v23_veca_reg));
5035 match(vReg);
5036
5037 op_cost(0);
5038 format %{ %}
5039 interface(REG_INTER);
5040 %}
5041
5042 operand vReg_V24()
5043 %{
5044 constraint(ALLOC_IN_RC(v24_veca_reg));
5045 match(vReg);
5046
5047 op_cost(0);
5048 format %{ %}
5049 interface(REG_INTER);
5050 %}
5051
5052 operand vecA()
5053 %{
5054 constraint(ALLOC_IN_RC(vectora_reg));
5055 match(VecA);
5056
5057 op_cost(0);
5058 format %{ %}
5059 interface(REG_INTER);
5060 %}
5061
5062 operand vecD()
5063 %{
5064 constraint(ALLOC_IN_RC(vectord_reg));
5065 match(VecD);
5066
5067 op_cost(0);
5068 format %{ %}
5069 interface(REG_INTER);
5070 %}
5071
5072 operand vecX()
5073 %{
5074 constraint(ALLOC_IN_RC(vectorx_reg));
5075 match(VecX);
5076
5077 op_cost(0);
5078 format %{ %}
5079 interface(REG_INTER);
5080 %}
5081
5082 operand vRegD_V0()
5083 %{
5084 constraint(ALLOC_IN_RC(v0_reg));
5085 match(RegD);
5086 op_cost(0);
5087 format %{ %}
5088 interface(REG_INTER);
5089 %}
5090
5091 operand vRegD_V1()
5092 %{
5093 constraint(ALLOC_IN_RC(v1_reg));
5094 match(RegD);
5095 op_cost(0);
5096 format %{ %}
5097 interface(REG_INTER);
5098 %}
5099
5100 operand vRegD_V2()
5101 %{
5102 constraint(ALLOC_IN_RC(v2_reg));
5103 match(RegD);
5104 op_cost(0);
5105 format %{ %}
5106 interface(REG_INTER);
5107 %}
5108
5109 operand vRegD_V3()
5110 %{
5111 constraint(ALLOC_IN_RC(v3_reg));
5112 match(RegD);
5113 op_cost(0);
5114 format %{ %}
5115 interface(REG_INTER);
5116 %}
5117
5118 operand vRegD_V4()
5119 %{
5120 constraint(ALLOC_IN_RC(v4_reg));
5121 match(RegD);
5122 op_cost(0);
5123 format %{ %}
5124 interface(REG_INTER);
5125 %}
5126
5127 operand vRegD_V5()
5128 %{
5129 constraint(ALLOC_IN_RC(v5_reg));
5130 match(RegD);
5131 op_cost(0);
5132 format %{ %}
5133 interface(REG_INTER);
5134 %}
5135
5136 operand vRegD_V6()
5137 %{
5138 constraint(ALLOC_IN_RC(v6_reg));
5139 match(RegD);
5140 op_cost(0);
5141 format %{ %}
5142 interface(REG_INTER);
5143 %}
5144
5145 operand vRegD_V7()
5146 %{
5147 constraint(ALLOC_IN_RC(v7_reg));
5148 match(RegD);
5149 op_cost(0);
5150 format %{ %}
5151 interface(REG_INTER);
5152 %}
5153
5154 operand vRegD_V12()
5155 %{
5156 constraint(ALLOC_IN_RC(v12_reg));
5157 match(RegD);
5158 op_cost(0);
5159 format %{ %}
5160 interface(REG_INTER);
5161 %}
5162
5163 operand vRegD_V13()
5164 %{
5165 constraint(ALLOC_IN_RC(v13_reg));
5166 match(RegD);
5167 op_cost(0);
5168 format %{ %}
5169 interface(REG_INTER);
5170 %}
5171
5172 operand pReg()
5173 %{
5174 constraint(ALLOC_IN_RC(pr_reg));
5175 match(RegVectMask);
5176 match(pRegGov);
5177 op_cost(0);
5178 format %{ %}
5179 interface(REG_INTER);
5180 %}
5181
5182 operand pRegGov()
5183 %{
5184 constraint(ALLOC_IN_RC(gov_pr));
5185 match(RegVectMask);
5186 match(pReg);
5187 op_cost(0);
5188 format %{ %}
5189 interface(REG_INTER);
5190 %}
5191
5192 operand pRegGov_P0()
5193 %{
5194 constraint(ALLOC_IN_RC(p0_reg));
5195 match(RegVectMask);
5196 op_cost(0);
5197 format %{ %}
5198 interface(REG_INTER);
5199 %}
5200
5201 operand pRegGov_P1()
5202 %{
5203 constraint(ALLOC_IN_RC(p1_reg));
5204 match(RegVectMask);
5205 op_cost(0);
5206 format %{ %}
5207 interface(REG_INTER);
5208 %}
5209
5210 // Flags register, used as output of signed compare instructions
5211
5212 // note that on AArch64 we also use this register as the output for
5213 // for floating point compare instructions (CmpF CmpD). this ensures
5214 // that ordered inequality tests use GT, GE, LT or LE none of which
5215 // pass through cases where the result is unordered i.e. one or both
5216 // inputs to the compare is a NaN. this means that the ideal code can
5217 // replace e.g. a GT with an LE and not end up capturing the NaN case
5218 // (where the comparison should always fail). EQ and NE tests are
5219 // always generated in ideal code so that unordered folds into the NE
5220 // case, matching the behaviour of AArch64 NE.
5221 //
5222 // This differs from x86 where the outputs of FP compares use a
5223 // special FP flags registers and where compares based on this
5224 // register are distinguished into ordered inequalities (cmpOpUCF) and
5225 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5226 // to explicitly handle the unordered case in branches. x86 also has
5227 // to include extra CMoveX rules to accept a cmpOpUCF input.
5228
5229 operand rFlagsReg()
5230 %{
5231 constraint(ALLOC_IN_RC(int_flags));
5232 match(RegFlags);
5233
5234 op_cost(0);
5235 format %{ "RFLAGS" %}
5236 interface(REG_INTER);
5237 %}
5238
5239 // Flags register, used as output of unsigned compare instructions
5240 operand rFlagsRegU()
5241 %{
5242 constraint(ALLOC_IN_RC(int_flags));
5243 match(RegFlags);
5244
5245 op_cost(0);
5246 format %{ "RFLAGSU" %}
5247 interface(REG_INTER);
5248 %}
5249
5250 // Special Registers
5251
5252 // Method Register
5253 operand inline_cache_RegP(iRegP reg)
5254 %{
5255 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5256 match(reg);
5257 match(iRegPNoSp);
5258 op_cost(0);
5259 format %{ %}
5260 interface(REG_INTER);
5261 %}
5262
5263 // Thread Register
5264 operand thread_RegP(iRegP reg)
5265 %{
5266 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5267 match(reg);
5268 op_cost(0);
5269 format %{ %}
5270 interface(REG_INTER);
5271 %}
5272
5273 //----------Memory Operands----------------------------------------------------
5274
5275 operand indirect(iRegP reg)
5276 %{
5277 constraint(ALLOC_IN_RC(ptr_reg));
5278 match(reg);
5279 op_cost(0);
5280 format %{ "[$reg]" %}
5281 interface(MEMORY_INTER) %{
5282 base($reg);
5283 index(0xffffffff);
5284 scale(0x0);
5285 disp(0x0);
5286 %}
5287 %}
5288
5289 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5290 %{
5291 constraint(ALLOC_IN_RC(ptr_reg));
5292 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5293 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5294 op_cost(0);
5295 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5296 interface(MEMORY_INTER) %{
5297 base($reg);
5298 index($ireg);
5299 scale($scale);
5300 disp(0x0);
5301 %}
5302 %}
5303
5304 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5305 %{
5306 constraint(ALLOC_IN_RC(ptr_reg));
5307 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5308 match(AddP reg (LShiftL lreg scale));
5309 op_cost(0);
5310 format %{ "$reg, $lreg lsl($scale)" %}
5311 interface(MEMORY_INTER) %{
5312 base($reg);
5313 index($lreg);
5314 scale($scale);
5315 disp(0x0);
5316 %}
5317 %}
5318
5319 operand indIndexI2L(iRegP reg, iRegI ireg)
5320 %{
5321 constraint(ALLOC_IN_RC(ptr_reg));
5322 match(AddP reg (ConvI2L ireg));
5323 op_cost(0);
5324 format %{ "$reg, $ireg, 0, I2L" %}
5325 interface(MEMORY_INTER) %{
5326 base($reg);
5327 index($ireg);
5328 scale(0x0);
5329 disp(0x0);
5330 %}
5331 %}
5332
5333 operand indIndex(iRegP reg, iRegL lreg)
5334 %{
5335 constraint(ALLOC_IN_RC(ptr_reg));
5336 match(AddP reg lreg);
5337 op_cost(0);
5338 format %{ "$reg, $lreg" %}
5339 interface(MEMORY_INTER) %{
5340 base($reg);
5341 index($lreg);
5342 scale(0x0);
5343 disp(0x0);
5344 %}
5345 %}
5346
5347 operand indOffI1(iRegP reg, immIOffset1 off)
5348 %{
5349 constraint(ALLOC_IN_RC(ptr_reg));
5350 match(AddP reg off);
5351 op_cost(0);
5352 format %{ "[$reg, $off]" %}
5353 interface(MEMORY_INTER) %{
5354 base($reg);
5355 index(0xffffffff);
5356 scale(0x0);
5357 disp($off);
5358 %}
5359 %}
5360
5361 operand indOffI2(iRegP reg, immIOffset2 off)
5362 %{
5363 constraint(ALLOC_IN_RC(ptr_reg));
5364 match(AddP reg off);
5365 op_cost(0);
5366 format %{ "[$reg, $off]" %}
5367 interface(MEMORY_INTER) %{
5368 base($reg);
5369 index(0xffffffff);
5370 scale(0x0);
5371 disp($off);
5372 %}
5373 %}
5374
5375 operand indOffI4(iRegP reg, immIOffset4 off)
5376 %{
5377 constraint(ALLOC_IN_RC(ptr_reg));
5378 match(AddP reg off);
5379 op_cost(0);
5380 format %{ "[$reg, $off]" %}
5381 interface(MEMORY_INTER) %{
5382 base($reg);
5383 index(0xffffffff);
5384 scale(0x0);
5385 disp($off);
5386 %}
5387 %}
5388
5389 operand indOffI8(iRegP reg, immIOffset8 off)
5390 %{
5391 constraint(ALLOC_IN_RC(ptr_reg));
5392 match(AddP reg off);
5393 op_cost(0);
5394 format %{ "[$reg, $off]" %}
5395 interface(MEMORY_INTER) %{
5396 base($reg);
5397 index(0xffffffff);
5398 scale(0x0);
5399 disp($off);
5400 %}
5401 %}
5402
5403 operand indOffI16(iRegP reg, immIOffset16 off)
5404 %{
5405 constraint(ALLOC_IN_RC(ptr_reg));
5406 match(AddP reg off);
5407 op_cost(0);
5408 format %{ "[$reg, $off]" %}
5409 interface(MEMORY_INTER) %{
5410 base($reg);
5411 index(0xffffffff);
5412 scale(0x0);
5413 disp($off);
5414 %}
5415 %}
5416
5417 operand indOffL1(iRegP reg, immLoffset1 off)
5418 %{
5419 constraint(ALLOC_IN_RC(ptr_reg));
5420 match(AddP reg off);
5421 op_cost(0);
5422 format %{ "[$reg, $off]" %}
5423 interface(MEMORY_INTER) %{
5424 base($reg);
5425 index(0xffffffff);
5426 scale(0x0);
5427 disp($off);
5428 %}
5429 %}
5430
5431 operand indOffL2(iRegP reg, immLoffset2 off)
5432 %{
5433 constraint(ALLOC_IN_RC(ptr_reg));
5434 match(AddP reg off);
5435 op_cost(0);
5436 format %{ "[$reg, $off]" %}
5437 interface(MEMORY_INTER) %{
5438 base($reg);
5439 index(0xffffffff);
5440 scale(0x0);
5441 disp($off);
5442 %}
5443 %}
5444
5445 operand indOffL4(iRegP reg, immLoffset4 off)
5446 %{
5447 constraint(ALLOC_IN_RC(ptr_reg));
5448 match(AddP reg off);
5449 op_cost(0);
5450 format %{ "[$reg, $off]" %}
5451 interface(MEMORY_INTER) %{
5452 base($reg);
5453 index(0xffffffff);
5454 scale(0x0);
5455 disp($off);
5456 %}
5457 %}
5458
5459 operand indOffL8(iRegP reg, immLoffset8 off)
5460 %{
5461 constraint(ALLOC_IN_RC(ptr_reg));
5462 match(AddP reg off);
5463 op_cost(0);
5464 format %{ "[$reg, $off]" %}
5465 interface(MEMORY_INTER) %{
5466 base($reg);
5467 index(0xffffffff);
5468 scale(0x0);
5469 disp($off);
5470 %}
5471 %}
5472
5473 operand indOffL16(iRegP reg, immLoffset16 off)
5474 %{
5475 constraint(ALLOC_IN_RC(ptr_reg));
5476 match(AddP reg off);
5477 op_cost(0);
5478 format %{ "[$reg, $off]" %}
5479 interface(MEMORY_INTER) %{
5480 base($reg);
5481 index(0xffffffff);
5482 scale(0x0);
5483 disp($off);
5484 %}
5485 %}
5486
5487 operand indirectX2P(iRegL reg)
5488 %{
5489 constraint(ALLOC_IN_RC(ptr_reg));
5490 match(CastX2P reg);
5491 op_cost(0);
5492 format %{ "[$reg]\t# long -> ptr" %}
5493 interface(MEMORY_INTER) %{
5494 base($reg);
5495 index(0xffffffff);
5496 scale(0x0);
5497 disp(0x0);
5498 %}
5499 %}
5500
5501 operand indOffX2P(iRegL reg, immLOffset off)
5502 %{
5503 constraint(ALLOC_IN_RC(ptr_reg));
5504 match(AddP (CastX2P reg) off);
5505 op_cost(0);
5506 format %{ "[$reg, $off]\t# long -> ptr" %}
5507 interface(MEMORY_INTER) %{
5508 base($reg);
5509 index(0xffffffff);
5510 scale(0x0);
5511 disp($off);
5512 %}
5513 %}
5514
5515 operand indirectN(iRegN reg)
5516 %{
5517 predicate(CompressedOops::shift() == 0);
5518 constraint(ALLOC_IN_RC(ptr_reg));
5519 match(DecodeN reg);
5520 op_cost(0);
5521 format %{ "[$reg]\t# narrow" %}
5522 interface(MEMORY_INTER) %{
5523 base($reg);
5524 index(0xffffffff);
5525 scale(0x0);
5526 disp(0x0);
5527 %}
5528 %}
5529
5530 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5531 %{
5532 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5533 constraint(ALLOC_IN_RC(ptr_reg));
5534 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5535 op_cost(0);
5536 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5537 interface(MEMORY_INTER) %{
5538 base($reg);
5539 index($ireg);
5540 scale($scale);
5541 disp(0x0);
5542 %}
5543 %}
5544
5545 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5546 %{
5547 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5548 constraint(ALLOC_IN_RC(ptr_reg));
5549 match(AddP (DecodeN reg) (LShiftL lreg scale));
5550 op_cost(0);
5551 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5552 interface(MEMORY_INTER) %{
5553 base($reg);
5554 index($lreg);
5555 scale($scale);
5556 disp(0x0);
5557 %}
5558 %}
5559
5560 operand indIndexI2LN(iRegN reg, iRegI ireg)
5561 %{
5562 predicate(CompressedOops::shift() == 0);
5563 constraint(ALLOC_IN_RC(ptr_reg));
5564 match(AddP (DecodeN reg) (ConvI2L ireg));
5565 op_cost(0);
5566 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5567 interface(MEMORY_INTER) %{
5568 base($reg);
5569 index($ireg);
5570 scale(0x0);
5571 disp(0x0);
5572 %}
5573 %}
5574
5575 operand indIndexN(iRegN reg, iRegL lreg)
5576 %{
5577 predicate(CompressedOops::shift() == 0);
5578 constraint(ALLOC_IN_RC(ptr_reg));
5579 match(AddP (DecodeN reg) lreg);
5580 op_cost(0);
5581 format %{ "$reg, $lreg\t# narrow" %}
5582 interface(MEMORY_INTER) %{
5583 base($reg);
5584 index($lreg);
5585 scale(0x0);
5586 disp(0x0);
5587 %}
5588 %}
5589
5590 operand indOffIN(iRegN reg, immIOffset off)
5591 %{
5592 predicate(CompressedOops::shift() == 0);
5593 constraint(ALLOC_IN_RC(ptr_reg));
5594 match(AddP (DecodeN reg) off);
5595 op_cost(0);
5596 format %{ "[$reg, $off]\t# narrow" %}
5597 interface(MEMORY_INTER) %{
5598 base($reg);
5599 index(0xffffffff);
5600 scale(0x0);
5601 disp($off);
5602 %}
5603 %}
5604
5605 operand indOffLN(iRegN reg, immLOffset off)
5606 %{
5607 predicate(CompressedOops::shift() == 0);
5608 constraint(ALLOC_IN_RC(ptr_reg));
5609 match(AddP (DecodeN reg) off);
5610 op_cost(0);
5611 format %{ "[$reg, $off]\t# narrow" %}
5612 interface(MEMORY_INTER) %{
5613 base($reg);
5614 index(0xffffffff);
5615 scale(0x0);
5616 disp($off);
5617 %}
5618 %}
5619
5620
5621 //----------Special Memory Operands--------------------------------------------
5622 // Stack Slot Operand - This operand is used for loading and storing temporary
5623 // values on the stack where a match requires a value to
5624 // flow through memory.
5625 operand stackSlotP(sRegP reg)
5626 %{
5627 constraint(ALLOC_IN_RC(stack_slots));
5628 op_cost(100);
5629 // No match rule because this operand is only generated in matching
5630 // match(RegP);
5631 format %{ "[$reg]" %}
5632 interface(MEMORY_INTER) %{
5633 base(0x1e); // RSP
5634 index(0x0); // No Index
5635 scale(0x0); // No Scale
5636 disp($reg); // Stack Offset
5637 %}
5638 %}
5639
5640 operand stackSlotI(sRegI reg)
5641 %{
5642 constraint(ALLOC_IN_RC(stack_slots));
5643 // No match rule because this operand is only generated in matching
5644 // match(RegI);
5645 format %{ "[$reg]" %}
5646 interface(MEMORY_INTER) %{
5647 base(0x1e); // RSP
5648 index(0x0); // No Index
5649 scale(0x0); // No Scale
5650 disp($reg); // Stack Offset
5651 %}
5652 %}
5653
5654 operand stackSlotF(sRegF reg)
5655 %{
5656 constraint(ALLOC_IN_RC(stack_slots));
5657 // No match rule because this operand is only generated in matching
5658 // match(RegF);
5659 format %{ "[$reg]" %}
5660 interface(MEMORY_INTER) %{
5661 base(0x1e); // RSP
5662 index(0x0); // No Index
5663 scale(0x0); // No Scale
5664 disp($reg); // Stack Offset
5665 %}
5666 %}
5667
5668 operand stackSlotD(sRegD reg)
5669 %{
5670 constraint(ALLOC_IN_RC(stack_slots));
5671 // No match rule because this operand is only generated in matching
5672 // match(RegD);
5673 format %{ "[$reg]" %}
5674 interface(MEMORY_INTER) %{
5675 base(0x1e); // RSP
5676 index(0x0); // No Index
5677 scale(0x0); // No Scale
5678 disp($reg); // Stack Offset
5679 %}
5680 %}
5681
5682 operand stackSlotL(sRegL reg)
5683 %{
5684 constraint(ALLOC_IN_RC(stack_slots));
5685 // No match rule because this operand is only generated in matching
5686 // match(RegL);
5687 format %{ "[$reg]" %}
5688 interface(MEMORY_INTER) %{
5689 base(0x1e); // RSP
5690 index(0x0); // No Index
5691 scale(0x0); // No Scale
5692 disp($reg); // Stack Offset
5693 %}
5694 %}
5695
5696 // Operands for expressing Control Flow
5697 // NOTE: Label is a predefined operand which should not be redefined in
5698 // the AD file. It is generically handled within the ADLC.
5699
5700 //----------Conditional Branch Operands----------------------------------------
5701 // Comparison Op - This is the operation of the comparison, and is limited to
5702 // the following set of codes:
5703 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5704 //
5705 // Other attributes of the comparison, such as unsignedness, are specified
5706 // by the comparison instruction that sets a condition code flags register.
5707 // That result is represented by a flags operand whose subtype is appropriate
5708 // to the unsignedness (etc.) of the comparison.
5709 //
5710 // Later, the instruction which matches both the Comparison Op (a Bool) and
5711 // the flags (produced by the Cmp) specifies the coding of the comparison op
5712 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5713
5714 // used for signed integral comparisons and fp comparisons
5715
5716 operand cmpOp()
5717 %{
5718 match(Bool);
5719
5720 format %{ "" %}
5721 interface(COND_INTER) %{
5722 equal(0x0, "eq");
5723 not_equal(0x1, "ne");
5724 less(0xb, "lt");
5725 greater_equal(0xa, "ge");
5726 less_equal(0xd, "le");
5727 greater(0xc, "gt");
5728 overflow(0x6, "vs");
5729 no_overflow(0x7, "vc");
5730 %}
5731 %}
5732
5733 // used for unsigned integral comparisons
5734
5735 operand cmpOpU()
5736 %{
5737 match(Bool);
5738
5739 format %{ "" %}
5740 interface(COND_INTER) %{
5741 equal(0x0, "eq");
5742 not_equal(0x1, "ne");
5743 less(0x3, "lo");
5744 greater_equal(0x2, "hs");
5745 less_equal(0x9, "ls");
5746 greater(0x8, "hi");
5747 overflow(0x6, "vs");
5748 no_overflow(0x7, "vc");
5749 %}
5750 %}
5751
5752 // used for certain integral comparisons which can be
5753 // converted to cbxx or tbxx instructions
5754
5755 operand cmpOpEqNe()
5756 %{
5757 match(Bool);
5758 op_cost(0);
5759 predicate(n->as_Bool()->_test._test == BoolTest::ne
5760 || n->as_Bool()->_test._test == BoolTest::eq);
5761
5762 format %{ "" %}
5763 interface(COND_INTER) %{
5764 equal(0x0, "eq");
5765 not_equal(0x1, "ne");
5766 less(0xb, "lt");
5767 greater_equal(0xa, "ge");
5768 less_equal(0xd, "le");
5769 greater(0xc, "gt");
5770 overflow(0x6, "vs");
5771 no_overflow(0x7, "vc");
5772 %}
5773 %}
5774
5775 // used for certain integral comparisons which can be
5776 // converted to cbxx or tbxx instructions
5777
5778 operand cmpOpLtGe()
5779 %{
5780 match(Bool);
5781 op_cost(0);
5782
5783 predicate(n->as_Bool()->_test._test == BoolTest::lt
5784 || n->as_Bool()->_test._test == BoolTest::ge);
5785
5786 format %{ "" %}
5787 interface(COND_INTER) %{
5788 equal(0x0, "eq");
5789 not_equal(0x1, "ne");
5790 less(0xb, "lt");
5791 greater_equal(0xa, "ge");
5792 less_equal(0xd, "le");
5793 greater(0xc, "gt");
5794 overflow(0x6, "vs");
5795 no_overflow(0x7, "vc");
5796 %}
5797 %}
5798
5799 // used for certain unsigned integral comparisons which can be
5800 // converted to cbxx or tbxx instructions
5801
5802 operand cmpOpUEqNeLeGt()
5803 %{
5804 match(Bool);
5805 op_cost(0);
5806
5807 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5808 n->as_Bool()->_test._test == BoolTest::ne ||
5809 n->as_Bool()->_test._test == BoolTest::le ||
5810 n->as_Bool()->_test._test == BoolTest::gt);
5811
5812 format %{ "" %}
5813 interface(COND_INTER) %{
5814 equal(0x0, "eq");
5815 not_equal(0x1, "ne");
5816 less(0x3, "lo");
5817 greater_equal(0x2, "hs");
5818 less_equal(0x9, "ls");
5819 greater(0x8, "hi");
5820 overflow(0x6, "vs");
5821 no_overflow(0x7, "vc");
5822 %}
5823 %}
5824
5825 // Special operand allowing long args to int ops to be truncated for free
5826
5827 operand iRegL2I(iRegL reg) %{
5828
5829 op_cost(0);
5830
5831 match(ConvL2I reg);
5832
5833 format %{ "l2i($reg)" %}
5834
5835 interface(REG_INTER)
5836 %}
5837
5838 operand iRegL2P(iRegL reg) %{
5839
5840 op_cost(0);
5841
5842 match(CastX2P reg);
5843
5844 format %{ "l2p($reg)" %}
5845
5846 interface(REG_INTER)
5847 %}
5848
5849 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5850 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5851 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5852 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5853
5854 //----------OPERAND CLASSES----------------------------------------------------
5855 // Operand Classes are groups of operands that are used as to simplify
5856 // instruction definitions by not requiring the AD writer to specify
5857 // separate instructions for every form of operand when the
5858 // instruction accepts multiple operand types with the same basic
5859 // encoding and format. The classic case of this is memory operands.
5860
5861 // memory is used to define read/write location for load/store
5862 // instruction defs. we can turn a memory op into an Address
5863
5864 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5865 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5866
5867 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5868 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5869
5870 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5871 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5872
5873 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5874 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5875
5876 // All of the memory operands. For the pipeline description.
5877 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5878 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5879 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5880
5881
5882 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5883 // operations. it allows the src to be either an iRegI or a (ConvL2I
5884 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5885 // can be elided because the 32-bit instruction will just employ the
5886 // lower 32 bits anyway.
5887 //
5888 // n.b. this does not elide all L2I conversions. if the truncated
5889 // value is consumed by more than one operation then the ConvL2I
5890 // cannot be bundled into the consuming nodes so an l2i gets planted
5891 // (actually a movw $dst $src) and the downstream instructions consume
5892 // the result of the l2i as an iRegI input. That's a shame since the
5893 // movw is actually redundant but its not too costly.
5894
5895 opclass iRegIorL2I(iRegI, iRegL2I);
5896 opclass iRegPorL2P(iRegP, iRegL2P);
5897
5898 //----------PIPELINE-----------------------------------------------------------
5899 // Rules which define the behavior of the target architectures pipeline.
5900
5901 // For specific pipelines, eg A53, define the stages of that pipeline
5902 //pipe_desc(ISS, EX1, EX2, WR);
5903 #define ISS S0
5904 #define EX1 S1
5905 #define EX2 S2
5906 #define WR S3
5907
5908 // Integer ALU reg operation
5909 pipeline %{
5910
5911 attributes %{
5912 // ARM instructions are of fixed length
5913 fixed_size_instructions; // Fixed size instructions TODO does
5914 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5915 // ARM instructions come in 32-bit word units
5916 instruction_unit_size = 4; // An instruction is 4 bytes long
5917 instruction_fetch_unit_size = 64; // The processor fetches one line
5918 instruction_fetch_units = 1; // of 64 bytes
5919 %}
5920
5921 // We don't use an actual pipeline model so don't care about resources
5922 // or description. we do use pipeline classes to introduce fixed
5923 // latencies
5924
5925 //----------RESOURCES----------------------------------------------------------
5926 // Resources are the functional units available to the machine
5927
5928 resources( INS0, INS1, INS01 = INS0 | INS1,
5929 ALU0, ALU1, ALU = ALU0 | ALU1,
5930 MAC,
5931 DIV,
5932 BRANCH,
5933 LDST,
5934 NEON_FP);
5935
5936 //----------PIPELINE DESCRIPTION-----------------------------------------------
5937 // Pipeline Description specifies the stages in the machine's pipeline
5938
5939 // Define the pipeline as a generic 6 stage pipeline
5940 pipe_desc(S0, S1, S2, S3, S4, S5);
5941
5942 //----------PIPELINE CLASSES---------------------------------------------------
5943 // Pipeline Classes describe the stages in which input and output are
5944 // referenced by the hardware pipeline.
5945
5946 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5947 %{
5948 single_instruction;
5949 src1 : S1(read);
5950 src2 : S2(read);
5951 dst : S5(write);
5952 INS01 : ISS;
5953 NEON_FP : S5;
5954 %}
5955
5956 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5957 %{
5958 single_instruction;
5959 src1 : S1(read);
5960 src2 : S2(read);
5961 dst : S5(write);
5962 INS01 : ISS;
5963 NEON_FP : S5;
5964 %}
5965
5966 pipe_class fp_uop_s(vRegF dst, vRegF src)
5967 %{
5968 single_instruction;
5969 src : S1(read);
5970 dst : S5(write);
5971 INS01 : ISS;
5972 NEON_FP : S5;
5973 %}
5974
5975 pipe_class fp_uop_d(vRegD dst, vRegD src)
5976 %{
5977 single_instruction;
5978 src : S1(read);
5979 dst : S5(write);
5980 INS01 : ISS;
5981 NEON_FP : S5;
5982 %}
5983
5984 pipe_class fp_d2f(vRegF dst, vRegD src)
5985 %{
5986 single_instruction;
5987 src : S1(read);
5988 dst : S5(write);
5989 INS01 : ISS;
5990 NEON_FP : S5;
5991 %}
5992
5993 pipe_class fp_f2d(vRegD dst, vRegF src)
5994 %{
5995 single_instruction;
5996 src : S1(read);
5997 dst : S5(write);
5998 INS01 : ISS;
5999 NEON_FP : S5;
6000 %}
6001
6002 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6003 %{
6004 single_instruction;
6005 src : S1(read);
6006 dst : S5(write);
6007 INS01 : ISS;
6008 NEON_FP : S5;
6009 %}
6010
6011 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6012 %{
6013 single_instruction;
6014 src : S1(read);
6015 dst : S5(write);
6016 INS01 : ISS;
6017 NEON_FP : S5;
6018 %}
6019
6020 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6021 %{
6022 single_instruction;
6023 src : S1(read);
6024 dst : S5(write);
6025 INS01 : ISS;
6026 NEON_FP : S5;
6027 %}
6028
6029 pipe_class fp_l2f(vRegF dst, iRegL src)
6030 %{
6031 single_instruction;
6032 src : S1(read);
6033 dst : S5(write);
6034 INS01 : ISS;
6035 NEON_FP : S5;
6036 %}
6037
6038 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6039 %{
6040 single_instruction;
6041 src : S1(read);
6042 dst : S5(write);
6043 INS01 : ISS;
6044 NEON_FP : S5;
6045 %}
6046
6047 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6048 %{
6049 single_instruction;
6050 src : S1(read);
6051 dst : S5(write);
6052 INS01 : ISS;
6053 NEON_FP : S5;
6054 %}
6055
6056 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6057 %{
6058 single_instruction;
6059 src : S1(read);
6060 dst : S5(write);
6061 INS01 : ISS;
6062 NEON_FP : S5;
6063 %}
6064
6065 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6066 %{
6067 single_instruction;
6068 src : S1(read);
6069 dst : S5(write);
6070 INS01 : ISS;
6071 NEON_FP : S5;
6072 %}
6073
6074 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6075 %{
6076 single_instruction;
6077 src1 : S1(read);
6078 src2 : S2(read);
6079 dst : S5(write);
6080 INS0 : ISS;
6081 NEON_FP : S5;
6082 %}
6083
6084 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6085 %{
6086 single_instruction;
6087 src1 : S1(read);
6088 src2 : S2(read);
6089 dst : S5(write);
6090 INS0 : ISS;
6091 NEON_FP : S5;
6092 %}
6093
6094 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6095 %{
6096 single_instruction;
6097 cr : S1(read);
6098 src1 : S1(read);
6099 src2 : S1(read);
6100 dst : S3(write);
6101 INS01 : ISS;
6102 NEON_FP : S3;
6103 %}
6104
6105 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6106 %{
6107 single_instruction;
6108 cr : S1(read);
6109 src1 : S1(read);
6110 src2 : S1(read);
6111 dst : S3(write);
6112 INS01 : ISS;
6113 NEON_FP : S3;
6114 %}
6115
6116 pipe_class fp_imm_s(vRegF dst)
6117 %{
6118 single_instruction;
6119 dst : S3(write);
6120 INS01 : ISS;
6121 NEON_FP : S3;
6122 %}
6123
6124 pipe_class fp_imm_d(vRegD dst)
6125 %{
6126 single_instruction;
6127 dst : S3(write);
6128 INS01 : ISS;
6129 NEON_FP : S3;
6130 %}
6131
6132 pipe_class fp_load_constant_s(vRegF dst)
6133 %{
6134 single_instruction;
6135 dst : S4(write);
6136 INS01 : ISS;
6137 NEON_FP : S4;
6138 %}
6139
6140 pipe_class fp_load_constant_d(vRegD dst)
6141 %{
6142 single_instruction;
6143 dst : S4(write);
6144 INS01 : ISS;
6145 NEON_FP : S4;
6146 %}
6147
6148 //------- Integer ALU operations --------------------------
6149
6150 // Integer ALU reg-reg operation
6151 // Operands needed in EX1, result generated in EX2
6152 // Eg. ADD x0, x1, x2
6153 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6154 %{
6155 single_instruction;
6156 dst : EX2(write);
6157 src1 : EX1(read);
6158 src2 : EX1(read);
6159 INS01 : ISS; // Dual issue as instruction 0 or 1
6160 ALU : EX2;
6161 %}
6162
6163 // Integer ALU reg-reg operation with constant shift
6164 // Shifted register must be available in LATE_ISS instead of EX1
6165 // Eg. ADD x0, x1, x2, LSL #2
6166 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6167 %{
6168 single_instruction;
6169 dst : EX2(write);
6170 src1 : EX1(read);
6171 src2 : ISS(read);
6172 INS01 : ISS;
6173 ALU : EX2;
6174 %}
6175
6176 // Integer ALU reg operation with constant shift
6177 // Eg. LSL x0, x1, #shift
6178 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6179 %{
6180 single_instruction;
6181 dst : EX2(write);
6182 src1 : ISS(read);
6183 INS01 : ISS;
6184 ALU : EX2;
6185 %}
6186
6187 // Integer ALU reg-reg operation with variable shift
6188 // Both operands must be available in LATE_ISS instead of EX1
6189 // Result is available in EX1 instead of EX2
6190 // Eg. LSLV x0, x1, x2
6191 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6192 %{
6193 single_instruction;
6194 dst : EX1(write);
6195 src1 : ISS(read);
6196 src2 : ISS(read);
6197 INS01 : ISS;
6198 ALU : EX1;
6199 %}
6200
6201 // Integer ALU reg-reg operation with extract
6202 // As for _vshift above, but result generated in EX2
6203 // Eg. EXTR x0, x1, x2, #N
6204 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6205 %{
6206 single_instruction;
6207 dst : EX2(write);
6208 src1 : ISS(read);
6209 src2 : ISS(read);
6210 INS1 : ISS; // Can only dual issue as Instruction 1
6211 ALU : EX1;
6212 %}
6213
6214 // Integer ALU reg operation
6215 // Eg. NEG x0, x1
6216 pipe_class ialu_reg(iRegI dst, iRegI src)
6217 %{
6218 single_instruction;
6219 dst : EX2(write);
6220 src : EX1(read);
6221 INS01 : ISS;
6222 ALU : EX2;
6223 %}
6224
6225 // Integer ALU reg mmediate operation
6226 // Eg. ADD x0, x1, #N
6227 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6228 %{
6229 single_instruction;
6230 dst : EX2(write);
6231 src1 : EX1(read);
6232 INS01 : ISS;
6233 ALU : EX2;
6234 %}
6235
6236 // Integer ALU immediate operation (no source operands)
6237 // Eg. MOV x0, #N
6238 pipe_class ialu_imm(iRegI dst)
6239 %{
6240 single_instruction;
6241 dst : EX1(write);
6242 INS01 : ISS;
6243 ALU : EX1;
6244 %}
6245
6246 //------- Compare operation -------------------------------
6247
6248 // Compare reg-reg
6249 // Eg. CMP x0, x1
6250 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6251 %{
6252 single_instruction;
6253 // fixed_latency(16);
6254 cr : EX2(write);
6255 op1 : EX1(read);
6256 op2 : EX1(read);
6257 INS01 : ISS;
6258 ALU : EX2;
6259 %}
6260
6261 // Compare reg-reg
6262 // Eg. CMP x0, #N
6263 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6264 %{
6265 single_instruction;
6266 // fixed_latency(16);
6267 cr : EX2(write);
6268 op1 : EX1(read);
6269 INS01 : ISS;
6270 ALU : EX2;
6271 %}
6272
6273 //------- Conditional instructions ------------------------
6274
6275 // Conditional no operands
6276 // Eg. CSINC x0, zr, zr, <cond>
6277 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6278 %{
6279 single_instruction;
6280 cr : EX1(read);
6281 dst : EX2(write);
6282 INS01 : ISS;
6283 ALU : EX2;
6284 %}
6285
6286 // Conditional 2 operand
6287 // EG. CSEL X0, X1, X2, <cond>
6288 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6289 %{
6290 single_instruction;
6291 cr : EX1(read);
6292 src1 : EX1(read);
6293 src2 : EX1(read);
6294 dst : EX2(write);
6295 INS01 : ISS;
6296 ALU : EX2;
6297 %}
6298
6299 // Conditional 2 operand
6300 // EG. CSEL X0, X1, X2, <cond>
6301 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6302 %{
6303 single_instruction;
6304 cr : EX1(read);
6305 src : EX1(read);
6306 dst : EX2(write);
6307 INS01 : ISS;
6308 ALU : EX2;
6309 %}
6310
6311 //------- Multiply pipeline operations --------------------
6312
6313 // Multiply reg-reg
6314 // Eg. MUL w0, w1, w2
6315 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6316 %{
6317 single_instruction;
6318 dst : WR(write);
6319 src1 : ISS(read);
6320 src2 : ISS(read);
6321 INS01 : ISS;
6322 MAC : WR;
6323 %}
6324
6325 // Multiply accumulate
6326 // Eg. MADD w0, w1, w2, w3
6327 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6328 %{
6329 single_instruction;
6330 dst : WR(write);
6331 src1 : ISS(read);
6332 src2 : ISS(read);
6333 src3 : ISS(read);
6334 INS01 : ISS;
6335 MAC : WR;
6336 %}
6337
6338 // Eg. MUL w0, w1, w2
6339 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6340 %{
6341 single_instruction;
6342 fixed_latency(3); // Maximum latency for 64 bit mul
6343 dst : WR(write);
6344 src1 : ISS(read);
6345 src2 : ISS(read);
6346 INS01 : ISS;
6347 MAC : WR;
6348 %}
6349
6350 // Multiply accumulate
6351 // Eg. MADD w0, w1, w2, w3
6352 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6353 %{
6354 single_instruction;
6355 fixed_latency(3); // Maximum latency for 64 bit mul
6356 dst : WR(write);
6357 src1 : ISS(read);
6358 src2 : ISS(read);
6359 src3 : ISS(read);
6360 INS01 : ISS;
6361 MAC : WR;
6362 %}
6363
6364 //------- Divide pipeline operations --------------------
6365
6366 // Eg. SDIV w0, w1, w2
6367 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6368 %{
6369 single_instruction;
6370 fixed_latency(8); // Maximum latency for 32 bit divide
6371 dst : WR(write);
6372 src1 : ISS(read);
6373 src2 : ISS(read);
6374 INS0 : ISS; // Can only dual issue as instruction 0
6375 DIV : WR;
6376 %}
6377
6378 // Eg. SDIV x0, x1, x2
6379 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6380 %{
6381 single_instruction;
6382 fixed_latency(16); // Maximum latency for 64 bit divide
6383 dst : WR(write);
6384 src1 : ISS(read);
6385 src2 : ISS(read);
6386 INS0 : ISS; // Can only dual issue as instruction 0
6387 DIV : WR;
6388 %}
6389
6390 //------- Load pipeline operations ------------------------
6391
6392 // Load - prefetch
6393 // Eg. PFRM <mem>
6394 pipe_class iload_prefetch(memory mem)
6395 %{
6396 single_instruction;
6397 mem : ISS(read);
6398 INS01 : ISS;
6399 LDST : WR;
6400 %}
6401
6402 // Load - reg, mem
6403 // Eg. LDR x0, <mem>
6404 pipe_class iload_reg_mem(iRegI dst, memory mem)
6405 %{
6406 single_instruction;
6407 dst : WR(write);
6408 mem : ISS(read);
6409 INS01 : ISS;
6410 LDST : WR;
6411 %}
6412
6413 // Load - reg, reg
6414 // Eg. LDR x0, [sp, x1]
6415 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6416 %{
6417 single_instruction;
6418 dst : WR(write);
6419 src : ISS(read);
6420 INS01 : ISS;
6421 LDST : WR;
6422 %}
6423
6424 //------- Store pipeline operations -----------------------
6425
6426 // Store - zr, mem
6427 // Eg. STR zr, <mem>
6428 pipe_class istore_mem(memory mem)
6429 %{
6430 single_instruction;
6431 mem : ISS(read);
6432 INS01 : ISS;
6433 LDST : WR;
6434 %}
6435
6436 // Store - reg, mem
6437 // Eg. STR x0, <mem>
6438 pipe_class istore_reg_mem(iRegI src, memory mem)
6439 %{
6440 single_instruction;
6441 mem : ISS(read);
6442 src : EX2(read);
6443 INS01 : ISS;
6444 LDST : WR;
6445 %}
6446
6447 // Store - reg, reg
6448 // Eg. STR x0, [sp, x1]
6449 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6450 %{
6451 single_instruction;
6452 dst : ISS(read);
6453 src : EX2(read);
6454 INS01 : ISS;
6455 LDST : WR;
6456 %}
6457
6458 //------- Store pipeline operations -----------------------
6459
6460 // Branch
6461 pipe_class pipe_branch()
6462 %{
6463 single_instruction;
6464 INS01 : ISS;
6465 BRANCH : EX1;
6466 %}
6467
6468 // Conditional branch
6469 pipe_class pipe_branch_cond(rFlagsReg cr)
6470 %{
6471 single_instruction;
6472 cr : EX1(read);
6473 INS01 : ISS;
6474 BRANCH : EX1;
6475 %}
6476
6477 // Compare & Branch
6478 // EG. CBZ/CBNZ
6479 pipe_class pipe_cmp_branch(iRegI op1)
6480 %{
6481 single_instruction;
6482 op1 : EX1(read);
6483 INS01 : ISS;
6484 BRANCH : EX1;
6485 %}
6486
6487 //------- Synchronisation operations ----------------------
6488
6489 // Any operation requiring serialization.
6490 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6491 pipe_class pipe_serial()
6492 %{
6493 single_instruction;
6494 force_serialization;
6495 fixed_latency(16);
6496 INS01 : ISS(2); // Cannot dual issue with any other instruction
6497 LDST : WR;
6498 %}
6499
6500 // Generic big/slow expanded idiom - also serialized
6501 pipe_class pipe_slow()
6502 %{
6503 instruction_count(10);
6504 multiple_bundles;
6505 force_serialization;
6506 fixed_latency(16);
6507 INS01 : ISS(2); // Cannot dual issue with any other instruction
6508 LDST : WR;
6509 %}
6510
6511 // Empty pipeline class
6512 pipe_class pipe_class_empty()
6513 %{
6514 single_instruction;
6515 fixed_latency(0);
6516 %}
6517
6518 // Default pipeline class.
6519 pipe_class pipe_class_default()
6520 %{
6521 single_instruction;
6522 fixed_latency(2);
6523 %}
6524
6525 // Pipeline class for compares.
6526 pipe_class pipe_class_compare()
6527 %{
6528 single_instruction;
6529 fixed_latency(16);
6530 %}
6531
6532 // Pipeline class for memory operations.
6533 pipe_class pipe_class_memory()
6534 %{
6535 single_instruction;
6536 fixed_latency(16);
6537 %}
6538
6539 // Pipeline class for call.
6540 pipe_class pipe_class_call()
6541 %{
6542 single_instruction;
6543 fixed_latency(100);
6544 %}
6545
6546 // Define the class for the Nop node.
6547 define %{
6548 MachNop = pipe_class_empty;
6549 %}
6550
6551 %}
6552 //----------INSTRUCTIONS-------------------------------------------------------
6553 //
6554 // match -- States which machine-independent subtree may be replaced
6555 // by this instruction.
6556 // ins_cost -- The estimated cost of this instruction is used by instruction
6557 // selection to identify a minimum cost tree of machine
6558 // instructions that matches a tree of machine-independent
6559 // instructions.
6560 // format -- A string providing the disassembly for this instruction.
6561 // The value of an instruction's operand may be inserted
6562 // by referring to it with a '$' prefix.
6563 // opcode -- Three instruction opcodes may be provided. These are referred
6564 // to within an encode class as $primary, $secondary, and $tertiary
6565 // rrspectively. The primary opcode is commonly used to
6566 // indicate the type of machine instruction, while secondary
6567 // and tertiary are often used for prefix options or addressing
6568 // modes.
6569 // ins_encode -- A list of encode classes with parameters. The encode class
6570 // name must have been defined in an 'enc_class' specification
6571 // in the encode section of the architecture description.
6572
6573 // ============================================================================
6574 // Memory (Load/Store) Instructions
6575
6576 // Load Instructions
6577
6578 // Load Byte (8 bit signed)
6579 instruct loadB(iRegINoSp dst, memory1 mem)
6580 %{
6581 match(Set dst (LoadB mem));
6582 predicate(!needs_acquiring_load(n));
6583
6584 ins_cost(4 * INSN_COST);
6585 format %{ "ldrsbw $dst, $mem\t# byte" %}
6586
6587 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6588
6589 ins_pipe(iload_reg_mem);
6590 %}
6591
6592 // Load Byte (8 bit signed) into long
6593 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6594 %{
6595 match(Set dst (ConvI2L (LoadB mem)));
6596 predicate(!needs_acquiring_load(n->in(1)));
6597
6598 ins_cost(4 * INSN_COST);
6599 format %{ "ldrsb $dst, $mem\t# byte" %}
6600
6601 ins_encode(aarch64_enc_ldrsb(dst, mem));
6602
6603 ins_pipe(iload_reg_mem);
6604 %}
6605
6606 // Load Byte (8 bit unsigned)
6607 instruct loadUB(iRegINoSp dst, memory1 mem)
6608 %{
6609 match(Set dst (LoadUB mem));
6610 predicate(!needs_acquiring_load(n));
6611
6612 ins_cost(4 * INSN_COST);
6613 format %{ "ldrbw $dst, $mem\t# byte" %}
6614
6615 ins_encode(aarch64_enc_ldrb(dst, mem));
6616
6617 ins_pipe(iload_reg_mem);
6618 %}
6619
6620 // Load Byte (8 bit unsigned) into long
6621 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6622 %{
6623 match(Set dst (ConvI2L (LoadUB mem)));
6624 predicate(!needs_acquiring_load(n->in(1)));
6625
6626 ins_cost(4 * INSN_COST);
6627 format %{ "ldrb $dst, $mem\t# byte" %}
6628
6629 ins_encode(aarch64_enc_ldrb(dst, mem));
6630
6631 ins_pipe(iload_reg_mem);
6632 %}
6633
6634 // Load Short (16 bit signed)
6635 instruct loadS(iRegINoSp dst, memory2 mem)
6636 %{
6637 match(Set dst (LoadS mem));
6638 predicate(!needs_acquiring_load(n));
6639
6640 ins_cost(4 * INSN_COST);
6641 format %{ "ldrshw $dst, $mem\t# short" %}
6642
6643 ins_encode(aarch64_enc_ldrshw(dst, mem));
6644
6645 ins_pipe(iload_reg_mem);
6646 %}
6647
6648 // Load Short (16 bit signed) into long
6649 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6650 %{
6651 match(Set dst (ConvI2L (LoadS mem)));
6652 predicate(!needs_acquiring_load(n->in(1)));
6653
6654 ins_cost(4 * INSN_COST);
6655 format %{ "ldrsh $dst, $mem\t# short" %}
6656
6657 ins_encode(aarch64_enc_ldrsh(dst, mem));
6658
6659 ins_pipe(iload_reg_mem);
6660 %}
6661
6662 // Load Char (16 bit unsigned)
6663 instruct loadUS(iRegINoSp dst, memory2 mem)
6664 %{
6665 match(Set dst (LoadUS mem));
6666 predicate(!needs_acquiring_load(n));
6667
6668 ins_cost(4 * INSN_COST);
6669 format %{ "ldrh $dst, $mem\t# short" %}
6670
6671 ins_encode(aarch64_enc_ldrh(dst, mem));
6672
6673 ins_pipe(iload_reg_mem);
6674 %}
6675
6676 // Load Short/Char (16 bit unsigned) into long
6677 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6678 %{
6679 match(Set dst (ConvI2L (LoadUS mem)));
6680 predicate(!needs_acquiring_load(n->in(1)));
6681
6682 ins_cost(4 * INSN_COST);
6683 format %{ "ldrh $dst, $mem\t# short" %}
6684
6685 ins_encode(aarch64_enc_ldrh(dst, mem));
6686
6687 ins_pipe(iload_reg_mem);
6688 %}
6689
6690 // Load Integer (32 bit signed)
6691 instruct loadI(iRegINoSp dst, memory4 mem)
6692 %{
6693 match(Set dst (LoadI mem));
6694 predicate(!needs_acquiring_load(n));
6695
6696 ins_cost(4 * INSN_COST);
6697 format %{ "ldrw $dst, $mem\t# int" %}
6698
6699 ins_encode(aarch64_enc_ldrw(dst, mem));
6700
6701 ins_pipe(iload_reg_mem);
6702 %}
6703
6704 // Load Integer (32 bit signed) into long
6705 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6706 %{
6707 match(Set dst (ConvI2L (LoadI mem)));
6708 predicate(!needs_acquiring_load(n->in(1)));
6709
6710 ins_cost(4 * INSN_COST);
6711 format %{ "ldrsw $dst, $mem\t# int" %}
6712
6713 ins_encode(aarch64_enc_ldrsw(dst, mem));
6714
6715 ins_pipe(iload_reg_mem);
6716 %}
6717
6718 // Load Integer (32 bit unsigned) into long
6719 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6720 %{
6721 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6722 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6723
6724 ins_cost(4 * INSN_COST);
6725 format %{ "ldrw $dst, $mem\t# int" %}
6726
6727 ins_encode(aarch64_enc_ldrw(dst, mem));
6728
6729 ins_pipe(iload_reg_mem);
6730 %}
6731
6732 // Load Long (64 bit signed)
6733 instruct loadL(iRegLNoSp dst, memory8 mem)
6734 %{
6735 match(Set dst (LoadL mem));
6736 predicate(!needs_acquiring_load(n));
6737
6738 ins_cost(4 * INSN_COST);
6739 format %{ "ldr $dst, $mem\t# int" %}
6740
6741 ins_encode(aarch64_enc_ldr(dst, mem));
6742
6743 ins_pipe(iload_reg_mem);
6744 %}
6745
6746 // Load Range
6747 instruct loadRange(iRegINoSp dst, memory4 mem)
6748 %{
6749 match(Set dst (LoadRange mem));
6750
6751 ins_cost(4 * INSN_COST);
6752 format %{ "ldrw $dst, $mem\t# range" %}
6753
6754 ins_encode(aarch64_enc_ldrw(dst, mem));
6755
6756 ins_pipe(iload_reg_mem);
6757 %}
6758
6759 // Load Pointer
6760 instruct loadP(iRegPNoSp dst, memory8 mem)
6761 %{
6762 match(Set dst (LoadP mem));
6763 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6764
6765 ins_cost(4 * INSN_COST);
6766 format %{ "ldr $dst, $mem\t# ptr" %}
6767
6768 ins_encode(aarch64_enc_ldr(dst, mem));
6769
6770 ins_pipe(iload_reg_mem);
6771 %}
6772
6773 // Load Compressed Pointer
6774 instruct loadN(iRegNNoSp dst, memory4 mem)
6775 %{
6776 match(Set dst (LoadN mem));
6777 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6778
6779 ins_cost(4 * INSN_COST);
6780 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6781
6782 ins_encode(aarch64_enc_ldrw(dst, mem));
6783
6784 ins_pipe(iload_reg_mem);
6785 %}
6786
6787 // Load Klass Pointer
6788 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6789 %{
6790 match(Set dst (LoadKlass mem));
6791 predicate(!needs_acquiring_load(n));
6792
6793 ins_cost(4 * INSN_COST);
6794 format %{ "ldr $dst, $mem\t# class" %}
6795
6796 ins_encode(aarch64_enc_ldr(dst, mem));
6797
6798 ins_pipe(iload_reg_mem);
6799 %}
6800
6801 // Load Narrow Klass Pointer
6802 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6803 %{
6804 match(Set dst (LoadNKlass mem));
6805 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6806
6807 ins_cost(4 * INSN_COST);
6808 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6809
6810 ins_encode(aarch64_enc_ldrw(dst, mem));
6811
6812 ins_pipe(iload_reg_mem);
6813 %}
6814
6815 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6816 %{
6817 match(Set dst (LoadNKlass mem));
6818 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6819
6820 ins_cost(4 * INSN_COST);
6821 format %{
6822 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6823 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6824 %}
6825 ins_encode %{
6826 // inlined aarch64_enc_ldrw
6827 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6828 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6829 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6830 %}
6831 ins_pipe(iload_reg_mem);
6832 %}
6833
6834 // Load Float
6835 instruct loadF(vRegF dst, memory4 mem)
6836 %{
6837 match(Set dst (LoadF mem));
6838 predicate(!needs_acquiring_load(n));
6839
6840 ins_cost(4 * INSN_COST);
6841 format %{ "ldrs $dst, $mem\t# float" %}
6842
6843 ins_encode( aarch64_enc_ldrs(dst, mem) );
6844
6845 ins_pipe(pipe_class_memory);
6846 %}
6847
6848 // Load Double
6849 instruct loadD(vRegD dst, memory8 mem)
6850 %{
6851 match(Set dst (LoadD mem));
6852 predicate(!needs_acquiring_load(n));
6853
6854 ins_cost(4 * INSN_COST);
6855 format %{ "ldrd $dst, $mem\t# double" %}
6856
6857 ins_encode( aarch64_enc_ldrd(dst, mem) );
6858
6859 ins_pipe(pipe_class_memory);
6860 %}
6861
6862
6863 // Load Int Constant
6864 instruct loadConI(iRegINoSp dst, immI src)
6865 %{
6866 match(Set dst src);
6867
6868 ins_cost(INSN_COST);
6869 format %{ "mov $dst, $src\t# int" %}
6870
6871 ins_encode( aarch64_enc_movw_imm(dst, src) );
6872
6873 ins_pipe(ialu_imm);
6874 %}
6875
6876 // Load Long Constant
6877 instruct loadConL(iRegLNoSp dst, immL src)
6878 %{
6879 match(Set dst src);
6880
6881 ins_cost(INSN_COST);
6882 format %{ "mov $dst, $src\t# long" %}
6883
6884 ins_encode( aarch64_enc_mov_imm(dst, src) );
6885
6886 ins_pipe(ialu_imm);
6887 %}
6888
6889 // Load Pointer Constant
6890
6891 instruct loadConP(iRegPNoSp dst, immP con)
6892 %{
6893 match(Set dst con);
6894
6895 ins_cost(INSN_COST * 4);
6896 format %{
6897 "mov $dst, $con\t# ptr"
6898 %}
6899
6900 ins_encode(aarch64_enc_mov_p(dst, con));
6901
6902 ins_pipe(ialu_imm);
6903 %}
6904
6905 // Load Null Pointer Constant
6906
6907 instruct loadConP0(iRegPNoSp dst, immP0 con)
6908 %{
6909 match(Set dst con);
6910
6911 ins_cost(INSN_COST);
6912 format %{ "mov $dst, $con\t# nullptr ptr" %}
6913
6914 ins_encode(aarch64_enc_mov_p0(dst, con));
6915
6916 ins_pipe(ialu_imm);
6917 %}
6918
6919 // Load Pointer Constant One
6920
6921 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6922 %{
6923 match(Set dst con);
6924
6925 ins_cost(INSN_COST);
6926 format %{ "mov $dst, $con\t# nullptr ptr" %}
6927
6928 ins_encode(aarch64_enc_mov_p1(dst, con));
6929
6930 ins_pipe(ialu_imm);
6931 %}
6932
6933 // Load Narrow Pointer Constant
6934
6935 instruct loadConN(iRegNNoSp dst, immN con)
6936 %{
6937 match(Set dst con);
6938
6939 ins_cost(INSN_COST * 4);
6940 format %{ "mov $dst, $con\t# compressed ptr" %}
6941
6942 ins_encode(aarch64_enc_mov_n(dst, con));
6943
6944 ins_pipe(ialu_imm);
6945 %}
6946
6947 // Load Narrow Null Pointer Constant
6948
6949 instruct loadConN0(iRegNNoSp dst, immN0 con)
6950 %{
6951 match(Set dst con);
6952
6953 ins_cost(INSN_COST);
6954 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6955
6956 ins_encode(aarch64_enc_mov_n0(dst, con));
6957
6958 ins_pipe(ialu_imm);
6959 %}
6960
6961 // Load Narrow Klass Constant
6962
6963 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6964 %{
6965 match(Set dst con);
6966
6967 ins_cost(INSN_COST);
6968 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6969
6970 ins_encode(aarch64_enc_mov_nk(dst, con));
6971
6972 ins_pipe(ialu_imm);
6973 %}
6974
6975 // Load Packed Float Constant
6976
6977 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6978 match(Set dst con);
6979 ins_cost(INSN_COST * 4);
6980 format %{ "fmovs $dst, $con"%}
6981 ins_encode %{
6982 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6983 %}
6984
6985 ins_pipe(fp_imm_s);
6986 %}
6987
6988 // Load Float Constant
6989
6990 instruct loadConF(vRegF dst, immF con) %{
6991 match(Set dst con);
6992
6993 ins_cost(INSN_COST * 4);
6994
6995 format %{
6996 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6997 %}
6998
6999 ins_encode %{
7000 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7001 %}
7002
7003 ins_pipe(fp_load_constant_s);
7004 %}
7005
7006 // Load Packed Double Constant
7007
7008 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7009 match(Set dst con);
7010 ins_cost(INSN_COST);
7011 format %{ "fmovd $dst, $con"%}
7012 ins_encode %{
7013 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7014 %}
7015
7016 ins_pipe(fp_imm_d);
7017 %}
7018
7019 // Load Double Constant
7020
7021 instruct loadConD(vRegD dst, immD con) %{
7022 match(Set dst con);
7023
7024 ins_cost(INSN_COST * 5);
7025 format %{
7026 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7027 %}
7028
7029 ins_encode %{
7030 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7031 %}
7032
7033 ins_pipe(fp_load_constant_d);
7034 %}
7035
7036 // Load Half Float Constant
7037 instruct loadConH(vRegF dst, immH con) %{
7038 match(Set dst con);
7039 format %{ "mov rscratch1, $con\n\t"
7040 "fmov $dst, rscratch1"
7041 %}
7042 ins_encode %{
7043 __ movw(rscratch1, (uint32_t)$con$$constant);
7044 __ fmovs($dst$$FloatRegister, rscratch1);
7045 %}
7046 ins_pipe(pipe_class_default);
7047 %}
7048
7049 // Store Instructions
7050
7051 // Store Byte
7052 instruct storeB(iRegIorL2I src, memory1 mem)
7053 %{
7054 match(Set mem (StoreB mem src));
7055 predicate(!needs_releasing_store(n));
7056
7057 ins_cost(INSN_COST);
7058 format %{ "strb $src, $mem\t# byte" %}
7059
7060 ins_encode(aarch64_enc_strb(src, mem));
7061
7062 ins_pipe(istore_reg_mem);
7063 %}
7064
7065
7066 instruct storeimmB0(immI0 zero, memory1 mem)
7067 %{
7068 match(Set mem (StoreB mem zero));
7069 predicate(!needs_releasing_store(n));
7070
7071 ins_cost(INSN_COST);
7072 format %{ "strb rscractch2, $mem\t# byte" %}
7073
7074 ins_encode(aarch64_enc_strb0(mem));
7075
7076 ins_pipe(istore_mem);
7077 %}
7078
7079 // Store Char/Short
7080 instruct storeC(iRegIorL2I src, memory2 mem)
7081 %{
7082 match(Set mem (StoreC mem src));
7083 predicate(!needs_releasing_store(n));
7084
7085 ins_cost(INSN_COST);
7086 format %{ "strh $src, $mem\t# short" %}
7087
7088 ins_encode(aarch64_enc_strh(src, mem));
7089
7090 ins_pipe(istore_reg_mem);
7091 %}
7092
7093 instruct storeimmC0(immI0 zero, memory2 mem)
7094 %{
7095 match(Set mem (StoreC mem zero));
7096 predicate(!needs_releasing_store(n));
7097
7098 ins_cost(INSN_COST);
7099 format %{ "strh zr, $mem\t# short" %}
7100
7101 ins_encode(aarch64_enc_strh0(mem));
7102
7103 ins_pipe(istore_mem);
7104 %}
7105
7106 // Store Integer
7107
7108 instruct storeI(iRegIorL2I src, memory4 mem)
7109 %{
7110 match(Set mem(StoreI mem src));
7111 predicate(!needs_releasing_store(n));
7112
7113 ins_cost(INSN_COST);
7114 format %{ "strw $src, $mem\t# int" %}
7115
7116 ins_encode(aarch64_enc_strw(src, mem));
7117
7118 ins_pipe(istore_reg_mem);
7119 %}
7120
7121 instruct storeimmI0(immI0 zero, memory4 mem)
7122 %{
7123 match(Set mem(StoreI mem zero));
7124 predicate(!needs_releasing_store(n));
7125
7126 ins_cost(INSN_COST);
7127 format %{ "strw zr, $mem\t# int" %}
7128
7129 ins_encode(aarch64_enc_strw0(mem));
7130
7131 ins_pipe(istore_mem);
7132 %}
7133
7134 // Store Long (64 bit signed)
7135 instruct storeL(iRegL src, memory8 mem)
7136 %{
7137 match(Set mem (StoreL mem src));
7138 predicate(!needs_releasing_store(n));
7139
7140 ins_cost(INSN_COST);
7141 format %{ "str $src, $mem\t# int" %}
7142
7143 ins_encode(aarch64_enc_str(src, mem));
7144
7145 ins_pipe(istore_reg_mem);
7146 %}
7147
7148 // Store Long (64 bit signed)
7149 instruct storeimmL0(immL0 zero, memory8 mem)
7150 %{
7151 match(Set mem (StoreL mem zero));
7152 predicate(!needs_releasing_store(n));
7153
7154 ins_cost(INSN_COST);
7155 format %{ "str zr, $mem\t# int" %}
7156
7157 ins_encode(aarch64_enc_str0(mem));
7158
7159 ins_pipe(istore_mem);
7160 %}
7161
7162 // Store Pointer
7163 instruct storeP(iRegP src, memory8 mem)
7164 %{
7165 match(Set mem (StoreP mem src));
7166 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7167
7168 ins_cost(INSN_COST);
7169 format %{ "str $src, $mem\t# ptr" %}
7170
7171 ins_encode(aarch64_enc_str(src, mem));
7172
7173 ins_pipe(istore_reg_mem);
7174 %}
7175
7176 // Store Pointer
7177 instruct storeimmP0(immP0 zero, memory8 mem)
7178 %{
7179 match(Set mem (StoreP mem zero));
7180 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7181
7182 ins_cost(INSN_COST);
7183 format %{ "str zr, $mem\t# ptr" %}
7184
7185 ins_encode(aarch64_enc_str0(mem));
7186
7187 ins_pipe(istore_mem);
7188 %}
7189
7190 // Store Compressed Pointer
7191 instruct storeN(iRegN src, memory4 mem)
7192 %{
7193 match(Set mem (StoreN mem src));
7194 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7195
7196 ins_cost(INSN_COST);
7197 format %{ "strw $src, $mem\t# compressed ptr" %}
7198
7199 ins_encode(aarch64_enc_strw(src, mem));
7200
7201 ins_pipe(istore_reg_mem);
7202 %}
7203
7204 instruct storeImmN0(immN0 zero, memory4 mem)
7205 %{
7206 match(Set mem (StoreN mem zero));
7207 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7208
7209 ins_cost(INSN_COST);
7210 format %{ "strw zr, $mem\t# compressed ptr" %}
7211
7212 ins_encode(aarch64_enc_strw0(mem));
7213
7214 ins_pipe(istore_mem);
7215 %}
7216
7217 // Store Float
7218 instruct storeF(vRegF src, memory4 mem)
7219 %{
7220 match(Set mem (StoreF mem src));
7221 predicate(!needs_releasing_store(n));
7222
7223 ins_cost(INSN_COST);
7224 format %{ "strs $src, $mem\t# float" %}
7225
7226 ins_encode( aarch64_enc_strs(src, mem) );
7227
7228 ins_pipe(pipe_class_memory);
7229 %}
7230
7231 // TODO
7232 // implement storeImmF0 and storeFImmPacked
7233
7234 // Store Double
7235 instruct storeD(vRegD src, memory8 mem)
7236 %{
7237 match(Set mem (StoreD mem src));
7238 predicate(!needs_releasing_store(n));
7239
7240 ins_cost(INSN_COST);
7241 format %{ "strd $src, $mem\t# double" %}
7242
7243 ins_encode( aarch64_enc_strd(src, mem) );
7244
7245 ins_pipe(pipe_class_memory);
7246 %}
7247
7248 // Store Compressed Klass Pointer
7249 instruct storeNKlass(iRegN src, memory4 mem)
7250 %{
7251 predicate(!needs_releasing_store(n));
7252 match(Set mem (StoreNKlass mem src));
7253
7254 ins_cost(INSN_COST);
7255 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7256
7257 ins_encode(aarch64_enc_strw(src, mem));
7258
7259 ins_pipe(istore_reg_mem);
7260 %}
7261
7262 // TODO
7263 // implement storeImmD0 and storeDImmPacked
7264
7265 // prefetch instructions
7266 // Must be safe to execute with invalid address (cannot fault).
7267
7268 instruct prefetchalloc( memory8 mem ) %{
7269 match(PrefetchAllocation mem);
7270
7271 ins_cost(INSN_COST);
7272 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7273
7274 ins_encode( aarch64_enc_prefetchw(mem) );
7275
7276 ins_pipe(iload_prefetch);
7277 %}
7278
7279 // ---------------- volatile loads and stores ----------------
7280
7281 // Load Byte (8 bit signed)
7282 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7283 %{
7284 match(Set dst (LoadB mem));
7285
7286 ins_cost(VOLATILE_REF_COST);
7287 format %{ "ldarsb $dst, $mem\t# byte" %}
7288
7289 ins_encode(aarch64_enc_ldarsb(dst, mem));
7290
7291 ins_pipe(pipe_serial);
7292 %}
7293
7294 // Load Byte (8 bit signed) into long
7295 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7296 %{
7297 match(Set dst (ConvI2L (LoadB mem)));
7298
7299 ins_cost(VOLATILE_REF_COST);
7300 format %{ "ldarsb $dst, $mem\t# byte" %}
7301
7302 ins_encode(aarch64_enc_ldarsb(dst, mem));
7303
7304 ins_pipe(pipe_serial);
7305 %}
7306
7307 // Load Byte (8 bit unsigned)
7308 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7309 %{
7310 match(Set dst (LoadUB mem));
7311
7312 ins_cost(VOLATILE_REF_COST);
7313 format %{ "ldarb $dst, $mem\t# byte" %}
7314
7315 ins_encode(aarch64_enc_ldarb(dst, mem));
7316
7317 ins_pipe(pipe_serial);
7318 %}
7319
7320 // Load Byte (8 bit unsigned) into long
7321 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7322 %{
7323 match(Set dst (ConvI2L (LoadUB mem)));
7324
7325 ins_cost(VOLATILE_REF_COST);
7326 format %{ "ldarb $dst, $mem\t# byte" %}
7327
7328 ins_encode(aarch64_enc_ldarb(dst, mem));
7329
7330 ins_pipe(pipe_serial);
7331 %}
7332
7333 // Load Short (16 bit signed)
7334 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7335 %{
7336 match(Set dst (LoadS mem));
7337
7338 ins_cost(VOLATILE_REF_COST);
7339 format %{ "ldarshw $dst, $mem\t# short" %}
7340
7341 ins_encode(aarch64_enc_ldarshw(dst, mem));
7342
7343 ins_pipe(pipe_serial);
7344 %}
7345
7346 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7347 %{
7348 match(Set dst (LoadUS mem));
7349
7350 ins_cost(VOLATILE_REF_COST);
7351 format %{ "ldarhw $dst, $mem\t# short" %}
7352
7353 ins_encode(aarch64_enc_ldarhw(dst, mem));
7354
7355 ins_pipe(pipe_serial);
7356 %}
7357
7358 // Load Short/Char (16 bit unsigned) into long
7359 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7360 %{
7361 match(Set dst (ConvI2L (LoadUS mem)));
7362
7363 ins_cost(VOLATILE_REF_COST);
7364 format %{ "ldarh $dst, $mem\t# short" %}
7365
7366 ins_encode(aarch64_enc_ldarh(dst, mem));
7367
7368 ins_pipe(pipe_serial);
7369 %}
7370
7371 // Load Short/Char (16 bit signed) into long
7372 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7373 %{
7374 match(Set dst (ConvI2L (LoadS mem)));
7375
7376 ins_cost(VOLATILE_REF_COST);
7377 format %{ "ldarh $dst, $mem\t# short" %}
7378
7379 ins_encode(aarch64_enc_ldarsh(dst, mem));
7380
7381 ins_pipe(pipe_serial);
7382 %}
7383
7384 // Load Integer (32 bit signed)
7385 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7386 %{
7387 match(Set dst (LoadI mem));
7388
7389 ins_cost(VOLATILE_REF_COST);
7390 format %{ "ldarw $dst, $mem\t# int" %}
7391
7392 ins_encode(aarch64_enc_ldarw(dst, mem));
7393
7394 ins_pipe(pipe_serial);
7395 %}
7396
7397 // Load Integer (32 bit unsigned) into long
7398 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7399 %{
7400 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7401
7402 ins_cost(VOLATILE_REF_COST);
7403 format %{ "ldarw $dst, $mem\t# int" %}
7404
7405 ins_encode(aarch64_enc_ldarw(dst, mem));
7406
7407 ins_pipe(pipe_serial);
7408 %}
7409
7410 // Load Long (64 bit signed)
7411 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7412 %{
7413 match(Set dst (LoadL mem));
7414
7415 ins_cost(VOLATILE_REF_COST);
7416 format %{ "ldar $dst, $mem\t# int" %}
7417
7418 ins_encode(aarch64_enc_ldar(dst, mem));
7419
7420 ins_pipe(pipe_serial);
7421 %}
7422
7423 // Load Pointer
7424 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7425 %{
7426 match(Set dst (LoadP mem));
7427 predicate(n->as_Load()->barrier_data() == 0);
7428
7429 ins_cost(VOLATILE_REF_COST);
7430 format %{ "ldar $dst, $mem\t# ptr" %}
7431
7432 ins_encode(aarch64_enc_ldar(dst, mem));
7433
7434 ins_pipe(pipe_serial);
7435 %}
7436
7437 // Load Compressed Pointer
7438 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7439 %{
7440 match(Set dst (LoadN mem));
7441 predicate(n->as_Load()->barrier_data() == 0);
7442
7443 ins_cost(VOLATILE_REF_COST);
7444 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7445
7446 ins_encode(aarch64_enc_ldarw(dst, mem));
7447
7448 ins_pipe(pipe_serial);
7449 %}
7450
7451 // Load Float
7452 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7453 %{
7454 match(Set dst (LoadF mem));
7455
7456 ins_cost(VOLATILE_REF_COST);
7457 format %{ "ldars $dst, $mem\t# float" %}
7458
7459 ins_encode( aarch64_enc_fldars(dst, mem) );
7460
7461 ins_pipe(pipe_serial);
7462 %}
7463
7464 // Load Double
7465 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7466 %{
7467 match(Set dst (LoadD mem));
7468
7469 ins_cost(VOLATILE_REF_COST);
7470 format %{ "ldard $dst, $mem\t# double" %}
7471
7472 ins_encode( aarch64_enc_fldard(dst, mem) );
7473
7474 ins_pipe(pipe_serial);
7475 %}
7476
7477 // Store Byte
7478 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7479 %{
7480 match(Set mem (StoreB mem src));
7481
7482 ins_cost(VOLATILE_REF_COST);
7483 format %{ "stlrb $src, $mem\t# byte" %}
7484
7485 ins_encode(aarch64_enc_stlrb(src, mem));
7486
7487 ins_pipe(pipe_class_memory);
7488 %}
7489
7490 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7491 %{
7492 match(Set mem (StoreB mem zero));
7493
7494 ins_cost(VOLATILE_REF_COST);
7495 format %{ "stlrb zr, $mem\t# byte" %}
7496
7497 ins_encode(aarch64_enc_stlrb0(mem));
7498
7499 ins_pipe(pipe_class_memory);
7500 %}
7501
7502 // Store Char/Short
7503 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7504 %{
7505 match(Set mem (StoreC mem src));
7506
7507 ins_cost(VOLATILE_REF_COST);
7508 format %{ "stlrh $src, $mem\t# short" %}
7509
7510 ins_encode(aarch64_enc_stlrh(src, mem));
7511
7512 ins_pipe(pipe_class_memory);
7513 %}
7514
7515 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7516 %{
7517 match(Set mem (StoreC mem zero));
7518
7519 ins_cost(VOLATILE_REF_COST);
7520 format %{ "stlrh zr, $mem\t# short" %}
7521
7522 ins_encode(aarch64_enc_stlrh0(mem));
7523
7524 ins_pipe(pipe_class_memory);
7525 %}
7526
7527 // Store Integer
7528
7529 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7530 %{
7531 match(Set mem(StoreI mem src));
7532
7533 ins_cost(VOLATILE_REF_COST);
7534 format %{ "stlrw $src, $mem\t# int" %}
7535
7536 ins_encode(aarch64_enc_stlrw(src, mem));
7537
7538 ins_pipe(pipe_class_memory);
7539 %}
7540
7541 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7542 %{
7543 match(Set mem(StoreI mem zero));
7544
7545 ins_cost(VOLATILE_REF_COST);
7546 format %{ "stlrw zr, $mem\t# int" %}
7547
7548 ins_encode(aarch64_enc_stlrw0(mem));
7549
7550 ins_pipe(pipe_class_memory);
7551 %}
7552
7553 // Store Long (64 bit signed)
7554 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7555 %{
7556 match(Set mem (StoreL mem src));
7557
7558 ins_cost(VOLATILE_REF_COST);
7559 format %{ "stlr $src, $mem\t# int" %}
7560
7561 ins_encode(aarch64_enc_stlr(src, mem));
7562
7563 ins_pipe(pipe_class_memory);
7564 %}
7565
7566 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7567 %{
7568 match(Set mem (StoreL mem zero));
7569
7570 ins_cost(VOLATILE_REF_COST);
7571 format %{ "stlr zr, $mem\t# int" %}
7572
7573 ins_encode(aarch64_enc_stlr0(mem));
7574
7575 ins_pipe(pipe_class_memory);
7576 %}
7577
7578 // Store Pointer
7579 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7580 %{
7581 match(Set mem (StoreP mem src));
7582 predicate(n->as_Store()->barrier_data() == 0);
7583
7584 ins_cost(VOLATILE_REF_COST);
7585 format %{ "stlr $src, $mem\t# ptr" %}
7586
7587 ins_encode(aarch64_enc_stlr(src, mem));
7588
7589 ins_pipe(pipe_class_memory);
7590 %}
7591
7592 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7593 %{
7594 match(Set mem (StoreP mem zero));
7595 predicate(n->as_Store()->barrier_data() == 0);
7596
7597 ins_cost(VOLATILE_REF_COST);
7598 format %{ "stlr zr, $mem\t# ptr" %}
7599
7600 ins_encode(aarch64_enc_stlr0(mem));
7601
7602 ins_pipe(pipe_class_memory);
7603 %}
7604
7605 // Store Compressed Pointer
7606 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7607 %{
7608 match(Set mem (StoreN mem src));
7609 predicate(n->as_Store()->barrier_data() == 0);
7610
7611 ins_cost(VOLATILE_REF_COST);
7612 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7613
7614 ins_encode(aarch64_enc_stlrw(src, mem));
7615
7616 ins_pipe(pipe_class_memory);
7617 %}
7618
7619 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7620 %{
7621 match(Set mem (StoreN mem zero));
7622 predicate(n->as_Store()->barrier_data() == 0);
7623
7624 ins_cost(VOLATILE_REF_COST);
7625 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7626
7627 ins_encode(aarch64_enc_stlrw0(mem));
7628
7629 ins_pipe(pipe_class_memory);
7630 %}
7631
7632 // Store Float
7633 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7634 %{
7635 match(Set mem (StoreF mem src));
7636
7637 ins_cost(VOLATILE_REF_COST);
7638 format %{ "stlrs $src, $mem\t# float" %}
7639
7640 ins_encode( aarch64_enc_fstlrs(src, mem) );
7641
7642 ins_pipe(pipe_class_memory);
7643 %}
7644
7645 // TODO
7646 // implement storeImmF0 and storeFImmPacked
7647
7648 // Store Double
7649 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7650 %{
7651 match(Set mem (StoreD mem src));
7652
7653 ins_cost(VOLATILE_REF_COST);
7654 format %{ "stlrd $src, $mem\t# double" %}
7655
7656 ins_encode( aarch64_enc_fstlrd(src, mem) );
7657
7658 ins_pipe(pipe_class_memory);
7659 %}
7660
7661 // ---------------- end of volatile loads and stores ----------------
7662
7663 instruct cacheWB(indirect addr)
7664 %{
7665 predicate(VM_Version::supports_data_cache_line_flush());
7666 match(CacheWB addr);
7667
7668 ins_cost(100);
7669 format %{"cache wb $addr" %}
7670 ins_encode %{
7671 assert($addr->index_position() < 0, "should be");
7672 assert($addr$$disp == 0, "should be");
7673 __ cache_wb(Address($addr$$base$$Register, 0));
7674 %}
7675 ins_pipe(pipe_slow); // XXX
7676 %}
7677
7678 instruct cacheWBPreSync()
7679 %{
7680 predicate(VM_Version::supports_data_cache_line_flush());
7681 match(CacheWBPreSync);
7682
7683 ins_cost(100);
7684 format %{"cache wb presync" %}
7685 ins_encode %{
7686 __ cache_wbsync(true);
7687 %}
7688 ins_pipe(pipe_slow); // XXX
7689 %}
7690
7691 instruct cacheWBPostSync()
7692 %{
7693 predicate(VM_Version::supports_data_cache_line_flush());
7694 match(CacheWBPostSync);
7695
7696 ins_cost(100);
7697 format %{"cache wb postsync" %}
7698 ins_encode %{
7699 __ cache_wbsync(false);
7700 %}
7701 ins_pipe(pipe_slow); // XXX
7702 %}
7703
7704 // ============================================================================
7705 // BSWAP Instructions
7706
7707 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7708 match(Set dst (ReverseBytesI src));
7709
7710 ins_cost(INSN_COST);
7711 format %{ "revw $dst, $src" %}
7712
7713 ins_encode %{
7714 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7715 %}
7716
7717 ins_pipe(ialu_reg);
7718 %}
7719
7720 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7721 match(Set dst (ReverseBytesL src));
7722
7723 ins_cost(INSN_COST);
7724 format %{ "rev $dst, $src" %}
7725
7726 ins_encode %{
7727 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7728 %}
7729
7730 ins_pipe(ialu_reg);
7731 %}
7732
7733 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7734 match(Set dst (ReverseBytesUS src));
7735
7736 ins_cost(INSN_COST);
7737 format %{ "rev16w $dst, $src" %}
7738
7739 ins_encode %{
7740 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7741 %}
7742
7743 ins_pipe(ialu_reg);
7744 %}
7745
7746 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7747 match(Set dst (ReverseBytesS src));
7748
7749 ins_cost(INSN_COST);
7750 format %{ "rev16w $dst, $src\n\t"
7751 "sbfmw $dst, $dst, #0, #15" %}
7752
7753 ins_encode %{
7754 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7755 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7756 %}
7757
7758 ins_pipe(ialu_reg);
7759 %}
7760
7761 // ============================================================================
7762 // Zero Count Instructions
7763
7764 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7765 match(Set dst (CountLeadingZerosI src));
7766
7767 ins_cost(INSN_COST);
7768 format %{ "clzw $dst, $src" %}
7769 ins_encode %{
7770 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7771 %}
7772
7773 ins_pipe(ialu_reg);
7774 %}
7775
7776 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7777 match(Set dst (CountLeadingZerosL src));
7778
7779 ins_cost(INSN_COST);
7780 format %{ "clz $dst, $src" %}
7781 ins_encode %{
7782 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7783 %}
7784
7785 ins_pipe(ialu_reg);
7786 %}
7787
7788 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7789 match(Set dst (CountTrailingZerosI src));
7790
7791 ins_cost(INSN_COST * 2);
7792 format %{ "rbitw $dst, $src\n\t"
7793 "clzw $dst, $dst" %}
7794 ins_encode %{
7795 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7796 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7797 %}
7798
7799 ins_pipe(ialu_reg);
7800 %}
7801
7802 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7803 match(Set dst (CountTrailingZerosL src));
7804
7805 ins_cost(INSN_COST * 2);
7806 format %{ "rbit $dst, $src\n\t"
7807 "clz $dst, $dst" %}
7808 ins_encode %{
7809 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7810 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7811 %}
7812
7813 ins_pipe(ialu_reg);
7814 %}
7815
7816 //---------- Population Count Instructions -------------------------------------
7817 //
7818
7819 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7820 match(Set dst (PopCountI src));
7821 effect(TEMP tmp);
7822 ins_cost(INSN_COST * 13);
7823
7824 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7825 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7826 "addv $tmp, $tmp\t# vector (8B)\n\t"
7827 "mov $dst, $tmp\t# vector (1D)" %}
7828 ins_encode %{
7829 __ fmovs($tmp$$FloatRegister, $src$$Register);
7830 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7831 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7832 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7833 %}
7834
7835 ins_pipe(pipe_class_default);
7836 %}
7837
7838 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7839 match(Set dst (PopCountI (LoadI mem)));
7840 effect(TEMP tmp);
7841 ins_cost(INSN_COST * 13);
7842
7843 format %{ "ldrs $tmp, $mem\n\t"
7844 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7845 "addv $tmp, $tmp\t# vector (8B)\n\t"
7846 "mov $dst, $tmp\t# vector (1D)" %}
7847 ins_encode %{
7848 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7849 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7850 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7851 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7852 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7853 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7854 %}
7855
7856 ins_pipe(pipe_class_default);
7857 %}
7858
7859 // Note: Long.bitCount(long) returns an int.
7860 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7861 match(Set dst (PopCountL src));
7862 effect(TEMP tmp);
7863 ins_cost(INSN_COST * 13);
7864
7865 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7866 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7867 "addv $tmp, $tmp\t# vector (8B)\n\t"
7868 "mov $dst, $tmp\t# vector (1D)" %}
7869 ins_encode %{
7870 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7871 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7872 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7873 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7874 %}
7875
7876 ins_pipe(pipe_class_default);
7877 %}
7878
7879 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7880 match(Set dst (PopCountL (LoadL mem)));
7881 effect(TEMP tmp);
7882 ins_cost(INSN_COST * 13);
7883
7884 format %{ "ldrd $tmp, $mem\n\t"
7885 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7886 "addv $tmp, $tmp\t# vector (8B)\n\t"
7887 "mov $dst, $tmp\t# vector (1D)" %}
7888 ins_encode %{
7889 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7890 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7891 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7892 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7893 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7894 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7895 %}
7896
7897 ins_pipe(pipe_class_default);
7898 %}
7899
7900 // ============================================================================
7901 // VerifyVectorAlignment Instruction
7902
7903 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7904 match(Set addr (VerifyVectorAlignment addr mask));
7905 effect(KILL cr);
7906 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7907 ins_encode %{
7908 Label Lskip;
7909 // check if masked bits of addr are zero
7910 __ tst($addr$$Register, $mask$$constant);
7911 __ br(Assembler::EQ, Lskip);
7912 __ stop("verify_vector_alignment found a misaligned vector memory access");
7913 __ bind(Lskip);
7914 %}
7915 ins_pipe(pipe_slow);
7916 %}
7917
7918 // ============================================================================
7919 // MemBar Instruction
7920
7921 instruct load_fence() %{
7922 match(LoadFence);
7923 ins_cost(VOLATILE_REF_COST);
7924
7925 format %{ "load_fence" %}
7926
7927 ins_encode %{
7928 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7929 %}
7930 ins_pipe(pipe_serial);
7931 %}
7932
7933 instruct unnecessary_membar_acquire() %{
7934 predicate(unnecessary_acquire(n));
7935 match(MemBarAcquire);
7936 ins_cost(0);
7937
7938 format %{ "membar_acquire (elided)" %}
7939
7940 ins_encode %{
7941 __ block_comment("membar_acquire (elided)");
7942 %}
7943
7944 ins_pipe(pipe_class_empty);
7945 %}
7946
7947 instruct membar_acquire() %{
7948 match(MemBarAcquire);
7949 ins_cost(VOLATILE_REF_COST);
7950
7951 format %{ "membar_acquire\n\t"
7952 "dmb ishld" %}
7953
7954 ins_encode %{
7955 __ block_comment("membar_acquire");
7956 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7957 %}
7958
7959 ins_pipe(pipe_serial);
7960 %}
7961
7962
7963 instruct membar_acquire_lock() %{
7964 match(MemBarAcquireLock);
7965 ins_cost(VOLATILE_REF_COST);
7966
7967 format %{ "membar_acquire_lock (elided)" %}
7968
7969 ins_encode %{
7970 __ block_comment("membar_acquire_lock (elided)");
7971 %}
7972
7973 ins_pipe(pipe_serial);
7974 %}
7975
7976 instruct store_fence() %{
7977 match(StoreFence);
7978 ins_cost(VOLATILE_REF_COST);
7979
7980 format %{ "store_fence" %}
7981
7982 ins_encode %{
7983 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7984 %}
7985 ins_pipe(pipe_serial);
7986 %}
7987
7988 instruct unnecessary_membar_release() %{
7989 predicate(unnecessary_release(n));
7990 match(MemBarRelease);
7991 ins_cost(0);
7992
7993 format %{ "membar_release (elided)" %}
7994
7995 ins_encode %{
7996 __ block_comment("membar_release (elided)");
7997 %}
7998 ins_pipe(pipe_serial);
7999 %}
8000
8001 instruct membar_release() %{
8002 match(MemBarRelease);
8003 ins_cost(VOLATILE_REF_COST);
8004
8005 format %{ "membar_release\n\t"
8006 "dmb ishst\n\tdmb ishld" %}
8007
8008 ins_encode %{
8009 __ block_comment("membar_release");
8010 // These will be merged if AlwaysMergeDMB is enabled.
8011 __ membar(Assembler::StoreStore);
8012 __ membar(Assembler::LoadStore);
8013 %}
8014 ins_pipe(pipe_serial);
8015 %}
8016
8017 instruct membar_storestore() %{
8018 match(MemBarStoreStore);
8019 match(StoreStoreFence);
8020 ins_cost(VOLATILE_REF_COST);
8021
8022 format %{ "MEMBAR-store-store" %}
8023
8024 ins_encode %{
8025 __ membar(Assembler::StoreStore);
8026 %}
8027 ins_pipe(pipe_serial);
8028 %}
8029
8030 instruct membar_release_lock() %{
8031 match(MemBarReleaseLock);
8032 ins_cost(VOLATILE_REF_COST);
8033
8034 format %{ "membar_release_lock (elided)" %}
8035
8036 ins_encode %{
8037 __ block_comment("membar_release_lock (elided)");
8038 %}
8039
8040 ins_pipe(pipe_serial);
8041 %}
8042
8043 instruct unnecessary_membar_volatile() %{
8044 predicate(unnecessary_volatile(n));
8045 match(MemBarVolatile);
8046 ins_cost(0);
8047
8048 format %{ "membar_volatile (elided)" %}
8049
8050 ins_encode %{
8051 __ block_comment("membar_volatile (elided)");
8052 %}
8053
8054 ins_pipe(pipe_serial);
8055 %}
8056
8057 instruct membar_volatile() %{
8058 match(MemBarVolatile);
8059 ins_cost(VOLATILE_REF_COST*100);
8060
8061 format %{ "membar_volatile\n\t"
8062 "dmb ish"%}
8063
8064 ins_encode %{
8065 __ block_comment("membar_volatile");
8066 __ membar(Assembler::StoreLoad);
8067 %}
8068
8069 ins_pipe(pipe_serial);
8070 %}
8071
8072 // ============================================================================
8073 // Cast/Convert Instructions
8074
8075 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8076 match(Set dst (CastX2P src));
8077
8078 ins_cost(INSN_COST);
8079 format %{ "mov $dst, $src\t# long -> ptr" %}
8080
8081 ins_encode %{
8082 if ($dst$$reg != $src$$reg) {
8083 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8084 }
8085 %}
8086
8087 ins_pipe(ialu_reg);
8088 %}
8089
8090 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8091 match(Set dst (CastI2N src));
8092
8093 ins_cost(INSN_COST);
8094 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8095
8096 ins_encode %{
8097 if ($dst$$reg != $src$$reg) {
8098 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8099 }
8100 %}
8101
8102 ins_pipe(ialu_reg);
8103 %}
8104
8105 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8106 match(Set dst (CastP2X src));
8107
8108 ins_cost(INSN_COST);
8109 format %{ "mov $dst, $src\t# ptr -> long" %}
8110
8111 ins_encode %{
8112 if ($dst$$reg != $src$$reg) {
8113 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8114 }
8115 %}
8116
8117 ins_pipe(ialu_reg);
8118 %}
8119
8120 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8121 match(Set dst (CastP2X src));
8122
8123 ins_cost(INSN_COST);
8124 format %{ "mov $dst, $src\t# ptr -> long" %}
8125
8126 ins_encode %{
8127 if ($dst$$reg != $src$$reg) {
8128 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8129 }
8130 %}
8131
8132 ins_pipe(ialu_reg);
8133 %}
8134
8135 // Convert oop into int for vectors alignment masking
8136 instruct convP2I(iRegINoSp dst, iRegP src) %{
8137 match(Set dst (ConvL2I (CastP2X src)));
8138
8139 ins_cost(INSN_COST);
8140 format %{ "movw $dst, $src\t# ptr -> int" %}
8141 ins_encode %{
8142 __ movw($dst$$Register, $src$$Register);
8143 %}
8144
8145 ins_pipe(ialu_reg);
8146 %}
8147
8148 // Convert compressed oop into int for vectors alignment masking
8149 // in case of 32bit oops (heap < 4Gb).
8150 instruct convN2I(iRegINoSp dst, iRegN src)
8151 %{
8152 predicate(CompressedOops::shift() == 0);
8153 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8154
8155 ins_cost(INSN_COST);
8156 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8157 ins_encode %{
8158 __ movw($dst$$Register, $src$$Register);
8159 %}
8160
8161 ins_pipe(ialu_reg);
8162 %}
8163
8164
8165 // Convert oop pointer into compressed form
8166 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8167 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8168 match(Set dst (EncodeP src));
8169 effect(KILL cr);
8170 ins_cost(INSN_COST * 3);
8171 format %{ "encode_heap_oop $dst, $src" %}
8172 ins_encode %{
8173 Register s = $src$$Register;
8174 Register d = $dst$$Register;
8175 __ encode_heap_oop(d, s);
8176 %}
8177 ins_pipe(ialu_reg);
8178 %}
8179
8180 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8181 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8182 match(Set dst (EncodeP src));
8183 ins_cost(INSN_COST * 3);
8184 format %{ "encode_heap_oop_not_null $dst, $src" %}
8185 ins_encode %{
8186 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8187 %}
8188 ins_pipe(ialu_reg);
8189 %}
8190
8191 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8192 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8193 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8194 match(Set dst (DecodeN src));
8195 ins_cost(INSN_COST * 3);
8196 format %{ "decode_heap_oop $dst, $src" %}
8197 ins_encode %{
8198 Register s = $src$$Register;
8199 Register d = $dst$$Register;
8200 __ decode_heap_oop(d, s);
8201 %}
8202 ins_pipe(ialu_reg);
8203 %}
8204
8205 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8206 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8207 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8208 match(Set dst (DecodeN src));
8209 ins_cost(INSN_COST * 3);
8210 format %{ "decode_heap_oop_not_null $dst, $src" %}
8211 ins_encode %{
8212 Register s = $src$$Register;
8213 Register d = $dst$$Register;
8214 __ decode_heap_oop_not_null(d, s);
8215 %}
8216 ins_pipe(ialu_reg);
8217 %}
8218
8219 // n.b. AArch64 implementations of encode_klass_not_null and
8220 // decode_klass_not_null do not modify the flags register so, unlike
8221 // Intel, we don't kill CR as a side effect here
8222
8223 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8224 match(Set dst (EncodePKlass src));
8225
8226 ins_cost(INSN_COST * 3);
8227 format %{ "encode_klass_not_null $dst,$src" %}
8228
8229 ins_encode %{
8230 Register src_reg = as_Register($src$$reg);
8231 Register dst_reg = as_Register($dst$$reg);
8232 __ encode_klass_not_null(dst_reg, src_reg);
8233 %}
8234
8235 ins_pipe(ialu_reg);
8236 %}
8237
8238 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8239 match(Set dst (DecodeNKlass src));
8240
8241 ins_cost(INSN_COST * 3);
8242 format %{ "decode_klass_not_null $dst,$src" %}
8243
8244 ins_encode %{
8245 Register src_reg = as_Register($src$$reg);
8246 Register dst_reg = as_Register($dst$$reg);
8247 if (dst_reg != src_reg) {
8248 __ decode_klass_not_null(dst_reg, src_reg);
8249 } else {
8250 __ decode_klass_not_null(dst_reg);
8251 }
8252 %}
8253
8254 ins_pipe(ialu_reg);
8255 %}
8256
8257 instruct checkCastPP(iRegPNoSp dst)
8258 %{
8259 match(Set dst (CheckCastPP dst));
8260
8261 size(0);
8262 format %{ "# checkcastPP of $dst" %}
8263 ins_encode(/* empty encoding */);
8264 ins_pipe(pipe_class_empty);
8265 %}
8266
8267 instruct castPP(iRegPNoSp dst)
8268 %{
8269 match(Set dst (CastPP dst));
8270
8271 size(0);
8272 format %{ "# castPP of $dst" %}
8273 ins_encode(/* empty encoding */);
8274 ins_pipe(pipe_class_empty);
8275 %}
8276
8277 instruct castII(iRegI dst)
8278 %{
8279 predicate(VerifyConstraintCasts == 0);
8280 match(Set dst (CastII dst));
8281
8282 size(0);
8283 format %{ "# castII of $dst" %}
8284 ins_encode(/* empty encoding */);
8285 ins_cost(0);
8286 ins_pipe(pipe_class_empty);
8287 %}
8288
8289 instruct castII_checked(iRegI dst, rFlagsReg cr)
8290 %{
8291 predicate(VerifyConstraintCasts > 0);
8292 match(Set dst (CastII dst));
8293 effect(KILL cr);
8294
8295 format %{ "# castII_checked of $dst" %}
8296 ins_encode %{
8297 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8298 %}
8299 ins_pipe(pipe_slow);
8300 %}
8301
8302 instruct castLL(iRegL dst)
8303 %{
8304 predicate(VerifyConstraintCasts == 0);
8305 match(Set dst (CastLL dst));
8306
8307 size(0);
8308 format %{ "# castLL of $dst" %}
8309 ins_encode(/* empty encoding */);
8310 ins_cost(0);
8311 ins_pipe(pipe_class_empty);
8312 %}
8313
8314 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8315 %{
8316 predicate(VerifyConstraintCasts > 0);
8317 match(Set dst (CastLL dst));
8318 effect(KILL cr);
8319
8320 format %{ "# castLL_checked of $dst" %}
8321 ins_encode %{
8322 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8323 %}
8324 ins_pipe(pipe_slow);
8325 %}
8326
8327 instruct castHH(vRegF dst)
8328 %{
8329 match(Set dst (CastHH dst));
8330 size(0);
8331 format %{ "# castHH of $dst" %}
8332 ins_encode(/* empty encoding */);
8333 ins_cost(0);
8334 ins_pipe(pipe_class_empty);
8335 %}
8336
8337 instruct castFF(vRegF dst)
8338 %{
8339 match(Set dst (CastFF dst));
8340
8341 size(0);
8342 format %{ "# castFF of $dst" %}
8343 ins_encode(/* empty encoding */);
8344 ins_cost(0);
8345 ins_pipe(pipe_class_empty);
8346 %}
8347
8348 instruct castDD(vRegD dst)
8349 %{
8350 match(Set dst (CastDD dst));
8351
8352 size(0);
8353 format %{ "# castDD of $dst" %}
8354 ins_encode(/* empty encoding */);
8355 ins_cost(0);
8356 ins_pipe(pipe_class_empty);
8357 %}
8358
8359 instruct castVV(vReg dst)
8360 %{
8361 match(Set dst (CastVV dst));
8362
8363 size(0);
8364 format %{ "# castVV of $dst" %}
8365 ins_encode(/* empty encoding */);
8366 ins_cost(0);
8367 ins_pipe(pipe_class_empty);
8368 %}
8369
8370 instruct castVVMask(pRegGov dst)
8371 %{
8372 match(Set dst (CastVV dst));
8373
8374 size(0);
8375 format %{ "# castVV of $dst" %}
8376 ins_encode(/* empty encoding */);
8377 ins_cost(0);
8378 ins_pipe(pipe_class_empty);
8379 %}
8380
8381 // Manifest a CmpU result in an integer register.
8382 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8383 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8384 %{
8385 match(Set dst (CmpU3 src1 src2));
8386 effect(KILL flags);
8387
8388 ins_cost(INSN_COST * 3);
8389 format %{
8390 "cmpw $src1, $src2\n\t"
8391 "csetw $dst, ne\n\t"
8392 "cnegw $dst, lo\t# CmpU3(reg)"
8393 %}
8394 ins_encode %{
8395 __ cmpw($src1$$Register, $src2$$Register);
8396 __ csetw($dst$$Register, Assembler::NE);
8397 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8398 %}
8399
8400 ins_pipe(pipe_class_default);
8401 %}
8402
8403 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8404 %{
8405 match(Set dst (CmpU3 src1 src2));
8406 effect(KILL flags);
8407
8408 ins_cost(INSN_COST * 3);
8409 format %{
8410 "subsw zr, $src1, $src2\n\t"
8411 "csetw $dst, ne\n\t"
8412 "cnegw $dst, lo\t# CmpU3(imm)"
8413 %}
8414 ins_encode %{
8415 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8416 __ csetw($dst$$Register, Assembler::NE);
8417 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8418 %}
8419
8420 ins_pipe(pipe_class_default);
8421 %}
8422
8423 // Manifest a CmpUL result in an integer register.
8424 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8425 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8426 %{
8427 match(Set dst (CmpUL3 src1 src2));
8428 effect(KILL flags);
8429
8430 ins_cost(INSN_COST * 3);
8431 format %{
8432 "cmp $src1, $src2\n\t"
8433 "csetw $dst, ne\n\t"
8434 "cnegw $dst, lo\t# CmpUL3(reg)"
8435 %}
8436 ins_encode %{
8437 __ cmp($src1$$Register, $src2$$Register);
8438 __ csetw($dst$$Register, Assembler::NE);
8439 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8440 %}
8441
8442 ins_pipe(pipe_class_default);
8443 %}
8444
8445 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8446 %{
8447 match(Set dst (CmpUL3 src1 src2));
8448 effect(KILL flags);
8449
8450 ins_cost(INSN_COST * 3);
8451 format %{
8452 "subs zr, $src1, $src2\n\t"
8453 "csetw $dst, ne\n\t"
8454 "cnegw $dst, lo\t# CmpUL3(imm)"
8455 %}
8456 ins_encode %{
8457 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8458 __ csetw($dst$$Register, Assembler::NE);
8459 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8460 %}
8461
8462 ins_pipe(pipe_class_default);
8463 %}
8464
8465 // Manifest a CmpL result in an integer register.
8466 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8467 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8468 %{
8469 match(Set dst (CmpL3 src1 src2));
8470 effect(KILL flags);
8471
8472 ins_cost(INSN_COST * 3);
8473 format %{
8474 "cmp $src1, $src2\n\t"
8475 "csetw $dst, ne\n\t"
8476 "cnegw $dst, lt\t# CmpL3(reg)"
8477 %}
8478 ins_encode %{
8479 __ cmp($src1$$Register, $src2$$Register);
8480 __ csetw($dst$$Register, Assembler::NE);
8481 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8482 %}
8483
8484 ins_pipe(pipe_class_default);
8485 %}
8486
8487 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8488 %{
8489 match(Set dst (CmpL3 src1 src2));
8490 effect(KILL flags);
8491
8492 ins_cost(INSN_COST * 3);
8493 format %{
8494 "subs zr, $src1, $src2\n\t"
8495 "csetw $dst, ne\n\t"
8496 "cnegw $dst, lt\t# CmpL3(imm)"
8497 %}
8498 ins_encode %{
8499 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8500 __ csetw($dst$$Register, Assembler::NE);
8501 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8502 %}
8503
8504 ins_pipe(pipe_class_default);
8505 %}
8506
8507 // ============================================================================
8508 // Conditional Move Instructions
8509
8510 // n.b. we have identical rules for both a signed compare op (cmpOp)
8511 // and an unsigned compare op (cmpOpU). it would be nice if we could
8512 // define an op class which merged both inputs and use it to type the
8513 // argument to a single rule. unfortunatelyt his fails because the
8514 // opclass does not live up to the COND_INTER interface of its
8515 // component operands. When the generic code tries to negate the
8516 // operand it ends up running the generci Machoper::negate method
8517 // which throws a ShouldNotHappen. So, we have to provide two flavours
8518 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8519
8520 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8521 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8522
8523 ins_cost(INSN_COST * 2);
8524 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8525
8526 ins_encode %{
8527 __ cselw(as_Register($dst$$reg),
8528 as_Register($src2$$reg),
8529 as_Register($src1$$reg),
8530 (Assembler::Condition)$cmp$$cmpcode);
8531 %}
8532
8533 ins_pipe(icond_reg_reg);
8534 %}
8535
8536 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8537 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8538
8539 ins_cost(INSN_COST * 2);
8540 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8541
8542 ins_encode %{
8543 __ cselw(as_Register($dst$$reg),
8544 as_Register($src2$$reg),
8545 as_Register($src1$$reg),
8546 (Assembler::Condition)$cmp$$cmpcode);
8547 %}
8548
8549 ins_pipe(icond_reg_reg);
8550 %}
8551
8552 // special cases where one arg is zero
8553
8554 // n.b. this is selected in preference to the rule above because it
8555 // avoids loading constant 0 into a source register
8556
8557 // TODO
8558 // we ought only to be able to cull one of these variants as the ideal
8559 // transforms ought always to order the zero consistently (to left/right?)
8560
8561 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8562 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8563
8564 ins_cost(INSN_COST * 2);
8565 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8566
8567 ins_encode %{
8568 __ cselw(as_Register($dst$$reg),
8569 as_Register($src$$reg),
8570 zr,
8571 (Assembler::Condition)$cmp$$cmpcode);
8572 %}
8573
8574 ins_pipe(icond_reg);
8575 %}
8576
8577 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8578 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8579
8580 ins_cost(INSN_COST * 2);
8581 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8582
8583 ins_encode %{
8584 __ cselw(as_Register($dst$$reg),
8585 as_Register($src$$reg),
8586 zr,
8587 (Assembler::Condition)$cmp$$cmpcode);
8588 %}
8589
8590 ins_pipe(icond_reg);
8591 %}
8592
8593 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8594 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8595
8596 ins_cost(INSN_COST * 2);
8597 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8598
8599 ins_encode %{
8600 __ cselw(as_Register($dst$$reg),
8601 zr,
8602 as_Register($src$$reg),
8603 (Assembler::Condition)$cmp$$cmpcode);
8604 %}
8605
8606 ins_pipe(icond_reg);
8607 %}
8608
8609 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8610 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8611
8612 ins_cost(INSN_COST * 2);
8613 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8614
8615 ins_encode %{
8616 __ cselw(as_Register($dst$$reg),
8617 zr,
8618 as_Register($src$$reg),
8619 (Assembler::Condition)$cmp$$cmpcode);
8620 %}
8621
8622 ins_pipe(icond_reg);
8623 %}
8624
8625 // special case for creating a boolean 0 or 1
8626
8627 // n.b. this is selected in preference to the rule above because it
8628 // avoids loading constants 0 and 1 into a source register
8629
8630 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8631 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8632
8633 ins_cost(INSN_COST * 2);
8634 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8635
8636 ins_encode %{
8637 // equivalently
8638 // cset(as_Register($dst$$reg),
8639 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8640 __ csincw(as_Register($dst$$reg),
8641 zr,
8642 zr,
8643 (Assembler::Condition)$cmp$$cmpcode);
8644 %}
8645
8646 ins_pipe(icond_none);
8647 %}
8648
8649 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8650 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8651
8652 ins_cost(INSN_COST * 2);
8653 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8654
8655 ins_encode %{
8656 // equivalently
8657 // cset(as_Register($dst$$reg),
8658 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8659 __ csincw(as_Register($dst$$reg),
8660 zr,
8661 zr,
8662 (Assembler::Condition)$cmp$$cmpcode);
8663 %}
8664
8665 ins_pipe(icond_none);
8666 %}
8667
8668 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8669 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8670
8671 ins_cost(INSN_COST * 2);
8672 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8673
8674 ins_encode %{
8675 __ csel(as_Register($dst$$reg),
8676 as_Register($src2$$reg),
8677 as_Register($src1$$reg),
8678 (Assembler::Condition)$cmp$$cmpcode);
8679 %}
8680
8681 ins_pipe(icond_reg_reg);
8682 %}
8683
8684 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8685 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8686
8687 ins_cost(INSN_COST * 2);
8688 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8689
8690 ins_encode %{
8691 __ csel(as_Register($dst$$reg),
8692 as_Register($src2$$reg),
8693 as_Register($src1$$reg),
8694 (Assembler::Condition)$cmp$$cmpcode);
8695 %}
8696
8697 ins_pipe(icond_reg_reg);
8698 %}
8699
8700 // special cases where one arg is zero
8701
8702 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8703 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8704
8705 ins_cost(INSN_COST * 2);
8706 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8707
8708 ins_encode %{
8709 __ csel(as_Register($dst$$reg),
8710 zr,
8711 as_Register($src$$reg),
8712 (Assembler::Condition)$cmp$$cmpcode);
8713 %}
8714
8715 ins_pipe(icond_reg);
8716 %}
8717
8718 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8719 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8720
8721 ins_cost(INSN_COST * 2);
8722 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8723
8724 ins_encode %{
8725 __ csel(as_Register($dst$$reg),
8726 zr,
8727 as_Register($src$$reg),
8728 (Assembler::Condition)$cmp$$cmpcode);
8729 %}
8730
8731 ins_pipe(icond_reg);
8732 %}
8733
8734 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8735 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8736
8737 ins_cost(INSN_COST * 2);
8738 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8739
8740 ins_encode %{
8741 __ csel(as_Register($dst$$reg),
8742 as_Register($src$$reg),
8743 zr,
8744 (Assembler::Condition)$cmp$$cmpcode);
8745 %}
8746
8747 ins_pipe(icond_reg);
8748 %}
8749
8750 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8751 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8752
8753 ins_cost(INSN_COST * 2);
8754 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8755
8756 ins_encode %{
8757 __ csel(as_Register($dst$$reg),
8758 as_Register($src$$reg),
8759 zr,
8760 (Assembler::Condition)$cmp$$cmpcode);
8761 %}
8762
8763 ins_pipe(icond_reg);
8764 %}
8765
8766 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8767 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8768
8769 ins_cost(INSN_COST * 2);
8770 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8771
8772 ins_encode %{
8773 __ csel(as_Register($dst$$reg),
8774 as_Register($src2$$reg),
8775 as_Register($src1$$reg),
8776 (Assembler::Condition)$cmp$$cmpcode);
8777 %}
8778
8779 ins_pipe(icond_reg_reg);
8780 %}
8781
8782 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8783 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8784
8785 ins_cost(INSN_COST * 2);
8786 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8787
8788 ins_encode %{
8789 __ csel(as_Register($dst$$reg),
8790 as_Register($src2$$reg),
8791 as_Register($src1$$reg),
8792 (Assembler::Condition)$cmp$$cmpcode);
8793 %}
8794
8795 ins_pipe(icond_reg_reg);
8796 %}
8797
8798 // special cases where one arg is zero
8799
8800 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8801 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8802
8803 ins_cost(INSN_COST * 2);
8804 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8805
8806 ins_encode %{
8807 __ csel(as_Register($dst$$reg),
8808 zr,
8809 as_Register($src$$reg),
8810 (Assembler::Condition)$cmp$$cmpcode);
8811 %}
8812
8813 ins_pipe(icond_reg);
8814 %}
8815
8816 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8817 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8818
8819 ins_cost(INSN_COST * 2);
8820 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8821
8822 ins_encode %{
8823 __ csel(as_Register($dst$$reg),
8824 zr,
8825 as_Register($src$$reg),
8826 (Assembler::Condition)$cmp$$cmpcode);
8827 %}
8828
8829 ins_pipe(icond_reg);
8830 %}
8831
8832 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8833 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8834
8835 ins_cost(INSN_COST * 2);
8836 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8837
8838 ins_encode %{
8839 __ csel(as_Register($dst$$reg),
8840 as_Register($src$$reg),
8841 zr,
8842 (Assembler::Condition)$cmp$$cmpcode);
8843 %}
8844
8845 ins_pipe(icond_reg);
8846 %}
8847
8848 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8849 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8850
8851 ins_cost(INSN_COST * 2);
8852 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8853
8854 ins_encode %{
8855 __ csel(as_Register($dst$$reg),
8856 as_Register($src$$reg),
8857 zr,
8858 (Assembler::Condition)$cmp$$cmpcode);
8859 %}
8860
8861 ins_pipe(icond_reg);
8862 %}
8863
8864 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8865 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8866
8867 ins_cost(INSN_COST * 2);
8868 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8869
8870 ins_encode %{
8871 __ cselw(as_Register($dst$$reg),
8872 as_Register($src2$$reg),
8873 as_Register($src1$$reg),
8874 (Assembler::Condition)$cmp$$cmpcode);
8875 %}
8876
8877 ins_pipe(icond_reg_reg);
8878 %}
8879
8880 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8881 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8882
8883 ins_cost(INSN_COST * 2);
8884 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8885
8886 ins_encode %{
8887 __ cselw(as_Register($dst$$reg),
8888 as_Register($src2$$reg),
8889 as_Register($src1$$reg),
8890 (Assembler::Condition)$cmp$$cmpcode);
8891 %}
8892
8893 ins_pipe(icond_reg_reg);
8894 %}
8895
8896 // special cases where one arg is zero
8897
8898 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8899 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8900
8901 ins_cost(INSN_COST * 2);
8902 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8903
8904 ins_encode %{
8905 __ cselw(as_Register($dst$$reg),
8906 zr,
8907 as_Register($src$$reg),
8908 (Assembler::Condition)$cmp$$cmpcode);
8909 %}
8910
8911 ins_pipe(icond_reg);
8912 %}
8913
8914 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8915 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8916
8917 ins_cost(INSN_COST * 2);
8918 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8919
8920 ins_encode %{
8921 __ cselw(as_Register($dst$$reg),
8922 zr,
8923 as_Register($src$$reg),
8924 (Assembler::Condition)$cmp$$cmpcode);
8925 %}
8926
8927 ins_pipe(icond_reg);
8928 %}
8929
8930 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8931 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8932
8933 ins_cost(INSN_COST * 2);
8934 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8935
8936 ins_encode %{
8937 __ cselw(as_Register($dst$$reg),
8938 as_Register($src$$reg),
8939 zr,
8940 (Assembler::Condition)$cmp$$cmpcode);
8941 %}
8942
8943 ins_pipe(icond_reg);
8944 %}
8945
8946 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8947 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8948
8949 ins_cost(INSN_COST * 2);
8950 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8951
8952 ins_encode %{
8953 __ cselw(as_Register($dst$$reg),
8954 as_Register($src$$reg),
8955 zr,
8956 (Assembler::Condition)$cmp$$cmpcode);
8957 %}
8958
8959 ins_pipe(icond_reg);
8960 %}
8961
8962 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8963 %{
8964 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8965
8966 ins_cost(INSN_COST * 3);
8967
8968 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8969 ins_encode %{
8970 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8971 __ fcsels(as_FloatRegister($dst$$reg),
8972 as_FloatRegister($src2$$reg),
8973 as_FloatRegister($src1$$reg),
8974 cond);
8975 %}
8976
8977 ins_pipe(fp_cond_reg_reg_s);
8978 %}
8979
8980 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8981 %{
8982 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8983
8984 ins_cost(INSN_COST * 3);
8985
8986 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8987 ins_encode %{
8988 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8989 __ fcsels(as_FloatRegister($dst$$reg),
8990 as_FloatRegister($src2$$reg),
8991 as_FloatRegister($src1$$reg),
8992 cond);
8993 %}
8994
8995 ins_pipe(fp_cond_reg_reg_s);
8996 %}
8997
8998 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8999 %{
9000 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9001
9002 ins_cost(INSN_COST * 3);
9003
9004 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9005 ins_encode %{
9006 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9007 __ fcseld(as_FloatRegister($dst$$reg),
9008 as_FloatRegister($src2$$reg),
9009 as_FloatRegister($src1$$reg),
9010 cond);
9011 %}
9012
9013 ins_pipe(fp_cond_reg_reg_d);
9014 %}
9015
9016 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9017 %{
9018 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9019
9020 ins_cost(INSN_COST * 3);
9021
9022 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9023 ins_encode %{
9024 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9025 __ fcseld(as_FloatRegister($dst$$reg),
9026 as_FloatRegister($src2$$reg),
9027 as_FloatRegister($src1$$reg),
9028 cond);
9029 %}
9030
9031 ins_pipe(fp_cond_reg_reg_d);
9032 %}
9033
9034 // ============================================================================
9035 // Arithmetic Instructions
9036 //
9037
9038 // Integer Addition
9039
9040 // TODO
9041 // these currently employ operations which do not set CR and hence are
9042 // not flagged as killing CR but we would like to isolate the cases
9043 // where we want to set flags from those where we don't. need to work
9044 // out how to do that.
9045
9046 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9047 match(Set dst (AddI src1 src2));
9048
9049 ins_cost(INSN_COST);
9050 format %{ "addw $dst, $src1, $src2" %}
9051
9052 ins_encode %{
9053 __ addw(as_Register($dst$$reg),
9054 as_Register($src1$$reg),
9055 as_Register($src2$$reg));
9056 %}
9057
9058 ins_pipe(ialu_reg_reg);
9059 %}
9060
9061 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9062 match(Set dst (AddI src1 src2));
9063
9064 ins_cost(INSN_COST);
9065 format %{ "addw $dst, $src1, $src2" %}
9066
9067 // use opcode to indicate that this is an add not a sub
9068 opcode(0x0);
9069
9070 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9071
9072 ins_pipe(ialu_reg_imm);
9073 %}
9074
9075 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9076 match(Set dst (AddI (ConvL2I src1) src2));
9077
9078 ins_cost(INSN_COST);
9079 format %{ "addw $dst, $src1, $src2" %}
9080
9081 // use opcode to indicate that this is an add not a sub
9082 opcode(0x0);
9083
9084 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9085
9086 ins_pipe(ialu_reg_imm);
9087 %}
9088
9089 // Pointer Addition
9090 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9091 match(Set dst (AddP src1 src2));
9092
9093 ins_cost(INSN_COST);
9094 format %{ "add $dst, $src1, $src2\t# ptr" %}
9095
9096 ins_encode %{
9097 __ add(as_Register($dst$$reg),
9098 as_Register($src1$$reg),
9099 as_Register($src2$$reg));
9100 %}
9101
9102 ins_pipe(ialu_reg_reg);
9103 %}
9104
9105 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9106 match(Set dst (AddP src1 (ConvI2L src2)));
9107
9108 ins_cost(1.9 * INSN_COST);
9109 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9110
9111 ins_encode %{
9112 __ add(as_Register($dst$$reg),
9113 as_Register($src1$$reg),
9114 as_Register($src2$$reg), ext::sxtw);
9115 %}
9116
9117 ins_pipe(ialu_reg_reg);
9118 %}
9119
9120 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9121 match(Set dst (AddP src1 (LShiftL src2 scale)));
9122
9123 ins_cost(1.9 * INSN_COST);
9124 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9125
9126 ins_encode %{
9127 __ lea(as_Register($dst$$reg),
9128 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9129 Address::lsl($scale$$constant)));
9130 %}
9131
9132 ins_pipe(ialu_reg_reg_shift);
9133 %}
9134
9135 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9136 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9137
9138 ins_cost(1.9 * INSN_COST);
9139 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9140
9141 ins_encode %{
9142 __ lea(as_Register($dst$$reg),
9143 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9144 Address::sxtw($scale$$constant)));
9145 %}
9146
9147 ins_pipe(ialu_reg_reg_shift);
9148 %}
9149
9150 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9151 match(Set dst (LShiftL (ConvI2L src) scale));
9152
9153 ins_cost(INSN_COST);
9154 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9155
9156 ins_encode %{
9157 __ sbfiz(as_Register($dst$$reg),
9158 as_Register($src$$reg),
9159 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9160 %}
9161
9162 ins_pipe(ialu_reg_shift);
9163 %}
9164
9165 // Pointer Immediate Addition
9166 // n.b. this needs to be more expensive than using an indirect memory
9167 // operand
9168 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9169 match(Set dst (AddP src1 src2));
9170
9171 ins_cost(INSN_COST);
9172 format %{ "add $dst, $src1, $src2\t# ptr" %}
9173
9174 // use opcode to indicate that this is an add not a sub
9175 opcode(0x0);
9176
9177 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9178
9179 ins_pipe(ialu_reg_imm);
9180 %}
9181
9182 // Long Addition
9183 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9184
9185 match(Set dst (AddL src1 src2));
9186
9187 ins_cost(INSN_COST);
9188 format %{ "add $dst, $src1, $src2" %}
9189
9190 ins_encode %{
9191 __ add(as_Register($dst$$reg),
9192 as_Register($src1$$reg),
9193 as_Register($src2$$reg));
9194 %}
9195
9196 ins_pipe(ialu_reg_reg);
9197 %}
9198
9199 // No constant pool entries requiredLong Immediate Addition.
9200 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9201 match(Set dst (AddL src1 src2));
9202
9203 ins_cost(INSN_COST);
9204 format %{ "add $dst, $src1, $src2" %}
9205
9206 // use opcode to indicate that this is an add not a sub
9207 opcode(0x0);
9208
9209 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9210
9211 ins_pipe(ialu_reg_imm);
9212 %}
9213
9214 // Integer Subtraction
9215 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9216 match(Set dst (SubI src1 src2));
9217
9218 ins_cost(INSN_COST);
9219 format %{ "subw $dst, $src1, $src2" %}
9220
9221 ins_encode %{
9222 __ subw(as_Register($dst$$reg),
9223 as_Register($src1$$reg),
9224 as_Register($src2$$reg));
9225 %}
9226
9227 ins_pipe(ialu_reg_reg);
9228 %}
9229
9230 // Immediate Subtraction
9231 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9232 match(Set dst (SubI src1 src2));
9233
9234 ins_cost(INSN_COST);
9235 format %{ "subw $dst, $src1, $src2" %}
9236
9237 // use opcode to indicate that this is a sub not an add
9238 opcode(0x1);
9239
9240 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9241
9242 ins_pipe(ialu_reg_imm);
9243 %}
9244
9245 // Long Subtraction
9246 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9247
9248 match(Set dst (SubL src1 src2));
9249
9250 ins_cost(INSN_COST);
9251 format %{ "sub $dst, $src1, $src2" %}
9252
9253 ins_encode %{
9254 __ sub(as_Register($dst$$reg),
9255 as_Register($src1$$reg),
9256 as_Register($src2$$reg));
9257 %}
9258
9259 ins_pipe(ialu_reg_reg);
9260 %}
9261
9262 // No constant pool entries requiredLong Immediate Subtraction.
9263 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9264 match(Set dst (SubL src1 src2));
9265
9266 ins_cost(INSN_COST);
9267 format %{ "sub$dst, $src1, $src2" %}
9268
9269 // use opcode to indicate that this is a sub not an add
9270 opcode(0x1);
9271
9272 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9273
9274 ins_pipe(ialu_reg_imm);
9275 %}
9276
9277 // Integer Negation (special case for sub)
9278
9279 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9280 match(Set dst (SubI zero src));
9281
9282 ins_cost(INSN_COST);
9283 format %{ "negw $dst, $src\t# int" %}
9284
9285 ins_encode %{
9286 __ negw(as_Register($dst$$reg),
9287 as_Register($src$$reg));
9288 %}
9289
9290 ins_pipe(ialu_reg);
9291 %}
9292
9293 // Long Negation
9294
9295 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9296 match(Set dst (SubL zero src));
9297
9298 ins_cost(INSN_COST);
9299 format %{ "neg $dst, $src\t# long" %}
9300
9301 ins_encode %{
9302 __ neg(as_Register($dst$$reg),
9303 as_Register($src$$reg));
9304 %}
9305
9306 ins_pipe(ialu_reg);
9307 %}
9308
9309 // Integer Multiply
9310
9311 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9312 match(Set dst (MulI src1 src2));
9313
9314 ins_cost(INSN_COST * 3);
9315 format %{ "mulw $dst, $src1, $src2" %}
9316
9317 ins_encode %{
9318 __ mulw(as_Register($dst$$reg),
9319 as_Register($src1$$reg),
9320 as_Register($src2$$reg));
9321 %}
9322
9323 ins_pipe(imul_reg_reg);
9324 %}
9325
9326 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9327 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9328
9329 ins_cost(INSN_COST * 3);
9330 format %{ "smull $dst, $src1, $src2" %}
9331
9332 ins_encode %{
9333 __ smull(as_Register($dst$$reg),
9334 as_Register($src1$$reg),
9335 as_Register($src2$$reg));
9336 %}
9337
9338 ins_pipe(imul_reg_reg);
9339 %}
9340
9341 // Long Multiply
9342
9343 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9344 match(Set dst (MulL src1 src2));
9345
9346 ins_cost(INSN_COST * 5);
9347 format %{ "mul $dst, $src1, $src2" %}
9348
9349 ins_encode %{
9350 __ mul(as_Register($dst$$reg),
9351 as_Register($src1$$reg),
9352 as_Register($src2$$reg));
9353 %}
9354
9355 ins_pipe(lmul_reg_reg);
9356 %}
9357
9358 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9359 %{
9360 match(Set dst (MulHiL src1 src2));
9361
9362 ins_cost(INSN_COST * 7);
9363 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9364
9365 ins_encode %{
9366 __ smulh(as_Register($dst$$reg),
9367 as_Register($src1$$reg),
9368 as_Register($src2$$reg));
9369 %}
9370
9371 ins_pipe(lmul_reg_reg);
9372 %}
9373
9374 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9375 %{
9376 match(Set dst (UMulHiL src1 src2));
9377
9378 ins_cost(INSN_COST * 7);
9379 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9380
9381 ins_encode %{
9382 __ umulh(as_Register($dst$$reg),
9383 as_Register($src1$$reg),
9384 as_Register($src2$$reg));
9385 %}
9386
9387 ins_pipe(lmul_reg_reg);
9388 %}
9389
9390 // Combined Integer Multiply & Add/Sub
9391
9392 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9393 match(Set dst (AddI src3 (MulI src1 src2)));
9394
9395 ins_cost(INSN_COST * 3);
9396 format %{ "madd $dst, $src1, $src2, $src3" %}
9397
9398 ins_encode %{
9399 __ maddw(as_Register($dst$$reg),
9400 as_Register($src1$$reg),
9401 as_Register($src2$$reg),
9402 as_Register($src3$$reg));
9403 %}
9404
9405 ins_pipe(imac_reg_reg);
9406 %}
9407
9408 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9409 match(Set dst (SubI src3 (MulI src1 src2)));
9410
9411 ins_cost(INSN_COST * 3);
9412 format %{ "msub $dst, $src1, $src2, $src3" %}
9413
9414 ins_encode %{
9415 __ msubw(as_Register($dst$$reg),
9416 as_Register($src1$$reg),
9417 as_Register($src2$$reg),
9418 as_Register($src3$$reg));
9419 %}
9420
9421 ins_pipe(imac_reg_reg);
9422 %}
9423
9424 // Combined Integer Multiply & Neg
9425
9426 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9427 match(Set dst (MulI (SubI zero src1) src2));
9428
9429 ins_cost(INSN_COST * 3);
9430 format %{ "mneg $dst, $src1, $src2" %}
9431
9432 ins_encode %{
9433 __ mnegw(as_Register($dst$$reg),
9434 as_Register($src1$$reg),
9435 as_Register($src2$$reg));
9436 %}
9437
9438 ins_pipe(imac_reg_reg);
9439 %}
9440
9441 // Combined Long Multiply & Add/Sub
9442
9443 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9444 match(Set dst (AddL src3 (MulL src1 src2)));
9445
9446 ins_cost(INSN_COST * 5);
9447 format %{ "madd $dst, $src1, $src2, $src3" %}
9448
9449 ins_encode %{
9450 __ madd(as_Register($dst$$reg),
9451 as_Register($src1$$reg),
9452 as_Register($src2$$reg),
9453 as_Register($src3$$reg));
9454 %}
9455
9456 ins_pipe(lmac_reg_reg);
9457 %}
9458
9459 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9460 match(Set dst (SubL src3 (MulL src1 src2)));
9461
9462 ins_cost(INSN_COST * 5);
9463 format %{ "msub $dst, $src1, $src2, $src3" %}
9464
9465 ins_encode %{
9466 __ msub(as_Register($dst$$reg),
9467 as_Register($src1$$reg),
9468 as_Register($src2$$reg),
9469 as_Register($src3$$reg));
9470 %}
9471
9472 ins_pipe(lmac_reg_reg);
9473 %}
9474
9475 // Combined Long Multiply & Neg
9476
9477 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9478 match(Set dst (MulL (SubL zero src1) src2));
9479
9480 ins_cost(INSN_COST * 5);
9481 format %{ "mneg $dst, $src1, $src2" %}
9482
9483 ins_encode %{
9484 __ mneg(as_Register($dst$$reg),
9485 as_Register($src1$$reg),
9486 as_Register($src2$$reg));
9487 %}
9488
9489 ins_pipe(lmac_reg_reg);
9490 %}
9491
9492 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9493
9494 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9495 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9496
9497 ins_cost(INSN_COST * 3);
9498 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9499
9500 ins_encode %{
9501 __ smaddl(as_Register($dst$$reg),
9502 as_Register($src1$$reg),
9503 as_Register($src2$$reg),
9504 as_Register($src3$$reg));
9505 %}
9506
9507 ins_pipe(imac_reg_reg);
9508 %}
9509
9510 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9511 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9512
9513 ins_cost(INSN_COST * 3);
9514 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9515
9516 ins_encode %{
9517 __ smsubl(as_Register($dst$$reg),
9518 as_Register($src1$$reg),
9519 as_Register($src2$$reg),
9520 as_Register($src3$$reg));
9521 %}
9522
9523 ins_pipe(imac_reg_reg);
9524 %}
9525
9526 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9527 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9528
9529 ins_cost(INSN_COST * 3);
9530 format %{ "smnegl $dst, $src1, $src2" %}
9531
9532 ins_encode %{
9533 __ smnegl(as_Register($dst$$reg),
9534 as_Register($src1$$reg),
9535 as_Register($src2$$reg));
9536 %}
9537
9538 ins_pipe(imac_reg_reg);
9539 %}
9540
9541 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9542
9543 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9544 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9545
9546 ins_cost(INSN_COST * 5);
9547 format %{ "mulw rscratch1, $src1, $src2\n\t"
9548 "maddw $dst, $src3, $src4, rscratch1" %}
9549
9550 ins_encode %{
9551 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9552 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9553
9554 ins_pipe(imac_reg_reg);
9555 %}
9556
9557 // Integer Divide
9558
9559 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9560 match(Set dst (DivI src1 src2));
9561
9562 ins_cost(INSN_COST * 19);
9563 format %{ "sdivw $dst, $src1, $src2" %}
9564
9565 ins_encode(aarch64_enc_divw(dst, src1, src2));
9566 ins_pipe(idiv_reg_reg);
9567 %}
9568
9569 // Long Divide
9570
9571 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9572 match(Set dst (DivL src1 src2));
9573
9574 ins_cost(INSN_COST * 35);
9575 format %{ "sdiv $dst, $src1, $src2" %}
9576
9577 ins_encode(aarch64_enc_div(dst, src1, src2));
9578 ins_pipe(ldiv_reg_reg);
9579 %}
9580
9581 // Integer Remainder
9582
9583 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9584 match(Set dst (ModI src1 src2));
9585
9586 ins_cost(INSN_COST * 22);
9587 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9588 "msubw $dst, rscratch1, $src2, $src1" %}
9589
9590 ins_encode(aarch64_enc_modw(dst, src1, src2));
9591 ins_pipe(idiv_reg_reg);
9592 %}
9593
9594 // Long Remainder
9595
9596 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9597 match(Set dst (ModL src1 src2));
9598
9599 ins_cost(INSN_COST * 38);
9600 format %{ "sdiv rscratch1, $src1, $src2\n"
9601 "msub $dst, rscratch1, $src2, $src1" %}
9602
9603 ins_encode(aarch64_enc_mod(dst, src1, src2));
9604 ins_pipe(ldiv_reg_reg);
9605 %}
9606
9607 // Unsigned Integer Divide
9608
9609 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9610 match(Set dst (UDivI src1 src2));
9611
9612 ins_cost(INSN_COST * 19);
9613 format %{ "udivw $dst, $src1, $src2" %}
9614
9615 ins_encode %{
9616 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9617 %}
9618
9619 ins_pipe(idiv_reg_reg);
9620 %}
9621
9622 // Unsigned Long Divide
9623
9624 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9625 match(Set dst (UDivL src1 src2));
9626
9627 ins_cost(INSN_COST * 35);
9628 format %{ "udiv $dst, $src1, $src2" %}
9629
9630 ins_encode %{
9631 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9632 %}
9633
9634 ins_pipe(ldiv_reg_reg);
9635 %}
9636
9637 // Unsigned Integer Remainder
9638
9639 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9640 match(Set dst (UModI src1 src2));
9641
9642 ins_cost(INSN_COST * 22);
9643 format %{ "udivw rscratch1, $src1, $src2\n\t"
9644 "msubw $dst, rscratch1, $src2, $src1" %}
9645
9646 ins_encode %{
9647 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9648 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9649 %}
9650
9651 ins_pipe(idiv_reg_reg);
9652 %}
9653
9654 // Unsigned Long Remainder
9655
9656 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9657 match(Set dst (UModL src1 src2));
9658
9659 ins_cost(INSN_COST * 38);
9660 format %{ "udiv rscratch1, $src1, $src2\n"
9661 "msub $dst, rscratch1, $src2, $src1" %}
9662
9663 ins_encode %{
9664 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9665 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9666 %}
9667
9668 ins_pipe(ldiv_reg_reg);
9669 %}
9670
9671 // Integer Shifts
9672
9673 // Shift Left Register
9674 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9675 match(Set dst (LShiftI src1 src2));
9676
9677 ins_cost(INSN_COST * 2);
9678 format %{ "lslvw $dst, $src1, $src2" %}
9679
9680 ins_encode %{
9681 __ lslvw(as_Register($dst$$reg),
9682 as_Register($src1$$reg),
9683 as_Register($src2$$reg));
9684 %}
9685
9686 ins_pipe(ialu_reg_reg_vshift);
9687 %}
9688
9689 // Shift Left Immediate
9690 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9691 match(Set dst (LShiftI src1 src2));
9692
9693 ins_cost(INSN_COST);
9694 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9695
9696 ins_encode %{
9697 __ lslw(as_Register($dst$$reg),
9698 as_Register($src1$$reg),
9699 $src2$$constant & 0x1f);
9700 %}
9701
9702 ins_pipe(ialu_reg_shift);
9703 %}
9704
9705 // Shift Right Logical Register
9706 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9707 match(Set dst (URShiftI src1 src2));
9708
9709 ins_cost(INSN_COST * 2);
9710 format %{ "lsrvw $dst, $src1, $src2" %}
9711
9712 ins_encode %{
9713 __ lsrvw(as_Register($dst$$reg),
9714 as_Register($src1$$reg),
9715 as_Register($src2$$reg));
9716 %}
9717
9718 ins_pipe(ialu_reg_reg_vshift);
9719 %}
9720
9721 // Shift Right Logical Immediate
9722 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9723 match(Set dst (URShiftI src1 src2));
9724
9725 ins_cost(INSN_COST);
9726 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9727
9728 ins_encode %{
9729 __ lsrw(as_Register($dst$$reg),
9730 as_Register($src1$$reg),
9731 $src2$$constant & 0x1f);
9732 %}
9733
9734 ins_pipe(ialu_reg_shift);
9735 %}
9736
9737 // Shift Right Arithmetic Register
9738 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9739 match(Set dst (RShiftI src1 src2));
9740
9741 ins_cost(INSN_COST * 2);
9742 format %{ "asrvw $dst, $src1, $src2" %}
9743
9744 ins_encode %{
9745 __ asrvw(as_Register($dst$$reg),
9746 as_Register($src1$$reg),
9747 as_Register($src2$$reg));
9748 %}
9749
9750 ins_pipe(ialu_reg_reg_vshift);
9751 %}
9752
9753 // Shift Right Arithmetic Immediate
9754 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9755 match(Set dst (RShiftI src1 src2));
9756
9757 ins_cost(INSN_COST);
9758 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9759
9760 ins_encode %{
9761 __ asrw(as_Register($dst$$reg),
9762 as_Register($src1$$reg),
9763 $src2$$constant & 0x1f);
9764 %}
9765
9766 ins_pipe(ialu_reg_shift);
9767 %}
9768
9769 // Combined Int Mask and Right Shift (using UBFM)
9770 // TODO
9771
9772 // Long Shifts
9773
9774 // Shift Left Register
9775 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9776 match(Set dst (LShiftL src1 src2));
9777
9778 ins_cost(INSN_COST * 2);
9779 format %{ "lslv $dst, $src1, $src2" %}
9780
9781 ins_encode %{
9782 __ lslv(as_Register($dst$$reg),
9783 as_Register($src1$$reg),
9784 as_Register($src2$$reg));
9785 %}
9786
9787 ins_pipe(ialu_reg_reg_vshift);
9788 %}
9789
9790 // Shift Left Immediate
9791 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9792 match(Set dst (LShiftL src1 src2));
9793
9794 ins_cost(INSN_COST);
9795 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9796
9797 ins_encode %{
9798 __ lsl(as_Register($dst$$reg),
9799 as_Register($src1$$reg),
9800 $src2$$constant & 0x3f);
9801 %}
9802
9803 ins_pipe(ialu_reg_shift);
9804 %}
9805
9806 // Shift Right Logical Register
9807 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9808 match(Set dst (URShiftL src1 src2));
9809
9810 ins_cost(INSN_COST * 2);
9811 format %{ "lsrv $dst, $src1, $src2" %}
9812
9813 ins_encode %{
9814 __ lsrv(as_Register($dst$$reg),
9815 as_Register($src1$$reg),
9816 as_Register($src2$$reg));
9817 %}
9818
9819 ins_pipe(ialu_reg_reg_vshift);
9820 %}
9821
9822 // Shift Right Logical Immediate
9823 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9824 match(Set dst (URShiftL src1 src2));
9825
9826 ins_cost(INSN_COST);
9827 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9828
9829 ins_encode %{
9830 __ lsr(as_Register($dst$$reg),
9831 as_Register($src1$$reg),
9832 $src2$$constant & 0x3f);
9833 %}
9834
9835 ins_pipe(ialu_reg_shift);
9836 %}
9837
9838 // A special-case pattern for card table stores.
9839 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9840 match(Set dst (URShiftL (CastP2X src1) src2));
9841
9842 ins_cost(INSN_COST);
9843 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9844
9845 ins_encode %{
9846 __ lsr(as_Register($dst$$reg),
9847 as_Register($src1$$reg),
9848 $src2$$constant & 0x3f);
9849 %}
9850
9851 ins_pipe(ialu_reg_shift);
9852 %}
9853
9854 // Shift Right Arithmetic Register
9855 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9856 match(Set dst (RShiftL src1 src2));
9857
9858 ins_cost(INSN_COST * 2);
9859 format %{ "asrv $dst, $src1, $src2" %}
9860
9861 ins_encode %{
9862 __ asrv(as_Register($dst$$reg),
9863 as_Register($src1$$reg),
9864 as_Register($src2$$reg));
9865 %}
9866
9867 ins_pipe(ialu_reg_reg_vshift);
9868 %}
9869
9870 // Shift Right Arithmetic Immediate
9871 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9872 match(Set dst (RShiftL src1 src2));
9873
9874 ins_cost(INSN_COST);
9875 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9876
9877 ins_encode %{
9878 __ asr(as_Register($dst$$reg),
9879 as_Register($src1$$reg),
9880 $src2$$constant & 0x3f);
9881 %}
9882
9883 ins_pipe(ialu_reg_shift);
9884 %}
9885
9886 // BEGIN This section of the file is automatically generated. Do not edit --------------
9887 // This section is generated from aarch64_ad.m4
9888
9889 // This pattern is automatically generated from aarch64_ad.m4
9890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9891 instruct regL_not_reg(iRegLNoSp dst,
9892 iRegL src1, immL_M1 m1,
9893 rFlagsReg cr) %{
9894 match(Set dst (XorL src1 m1));
9895 ins_cost(INSN_COST);
9896 format %{ "eon $dst, $src1, zr" %}
9897
9898 ins_encode %{
9899 __ eon(as_Register($dst$$reg),
9900 as_Register($src1$$reg),
9901 zr,
9902 Assembler::LSL, 0);
9903 %}
9904
9905 ins_pipe(ialu_reg);
9906 %}
9907
9908 // This pattern is automatically generated from aarch64_ad.m4
9909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9910 instruct regI_not_reg(iRegINoSp dst,
9911 iRegIorL2I src1, immI_M1 m1,
9912 rFlagsReg cr) %{
9913 match(Set dst (XorI src1 m1));
9914 ins_cost(INSN_COST);
9915 format %{ "eonw $dst, $src1, zr" %}
9916
9917 ins_encode %{
9918 __ eonw(as_Register($dst$$reg),
9919 as_Register($src1$$reg),
9920 zr,
9921 Assembler::LSL, 0);
9922 %}
9923
9924 ins_pipe(ialu_reg);
9925 %}
9926
9927 // This pattern is automatically generated from aarch64_ad.m4
9928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9929 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9930 immI0 zero, iRegIorL2I src1, immI src2) %{
9931 match(Set dst (SubI zero (URShiftI src1 src2)));
9932
9933 ins_cost(1.9 * INSN_COST);
9934 format %{ "negw $dst, $src1, LSR $src2" %}
9935
9936 ins_encode %{
9937 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9938 Assembler::LSR, $src2$$constant & 0x1f);
9939 %}
9940
9941 ins_pipe(ialu_reg_shift);
9942 %}
9943
9944 // This pattern is automatically generated from aarch64_ad.m4
9945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9946 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9947 immI0 zero, iRegIorL2I src1, immI src2) %{
9948 match(Set dst (SubI zero (RShiftI src1 src2)));
9949
9950 ins_cost(1.9 * INSN_COST);
9951 format %{ "negw $dst, $src1, ASR $src2" %}
9952
9953 ins_encode %{
9954 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9955 Assembler::ASR, $src2$$constant & 0x1f);
9956 %}
9957
9958 ins_pipe(ialu_reg_shift);
9959 %}
9960
9961 // This pattern is automatically generated from aarch64_ad.m4
9962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9963 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9964 immI0 zero, iRegIorL2I src1, immI src2) %{
9965 match(Set dst (SubI zero (LShiftI src1 src2)));
9966
9967 ins_cost(1.9 * INSN_COST);
9968 format %{ "negw $dst, $src1, LSL $src2" %}
9969
9970 ins_encode %{
9971 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9972 Assembler::LSL, $src2$$constant & 0x1f);
9973 %}
9974
9975 ins_pipe(ialu_reg_shift);
9976 %}
9977
9978 // This pattern is automatically generated from aarch64_ad.m4
9979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9980 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9981 immL0 zero, iRegL src1, immI src2) %{
9982 match(Set dst (SubL zero (URShiftL src1 src2)));
9983
9984 ins_cost(1.9 * INSN_COST);
9985 format %{ "neg $dst, $src1, LSR $src2" %}
9986
9987 ins_encode %{
9988 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9989 Assembler::LSR, $src2$$constant & 0x3f);
9990 %}
9991
9992 ins_pipe(ialu_reg_shift);
9993 %}
9994
9995 // This pattern is automatically generated from aarch64_ad.m4
9996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9997 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9998 immL0 zero, iRegL src1, immI src2) %{
9999 match(Set dst (SubL zero (RShiftL src1 src2)));
10000
10001 ins_cost(1.9 * INSN_COST);
10002 format %{ "neg $dst, $src1, ASR $src2" %}
10003
10004 ins_encode %{
10005 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10006 Assembler::ASR, $src2$$constant & 0x3f);
10007 %}
10008
10009 ins_pipe(ialu_reg_shift);
10010 %}
10011
10012 // This pattern is automatically generated from aarch64_ad.m4
10013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10014 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10015 immL0 zero, iRegL src1, immI src2) %{
10016 match(Set dst (SubL zero (LShiftL src1 src2)));
10017
10018 ins_cost(1.9 * INSN_COST);
10019 format %{ "neg $dst, $src1, LSL $src2" %}
10020
10021 ins_encode %{
10022 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10023 Assembler::LSL, $src2$$constant & 0x3f);
10024 %}
10025
10026 ins_pipe(ialu_reg_shift);
10027 %}
10028
10029 // This pattern is automatically generated from aarch64_ad.m4
10030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10031 instruct AndI_reg_not_reg(iRegINoSp dst,
10032 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10033 match(Set dst (AndI src1 (XorI src2 m1)));
10034 ins_cost(INSN_COST);
10035 format %{ "bicw $dst, $src1, $src2" %}
10036
10037 ins_encode %{
10038 __ bicw(as_Register($dst$$reg),
10039 as_Register($src1$$reg),
10040 as_Register($src2$$reg),
10041 Assembler::LSL, 0);
10042 %}
10043
10044 ins_pipe(ialu_reg_reg);
10045 %}
10046
10047 // This pattern is automatically generated from aarch64_ad.m4
10048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10049 instruct AndL_reg_not_reg(iRegLNoSp dst,
10050 iRegL src1, iRegL src2, immL_M1 m1) %{
10051 match(Set dst (AndL src1 (XorL src2 m1)));
10052 ins_cost(INSN_COST);
10053 format %{ "bic $dst, $src1, $src2" %}
10054
10055 ins_encode %{
10056 __ bic(as_Register($dst$$reg),
10057 as_Register($src1$$reg),
10058 as_Register($src2$$reg),
10059 Assembler::LSL, 0);
10060 %}
10061
10062 ins_pipe(ialu_reg_reg);
10063 %}
10064
10065 // This pattern is automatically generated from aarch64_ad.m4
10066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10067 instruct OrI_reg_not_reg(iRegINoSp dst,
10068 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10069 match(Set dst (OrI src1 (XorI src2 m1)));
10070 ins_cost(INSN_COST);
10071 format %{ "ornw $dst, $src1, $src2" %}
10072
10073 ins_encode %{
10074 __ ornw(as_Register($dst$$reg),
10075 as_Register($src1$$reg),
10076 as_Register($src2$$reg),
10077 Assembler::LSL, 0);
10078 %}
10079
10080 ins_pipe(ialu_reg_reg);
10081 %}
10082
10083 // This pattern is automatically generated from aarch64_ad.m4
10084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10085 instruct OrL_reg_not_reg(iRegLNoSp dst,
10086 iRegL src1, iRegL src2, immL_M1 m1) %{
10087 match(Set dst (OrL src1 (XorL src2 m1)));
10088 ins_cost(INSN_COST);
10089 format %{ "orn $dst, $src1, $src2" %}
10090
10091 ins_encode %{
10092 __ orn(as_Register($dst$$reg),
10093 as_Register($src1$$reg),
10094 as_Register($src2$$reg),
10095 Assembler::LSL, 0);
10096 %}
10097
10098 ins_pipe(ialu_reg_reg);
10099 %}
10100
10101 // This pattern is automatically generated from aarch64_ad.m4
10102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10103 instruct XorI_reg_not_reg(iRegINoSp dst,
10104 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10105 match(Set dst (XorI m1 (XorI src2 src1)));
10106 ins_cost(INSN_COST);
10107 format %{ "eonw $dst, $src1, $src2" %}
10108
10109 ins_encode %{
10110 __ eonw(as_Register($dst$$reg),
10111 as_Register($src1$$reg),
10112 as_Register($src2$$reg),
10113 Assembler::LSL, 0);
10114 %}
10115
10116 ins_pipe(ialu_reg_reg);
10117 %}
10118
10119 // This pattern is automatically generated from aarch64_ad.m4
10120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10121 instruct XorL_reg_not_reg(iRegLNoSp dst,
10122 iRegL src1, iRegL src2, immL_M1 m1) %{
10123 match(Set dst (XorL m1 (XorL src2 src1)));
10124 ins_cost(INSN_COST);
10125 format %{ "eon $dst, $src1, $src2" %}
10126
10127 ins_encode %{
10128 __ eon(as_Register($dst$$reg),
10129 as_Register($src1$$reg),
10130 as_Register($src2$$reg),
10131 Assembler::LSL, 0);
10132 %}
10133
10134 ins_pipe(ialu_reg_reg);
10135 %}
10136
10137 // This pattern is automatically generated from aarch64_ad.m4
10138 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10139 // val & (-1 ^ (val >>> shift)) ==> bicw
10140 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10141 iRegIorL2I src1, iRegIorL2I src2,
10142 immI src3, immI_M1 src4) %{
10143 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10144 ins_cost(1.9 * INSN_COST);
10145 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10146
10147 ins_encode %{
10148 __ bicw(as_Register($dst$$reg),
10149 as_Register($src1$$reg),
10150 as_Register($src2$$reg),
10151 Assembler::LSR,
10152 $src3$$constant & 0x1f);
10153 %}
10154
10155 ins_pipe(ialu_reg_reg_shift);
10156 %}
10157
10158 // This pattern is automatically generated from aarch64_ad.m4
10159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10160 // val & (-1 ^ (val >>> shift)) ==> bic
10161 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10162 iRegL src1, iRegL src2,
10163 immI src3, immL_M1 src4) %{
10164 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10165 ins_cost(1.9 * INSN_COST);
10166 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10167
10168 ins_encode %{
10169 __ bic(as_Register($dst$$reg),
10170 as_Register($src1$$reg),
10171 as_Register($src2$$reg),
10172 Assembler::LSR,
10173 $src3$$constant & 0x3f);
10174 %}
10175
10176 ins_pipe(ialu_reg_reg_shift);
10177 %}
10178
10179 // This pattern is automatically generated from aarch64_ad.m4
10180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10181 // val & (-1 ^ (val >> shift)) ==> bicw
10182 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10183 iRegIorL2I src1, iRegIorL2I src2,
10184 immI src3, immI_M1 src4) %{
10185 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10186 ins_cost(1.9 * INSN_COST);
10187 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10188
10189 ins_encode %{
10190 __ bicw(as_Register($dst$$reg),
10191 as_Register($src1$$reg),
10192 as_Register($src2$$reg),
10193 Assembler::ASR,
10194 $src3$$constant & 0x1f);
10195 %}
10196
10197 ins_pipe(ialu_reg_reg_shift);
10198 %}
10199
10200 // This pattern is automatically generated from aarch64_ad.m4
10201 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10202 // val & (-1 ^ (val >> shift)) ==> bic
10203 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10204 iRegL src1, iRegL src2,
10205 immI src3, immL_M1 src4) %{
10206 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10207 ins_cost(1.9 * INSN_COST);
10208 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10209
10210 ins_encode %{
10211 __ bic(as_Register($dst$$reg),
10212 as_Register($src1$$reg),
10213 as_Register($src2$$reg),
10214 Assembler::ASR,
10215 $src3$$constant & 0x3f);
10216 %}
10217
10218 ins_pipe(ialu_reg_reg_shift);
10219 %}
10220
10221 // This pattern is automatically generated from aarch64_ad.m4
10222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10223 // val & (-1 ^ (val ror shift)) ==> bicw
10224 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10225 iRegIorL2I src1, iRegIorL2I src2,
10226 immI src3, immI_M1 src4) %{
10227 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10228 ins_cost(1.9 * INSN_COST);
10229 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10230
10231 ins_encode %{
10232 __ bicw(as_Register($dst$$reg),
10233 as_Register($src1$$reg),
10234 as_Register($src2$$reg),
10235 Assembler::ROR,
10236 $src3$$constant & 0x1f);
10237 %}
10238
10239 ins_pipe(ialu_reg_reg_shift);
10240 %}
10241
10242 // This pattern is automatically generated from aarch64_ad.m4
10243 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10244 // val & (-1 ^ (val ror shift)) ==> bic
10245 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10246 iRegL src1, iRegL src2,
10247 immI src3, immL_M1 src4) %{
10248 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10249 ins_cost(1.9 * INSN_COST);
10250 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10251
10252 ins_encode %{
10253 __ bic(as_Register($dst$$reg),
10254 as_Register($src1$$reg),
10255 as_Register($src2$$reg),
10256 Assembler::ROR,
10257 $src3$$constant & 0x3f);
10258 %}
10259
10260 ins_pipe(ialu_reg_reg_shift);
10261 %}
10262
10263 // This pattern is automatically generated from aarch64_ad.m4
10264 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10265 // val & (-1 ^ (val << shift)) ==> bicw
10266 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10267 iRegIorL2I src1, iRegIorL2I src2,
10268 immI src3, immI_M1 src4) %{
10269 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10270 ins_cost(1.9 * INSN_COST);
10271 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10272
10273 ins_encode %{
10274 __ bicw(as_Register($dst$$reg),
10275 as_Register($src1$$reg),
10276 as_Register($src2$$reg),
10277 Assembler::LSL,
10278 $src3$$constant & 0x1f);
10279 %}
10280
10281 ins_pipe(ialu_reg_reg_shift);
10282 %}
10283
10284 // This pattern is automatically generated from aarch64_ad.m4
10285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10286 // val & (-1 ^ (val << shift)) ==> bic
10287 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10288 iRegL src1, iRegL src2,
10289 immI src3, immL_M1 src4) %{
10290 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10291 ins_cost(1.9 * INSN_COST);
10292 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10293
10294 ins_encode %{
10295 __ bic(as_Register($dst$$reg),
10296 as_Register($src1$$reg),
10297 as_Register($src2$$reg),
10298 Assembler::LSL,
10299 $src3$$constant & 0x3f);
10300 %}
10301
10302 ins_pipe(ialu_reg_reg_shift);
10303 %}
10304
10305 // This pattern is automatically generated from aarch64_ad.m4
10306 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10307 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10308 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10309 iRegIorL2I src1, iRegIorL2I src2,
10310 immI src3, immI_M1 src4) %{
10311 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10312 ins_cost(1.9 * INSN_COST);
10313 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10314
10315 ins_encode %{
10316 __ eonw(as_Register($dst$$reg),
10317 as_Register($src1$$reg),
10318 as_Register($src2$$reg),
10319 Assembler::LSR,
10320 $src3$$constant & 0x1f);
10321 %}
10322
10323 ins_pipe(ialu_reg_reg_shift);
10324 %}
10325
10326 // This pattern is automatically generated from aarch64_ad.m4
10327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10328 // val ^ (-1 ^ (val >>> shift)) ==> eon
10329 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10330 iRegL src1, iRegL src2,
10331 immI src3, immL_M1 src4) %{
10332 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10333 ins_cost(1.9 * INSN_COST);
10334 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10335
10336 ins_encode %{
10337 __ eon(as_Register($dst$$reg),
10338 as_Register($src1$$reg),
10339 as_Register($src2$$reg),
10340 Assembler::LSR,
10341 $src3$$constant & 0x3f);
10342 %}
10343
10344 ins_pipe(ialu_reg_reg_shift);
10345 %}
10346
10347 // This pattern is automatically generated from aarch64_ad.m4
10348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10349 // val ^ (-1 ^ (val >> shift)) ==> eonw
10350 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10351 iRegIorL2I src1, iRegIorL2I src2,
10352 immI src3, immI_M1 src4) %{
10353 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10354 ins_cost(1.9 * INSN_COST);
10355 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10356
10357 ins_encode %{
10358 __ eonw(as_Register($dst$$reg),
10359 as_Register($src1$$reg),
10360 as_Register($src2$$reg),
10361 Assembler::ASR,
10362 $src3$$constant & 0x1f);
10363 %}
10364
10365 ins_pipe(ialu_reg_reg_shift);
10366 %}
10367
10368 // This pattern is automatically generated from aarch64_ad.m4
10369 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10370 // val ^ (-1 ^ (val >> shift)) ==> eon
10371 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10372 iRegL src1, iRegL src2,
10373 immI src3, immL_M1 src4) %{
10374 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10375 ins_cost(1.9 * INSN_COST);
10376 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10377
10378 ins_encode %{
10379 __ eon(as_Register($dst$$reg),
10380 as_Register($src1$$reg),
10381 as_Register($src2$$reg),
10382 Assembler::ASR,
10383 $src3$$constant & 0x3f);
10384 %}
10385
10386 ins_pipe(ialu_reg_reg_shift);
10387 %}
10388
10389 // This pattern is automatically generated from aarch64_ad.m4
10390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10391 // val ^ (-1 ^ (val ror shift)) ==> eonw
10392 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10393 iRegIorL2I src1, iRegIorL2I src2,
10394 immI src3, immI_M1 src4) %{
10395 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10396 ins_cost(1.9 * INSN_COST);
10397 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10398
10399 ins_encode %{
10400 __ eonw(as_Register($dst$$reg),
10401 as_Register($src1$$reg),
10402 as_Register($src2$$reg),
10403 Assembler::ROR,
10404 $src3$$constant & 0x1f);
10405 %}
10406
10407 ins_pipe(ialu_reg_reg_shift);
10408 %}
10409
10410 // This pattern is automatically generated from aarch64_ad.m4
10411 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10412 // val ^ (-1 ^ (val ror shift)) ==> eon
10413 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10414 iRegL src1, iRegL src2,
10415 immI src3, immL_M1 src4) %{
10416 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10417 ins_cost(1.9 * INSN_COST);
10418 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10419
10420 ins_encode %{
10421 __ eon(as_Register($dst$$reg),
10422 as_Register($src1$$reg),
10423 as_Register($src2$$reg),
10424 Assembler::ROR,
10425 $src3$$constant & 0x3f);
10426 %}
10427
10428 ins_pipe(ialu_reg_reg_shift);
10429 %}
10430
10431 // This pattern is automatically generated from aarch64_ad.m4
10432 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10433 // val ^ (-1 ^ (val << shift)) ==> eonw
10434 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10435 iRegIorL2I src1, iRegIorL2I src2,
10436 immI src3, immI_M1 src4) %{
10437 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10438 ins_cost(1.9 * INSN_COST);
10439 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10440
10441 ins_encode %{
10442 __ eonw(as_Register($dst$$reg),
10443 as_Register($src1$$reg),
10444 as_Register($src2$$reg),
10445 Assembler::LSL,
10446 $src3$$constant & 0x1f);
10447 %}
10448
10449 ins_pipe(ialu_reg_reg_shift);
10450 %}
10451
10452 // This pattern is automatically generated from aarch64_ad.m4
10453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10454 // val ^ (-1 ^ (val << shift)) ==> eon
10455 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10456 iRegL src1, iRegL src2,
10457 immI src3, immL_M1 src4) %{
10458 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10459 ins_cost(1.9 * INSN_COST);
10460 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10461
10462 ins_encode %{
10463 __ eon(as_Register($dst$$reg),
10464 as_Register($src1$$reg),
10465 as_Register($src2$$reg),
10466 Assembler::LSL,
10467 $src3$$constant & 0x3f);
10468 %}
10469
10470 ins_pipe(ialu_reg_reg_shift);
10471 %}
10472
10473 // This pattern is automatically generated from aarch64_ad.m4
10474 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10475 // val | (-1 ^ (val >>> shift)) ==> ornw
10476 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10477 iRegIorL2I src1, iRegIorL2I src2,
10478 immI src3, immI_M1 src4) %{
10479 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10480 ins_cost(1.9 * INSN_COST);
10481 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10482
10483 ins_encode %{
10484 __ ornw(as_Register($dst$$reg),
10485 as_Register($src1$$reg),
10486 as_Register($src2$$reg),
10487 Assembler::LSR,
10488 $src3$$constant & 0x1f);
10489 %}
10490
10491 ins_pipe(ialu_reg_reg_shift);
10492 %}
10493
10494 // This pattern is automatically generated from aarch64_ad.m4
10495 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10496 // val | (-1 ^ (val >>> shift)) ==> orn
10497 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10498 iRegL src1, iRegL src2,
10499 immI src3, immL_M1 src4) %{
10500 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10501 ins_cost(1.9 * INSN_COST);
10502 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10503
10504 ins_encode %{
10505 __ orn(as_Register($dst$$reg),
10506 as_Register($src1$$reg),
10507 as_Register($src2$$reg),
10508 Assembler::LSR,
10509 $src3$$constant & 0x3f);
10510 %}
10511
10512 ins_pipe(ialu_reg_reg_shift);
10513 %}
10514
10515 // This pattern is automatically generated from aarch64_ad.m4
10516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10517 // val | (-1 ^ (val >> shift)) ==> ornw
10518 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10519 iRegIorL2I src1, iRegIorL2I src2,
10520 immI src3, immI_M1 src4) %{
10521 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10522 ins_cost(1.9 * INSN_COST);
10523 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10524
10525 ins_encode %{
10526 __ ornw(as_Register($dst$$reg),
10527 as_Register($src1$$reg),
10528 as_Register($src2$$reg),
10529 Assembler::ASR,
10530 $src3$$constant & 0x1f);
10531 %}
10532
10533 ins_pipe(ialu_reg_reg_shift);
10534 %}
10535
10536 // This pattern is automatically generated from aarch64_ad.m4
10537 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10538 // val | (-1 ^ (val >> shift)) ==> orn
10539 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10540 iRegL src1, iRegL src2,
10541 immI src3, immL_M1 src4) %{
10542 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10543 ins_cost(1.9 * INSN_COST);
10544 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10545
10546 ins_encode %{
10547 __ orn(as_Register($dst$$reg),
10548 as_Register($src1$$reg),
10549 as_Register($src2$$reg),
10550 Assembler::ASR,
10551 $src3$$constant & 0x3f);
10552 %}
10553
10554 ins_pipe(ialu_reg_reg_shift);
10555 %}
10556
10557 // This pattern is automatically generated from aarch64_ad.m4
10558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10559 // val | (-1 ^ (val ror shift)) ==> ornw
10560 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10561 iRegIorL2I src1, iRegIorL2I src2,
10562 immI src3, immI_M1 src4) %{
10563 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10564 ins_cost(1.9 * INSN_COST);
10565 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10566
10567 ins_encode %{
10568 __ ornw(as_Register($dst$$reg),
10569 as_Register($src1$$reg),
10570 as_Register($src2$$reg),
10571 Assembler::ROR,
10572 $src3$$constant & 0x1f);
10573 %}
10574
10575 ins_pipe(ialu_reg_reg_shift);
10576 %}
10577
10578 // This pattern is automatically generated from aarch64_ad.m4
10579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10580 // val | (-1 ^ (val ror shift)) ==> orn
10581 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10582 iRegL src1, iRegL src2,
10583 immI src3, immL_M1 src4) %{
10584 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10585 ins_cost(1.9 * INSN_COST);
10586 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10587
10588 ins_encode %{
10589 __ orn(as_Register($dst$$reg),
10590 as_Register($src1$$reg),
10591 as_Register($src2$$reg),
10592 Assembler::ROR,
10593 $src3$$constant & 0x3f);
10594 %}
10595
10596 ins_pipe(ialu_reg_reg_shift);
10597 %}
10598
10599 // This pattern is automatically generated from aarch64_ad.m4
10600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10601 // val | (-1 ^ (val << shift)) ==> ornw
10602 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10603 iRegIorL2I src1, iRegIorL2I src2,
10604 immI src3, immI_M1 src4) %{
10605 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10606 ins_cost(1.9 * INSN_COST);
10607 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10608
10609 ins_encode %{
10610 __ ornw(as_Register($dst$$reg),
10611 as_Register($src1$$reg),
10612 as_Register($src2$$reg),
10613 Assembler::LSL,
10614 $src3$$constant & 0x1f);
10615 %}
10616
10617 ins_pipe(ialu_reg_reg_shift);
10618 %}
10619
10620 // This pattern is automatically generated from aarch64_ad.m4
10621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10622 // val | (-1 ^ (val << shift)) ==> orn
10623 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10624 iRegL src1, iRegL src2,
10625 immI src3, immL_M1 src4) %{
10626 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10627 ins_cost(1.9 * INSN_COST);
10628 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10629
10630 ins_encode %{
10631 __ orn(as_Register($dst$$reg),
10632 as_Register($src1$$reg),
10633 as_Register($src2$$reg),
10634 Assembler::LSL,
10635 $src3$$constant & 0x3f);
10636 %}
10637
10638 ins_pipe(ialu_reg_reg_shift);
10639 %}
10640
10641 // This pattern is automatically generated from aarch64_ad.m4
10642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10643 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10644 iRegIorL2I src1, iRegIorL2I src2,
10645 immI src3) %{
10646 match(Set dst (AndI src1 (URShiftI src2 src3)));
10647
10648 ins_cost(1.9 * INSN_COST);
10649 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10650
10651 ins_encode %{
10652 __ andw(as_Register($dst$$reg),
10653 as_Register($src1$$reg),
10654 as_Register($src2$$reg),
10655 Assembler::LSR,
10656 $src3$$constant & 0x1f);
10657 %}
10658
10659 ins_pipe(ialu_reg_reg_shift);
10660 %}
10661
10662 // This pattern is automatically generated from aarch64_ad.m4
10663 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10664 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10665 iRegL src1, iRegL src2,
10666 immI src3) %{
10667 match(Set dst (AndL src1 (URShiftL src2 src3)));
10668
10669 ins_cost(1.9 * INSN_COST);
10670 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10671
10672 ins_encode %{
10673 __ andr(as_Register($dst$$reg),
10674 as_Register($src1$$reg),
10675 as_Register($src2$$reg),
10676 Assembler::LSR,
10677 $src3$$constant & 0x3f);
10678 %}
10679
10680 ins_pipe(ialu_reg_reg_shift);
10681 %}
10682
10683 // This pattern is automatically generated from aarch64_ad.m4
10684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10685 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10686 iRegIorL2I src1, iRegIorL2I src2,
10687 immI src3) %{
10688 match(Set dst (AndI src1 (RShiftI src2 src3)));
10689
10690 ins_cost(1.9 * INSN_COST);
10691 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10692
10693 ins_encode %{
10694 __ andw(as_Register($dst$$reg),
10695 as_Register($src1$$reg),
10696 as_Register($src2$$reg),
10697 Assembler::ASR,
10698 $src3$$constant & 0x1f);
10699 %}
10700
10701 ins_pipe(ialu_reg_reg_shift);
10702 %}
10703
10704 // This pattern is automatically generated from aarch64_ad.m4
10705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10706 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10707 iRegL src1, iRegL src2,
10708 immI src3) %{
10709 match(Set dst (AndL src1 (RShiftL src2 src3)));
10710
10711 ins_cost(1.9 * INSN_COST);
10712 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10713
10714 ins_encode %{
10715 __ andr(as_Register($dst$$reg),
10716 as_Register($src1$$reg),
10717 as_Register($src2$$reg),
10718 Assembler::ASR,
10719 $src3$$constant & 0x3f);
10720 %}
10721
10722 ins_pipe(ialu_reg_reg_shift);
10723 %}
10724
10725 // This pattern is automatically generated from aarch64_ad.m4
10726 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10727 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10728 iRegIorL2I src1, iRegIorL2I src2,
10729 immI src3) %{
10730 match(Set dst (AndI src1 (LShiftI src2 src3)));
10731
10732 ins_cost(1.9 * INSN_COST);
10733 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10734
10735 ins_encode %{
10736 __ andw(as_Register($dst$$reg),
10737 as_Register($src1$$reg),
10738 as_Register($src2$$reg),
10739 Assembler::LSL,
10740 $src3$$constant & 0x1f);
10741 %}
10742
10743 ins_pipe(ialu_reg_reg_shift);
10744 %}
10745
10746 // This pattern is automatically generated from aarch64_ad.m4
10747 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10748 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10749 iRegL src1, iRegL src2,
10750 immI src3) %{
10751 match(Set dst (AndL src1 (LShiftL src2 src3)));
10752
10753 ins_cost(1.9 * INSN_COST);
10754 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10755
10756 ins_encode %{
10757 __ andr(as_Register($dst$$reg),
10758 as_Register($src1$$reg),
10759 as_Register($src2$$reg),
10760 Assembler::LSL,
10761 $src3$$constant & 0x3f);
10762 %}
10763
10764 ins_pipe(ialu_reg_reg_shift);
10765 %}
10766
10767 // This pattern is automatically generated from aarch64_ad.m4
10768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10769 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10770 iRegIorL2I src1, iRegIorL2I src2,
10771 immI src3) %{
10772 match(Set dst (AndI src1 (RotateRight src2 src3)));
10773
10774 ins_cost(1.9 * INSN_COST);
10775 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10776
10777 ins_encode %{
10778 __ andw(as_Register($dst$$reg),
10779 as_Register($src1$$reg),
10780 as_Register($src2$$reg),
10781 Assembler::ROR,
10782 $src3$$constant & 0x1f);
10783 %}
10784
10785 ins_pipe(ialu_reg_reg_shift);
10786 %}
10787
10788 // This pattern is automatically generated from aarch64_ad.m4
10789 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10790 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10791 iRegL src1, iRegL src2,
10792 immI src3) %{
10793 match(Set dst (AndL src1 (RotateRight src2 src3)));
10794
10795 ins_cost(1.9 * INSN_COST);
10796 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10797
10798 ins_encode %{
10799 __ andr(as_Register($dst$$reg),
10800 as_Register($src1$$reg),
10801 as_Register($src2$$reg),
10802 Assembler::ROR,
10803 $src3$$constant & 0x3f);
10804 %}
10805
10806 ins_pipe(ialu_reg_reg_shift);
10807 %}
10808
10809 // This pattern is automatically generated from aarch64_ad.m4
10810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10811 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10812 iRegIorL2I src1, iRegIorL2I src2,
10813 immI src3) %{
10814 match(Set dst (XorI src1 (URShiftI src2 src3)));
10815
10816 ins_cost(1.9 * INSN_COST);
10817 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10818
10819 ins_encode %{
10820 __ eorw(as_Register($dst$$reg),
10821 as_Register($src1$$reg),
10822 as_Register($src2$$reg),
10823 Assembler::LSR,
10824 $src3$$constant & 0x1f);
10825 %}
10826
10827 ins_pipe(ialu_reg_reg_shift);
10828 %}
10829
10830 // This pattern is automatically generated from aarch64_ad.m4
10831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10832 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10833 iRegL src1, iRegL src2,
10834 immI src3) %{
10835 match(Set dst (XorL src1 (URShiftL src2 src3)));
10836
10837 ins_cost(1.9 * INSN_COST);
10838 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10839
10840 ins_encode %{
10841 __ eor(as_Register($dst$$reg),
10842 as_Register($src1$$reg),
10843 as_Register($src2$$reg),
10844 Assembler::LSR,
10845 $src3$$constant & 0x3f);
10846 %}
10847
10848 ins_pipe(ialu_reg_reg_shift);
10849 %}
10850
10851 // This pattern is automatically generated from aarch64_ad.m4
10852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10853 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10854 iRegIorL2I src1, iRegIorL2I src2,
10855 immI src3) %{
10856 match(Set dst (XorI src1 (RShiftI src2 src3)));
10857
10858 ins_cost(1.9 * INSN_COST);
10859 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10860
10861 ins_encode %{
10862 __ eorw(as_Register($dst$$reg),
10863 as_Register($src1$$reg),
10864 as_Register($src2$$reg),
10865 Assembler::ASR,
10866 $src3$$constant & 0x1f);
10867 %}
10868
10869 ins_pipe(ialu_reg_reg_shift);
10870 %}
10871
10872 // This pattern is automatically generated from aarch64_ad.m4
10873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10874 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10875 iRegL src1, iRegL src2,
10876 immI src3) %{
10877 match(Set dst (XorL src1 (RShiftL src2 src3)));
10878
10879 ins_cost(1.9 * INSN_COST);
10880 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10881
10882 ins_encode %{
10883 __ eor(as_Register($dst$$reg),
10884 as_Register($src1$$reg),
10885 as_Register($src2$$reg),
10886 Assembler::ASR,
10887 $src3$$constant & 0x3f);
10888 %}
10889
10890 ins_pipe(ialu_reg_reg_shift);
10891 %}
10892
10893 // This pattern is automatically generated from aarch64_ad.m4
10894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10895 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10896 iRegIorL2I src1, iRegIorL2I src2,
10897 immI src3) %{
10898 match(Set dst (XorI src1 (LShiftI src2 src3)));
10899
10900 ins_cost(1.9 * INSN_COST);
10901 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10902
10903 ins_encode %{
10904 __ eorw(as_Register($dst$$reg),
10905 as_Register($src1$$reg),
10906 as_Register($src2$$reg),
10907 Assembler::LSL,
10908 $src3$$constant & 0x1f);
10909 %}
10910
10911 ins_pipe(ialu_reg_reg_shift);
10912 %}
10913
10914 // This pattern is automatically generated from aarch64_ad.m4
10915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10916 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10917 iRegL src1, iRegL src2,
10918 immI src3) %{
10919 match(Set dst (XorL src1 (LShiftL src2 src3)));
10920
10921 ins_cost(1.9 * INSN_COST);
10922 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10923
10924 ins_encode %{
10925 __ eor(as_Register($dst$$reg),
10926 as_Register($src1$$reg),
10927 as_Register($src2$$reg),
10928 Assembler::LSL,
10929 $src3$$constant & 0x3f);
10930 %}
10931
10932 ins_pipe(ialu_reg_reg_shift);
10933 %}
10934
10935 // This pattern is automatically generated from aarch64_ad.m4
10936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10937 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10938 iRegIorL2I src1, iRegIorL2I src2,
10939 immI src3) %{
10940 match(Set dst (XorI src1 (RotateRight src2 src3)));
10941
10942 ins_cost(1.9 * INSN_COST);
10943 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10944
10945 ins_encode %{
10946 __ eorw(as_Register($dst$$reg),
10947 as_Register($src1$$reg),
10948 as_Register($src2$$reg),
10949 Assembler::ROR,
10950 $src3$$constant & 0x1f);
10951 %}
10952
10953 ins_pipe(ialu_reg_reg_shift);
10954 %}
10955
10956 // This pattern is automatically generated from aarch64_ad.m4
10957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10958 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10959 iRegL src1, iRegL src2,
10960 immI src3) %{
10961 match(Set dst (XorL src1 (RotateRight src2 src3)));
10962
10963 ins_cost(1.9 * INSN_COST);
10964 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10965
10966 ins_encode %{
10967 __ eor(as_Register($dst$$reg),
10968 as_Register($src1$$reg),
10969 as_Register($src2$$reg),
10970 Assembler::ROR,
10971 $src3$$constant & 0x3f);
10972 %}
10973
10974 ins_pipe(ialu_reg_reg_shift);
10975 %}
10976
10977 // This pattern is automatically generated from aarch64_ad.m4
10978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10979 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10980 iRegIorL2I src1, iRegIorL2I src2,
10981 immI src3) %{
10982 match(Set dst (OrI src1 (URShiftI src2 src3)));
10983
10984 ins_cost(1.9 * INSN_COST);
10985 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10986
10987 ins_encode %{
10988 __ orrw(as_Register($dst$$reg),
10989 as_Register($src1$$reg),
10990 as_Register($src2$$reg),
10991 Assembler::LSR,
10992 $src3$$constant & 0x1f);
10993 %}
10994
10995 ins_pipe(ialu_reg_reg_shift);
10996 %}
10997
10998 // This pattern is automatically generated from aarch64_ad.m4
10999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11000 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11001 iRegL src1, iRegL src2,
11002 immI src3) %{
11003 match(Set dst (OrL src1 (URShiftL src2 src3)));
11004
11005 ins_cost(1.9 * INSN_COST);
11006 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11007
11008 ins_encode %{
11009 __ orr(as_Register($dst$$reg),
11010 as_Register($src1$$reg),
11011 as_Register($src2$$reg),
11012 Assembler::LSR,
11013 $src3$$constant & 0x3f);
11014 %}
11015
11016 ins_pipe(ialu_reg_reg_shift);
11017 %}
11018
11019 // This pattern is automatically generated from aarch64_ad.m4
11020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11021 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11022 iRegIorL2I src1, iRegIorL2I src2,
11023 immI src3) %{
11024 match(Set dst (OrI src1 (RShiftI src2 src3)));
11025
11026 ins_cost(1.9 * INSN_COST);
11027 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11028
11029 ins_encode %{
11030 __ orrw(as_Register($dst$$reg),
11031 as_Register($src1$$reg),
11032 as_Register($src2$$reg),
11033 Assembler::ASR,
11034 $src3$$constant & 0x1f);
11035 %}
11036
11037 ins_pipe(ialu_reg_reg_shift);
11038 %}
11039
11040 // This pattern is automatically generated from aarch64_ad.m4
11041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11042 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11043 iRegL src1, iRegL src2,
11044 immI src3) %{
11045 match(Set dst (OrL src1 (RShiftL src2 src3)));
11046
11047 ins_cost(1.9 * INSN_COST);
11048 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11049
11050 ins_encode %{
11051 __ orr(as_Register($dst$$reg),
11052 as_Register($src1$$reg),
11053 as_Register($src2$$reg),
11054 Assembler::ASR,
11055 $src3$$constant & 0x3f);
11056 %}
11057
11058 ins_pipe(ialu_reg_reg_shift);
11059 %}
11060
11061 // This pattern is automatically generated from aarch64_ad.m4
11062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11063 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11064 iRegIorL2I src1, iRegIorL2I src2,
11065 immI src3) %{
11066 match(Set dst (OrI src1 (LShiftI src2 src3)));
11067
11068 ins_cost(1.9 * INSN_COST);
11069 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11070
11071 ins_encode %{
11072 __ orrw(as_Register($dst$$reg),
11073 as_Register($src1$$reg),
11074 as_Register($src2$$reg),
11075 Assembler::LSL,
11076 $src3$$constant & 0x1f);
11077 %}
11078
11079 ins_pipe(ialu_reg_reg_shift);
11080 %}
11081
11082 // This pattern is automatically generated from aarch64_ad.m4
11083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11084 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11085 iRegL src1, iRegL src2,
11086 immI src3) %{
11087 match(Set dst (OrL src1 (LShiftL src2 src3)));
11088
11089 ins_cost(1.9 * INSN_COST);
11090 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11091
11092 ins_encode %{
11093 __ orr(as_Register($dst$$reg),
11094 as_Register($src1$$reg),
11095 as_Register($src2$$reg),
11096 Assembler::LSL,
11097 $src3$$constant & 0x3f);
11098 %}
11099
11100 ins_pipe(ialu_reg_reg_shift);
11101 %}
11102
11103 // This pattern is automatically generated from aarch64_ad.m4
11104 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11105 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11106 iRegIorL2I src1, iRegIorL2I src2,
11107 immI src3) %{
11108 match(Set dst (OrI src1 (RotateRight src2 src3)));
11109
11110 ins_cost(1.9 * INSN_COST);
11111 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11112
11113 ins_encode %{
11114 __ orrw(as_Register($dst$$reg),
11115 as_Register($src1$$reg),
11116 as_Register($src2$$reg),
11117 Assembler::ROR,
11118 $src3$$constant & 0x1f);
11119 %}
11120
11121 ins_pipe(ialu_reg_reg_shift);
11122 %}
11123
11124 // This pattern is automatically generated from aarch64_ad.m4
11125 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11126 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11127 iRegL src1, iRegL src2,
11128 immI src3) %{
11129 match(Set dst (OrL src1 (RotateRight src2 src3)));
11130
11131 ins_cost(1.9 * INSN_COST);
11132 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11133
11134 ins_encode %{
11135 __ orr(as_Register($dst$$reg),
11136 as_Register($src1$$reg),
11137 as_Register($src2$$reg),
11138 Assembler::ROR,
11139 $src3$$constant & 0x3f);
11140 %}
11141
11142 ins_pipe(ialu_reg_reg_shift);
11143 %}
11144
11145 // This pattern is automatically generated from aarch64_ad.m4
11146 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11147 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11148 iRegIorL2I src1, iRegIorL2I src2,
11149 immI src3) %{
11150 match(Set dst (AddI src1 (URShiftI src2 src3)));
11151
11152 ins_cost(1.9 * INSN_COST);
11153 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11154
11155 ins_encode %{
11156 __ addw(as_Register($dst$$reg),
11157 as_Register($src1$$reg),
11158 as_Register($src2$$reg),
11159 Assembler::LSR,
11160 $src3$$constant & 0x1f);
11161 %}
11162
11163 ins_pipe(ialu_reg_reg_shift);
11164 %}
11165
11166 // This pattern is automatically generated from aarch64_ad.m4
11167 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11168 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11169 iRegL src1, iRegL src2,
11170 immI src3) %{
11171 match(Set dst (AddL src1 (URShiftL src2 src3)));
11172
11173 ins_cost(1.9 * INSN_COST);
11174 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11175
11176 ins_encode %{
11177 __ add(as_Register($dst$$reg),
11178 as_Register($src1$$reg),
11179 as_Register($src2$$reg),
11180 Assembler::LSR,
11181 $src3$$constant & 0x3f);
11182 %}
11183
11184 ins_pipe(ialu_reg_reg_shift);
11185 %}
11186
11187 // This pattern is automatically generated from aarch64_ad.m4
11188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11189 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11190 iRegIorL2I src1, iRegIorL2I src2,
11191 immI src3) %{
11192 match(Set dst (AddI src1 (RShiftI src2 src3)));
11193
11194 ins_cost(1.9 * INSN_COST);
11195 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11196
11197 ins_encode %{
11198 __ addw(as_Register($dst$$reg),
11199 as_Register($src1$$reg),
11200 as_Register($src2$$reg),
11201 Assembler::ASR,
11202 $src3$$constant & 0x1f);
11203 %}
11204
11205 ins_pipe(ialu_reg_reg_shift);
11206 %}
11207
11208 // This pattern is automatically generated from aarch64_ad.m4
11209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11210 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11211 iRegL src1, iRegL src2,
11212 immI src3) %{
11213 match(Set dst (AddL src1 (RShiftL src2 src3)));
11214
11215 ins_cost(1.9 * INSN_COST);
11216 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11217
11218 ins_encode %{
11219 __ add(as_Register($dst$$reg),
11220 as_Register($src1$$reg),
11221 as_Register($src2$$reg),
11222 Assembler::ASR,
11223 $src3$$constant & 0x3f);
11224 %}
11225
11226 ins_pipe(ialu_reg_reg_shift);
11227 %}
11228
11229 // This pattern is automatically generated from aarch64_ad.m4
11230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11231 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11232 iRegIorL2I src1, iRegIorL2I src2,
11233 immI src3) %{
11234 match(Set dst (AddI src1 (LShiftI src2 src3)));
11235
11236 ins_cost(1.9 * INSN_COST);
11237 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11238
11239 ins_encode %{
11240 __ addw(as_Register($dst$$reg),
11241 as_Register($src1$$reg),
11242 as_Register($src2$$reg),
11243 Assembler::LSL,
11244 $src3$$constant & 0x1f);
11245 %}
11246
11247 ins_pipe(ialu_reg_reg_shift);
11248 %}
11249
11250 // This pattern is automatically generated from aarch64_ad.m4
11251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11252 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11253 iRegL src1, iRegL src2,
11254 immI src3) %{
11255 match(Set dst (AddL src1 (LShiftL src2 src3)));
11256
11257 ins_cost(1.9 * INSN_COST);
11258 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11259
11260 ins_encode %{
11261 __ add(as_Register($dst$$reg),
11262 as_Register($src1$$reg),
11263 as_Register($src2$$reg),
11264 Assembler::LSL,
11265 $src3$$constant & 0x3f);
11266 %}
11267
11268 ins_pipe(ialu_reg_reg_shift);
11269 %}
11270
11271 // This pattern is automatically generated from aarch64_ad.m4
11272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11273 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11274 iRegIorL2I src1, iRegIorL2I src2,
11275 immI src3) %{
11276 match(Set dst (SubI src1 (URShiftI src2 src3)));
11277
11278 ins_cost(1.9 * INSN_COST);
11279 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11280
11281 ins_encode %{
11282 __ subw(as_Register($dst$$reg),
11283 as_Register($src1$$reg),
11284 as_Register($src2$$reg),
11285 Assembler::LSR,
11286 $src3$$constant & 0x1f);
11287 %}
11288
11289 ins_pipe(ialu_reg_reg_shift);
11290 %}
11291
11292 // This pattern is automatically generated from aarch64_ad.m4
11293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11294 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11295 iRegL src1, iRegL src2,
11296 immI src3) %{
11297 match(Set dst (SubL src1 (URShiftL src2 src3)));
11298
11299 ins_cost(1.9 * INSN_COST);
11300 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11301
11302 ins_encode %{
11303 __ sub(as_Register($dst$$reg),
11304 as_Register($src1$$reg),
11305 as_Register($src2$$reg),
11306 Assembler::LSR,
11307 $src3$$constant & 0x3f);
11308 %}
11309
11310 ins_pipe(ialu_reg_reg_shift);
11311 %}
11312
11313 // This pattern is automatically generated from aarch64_ad.m4
11314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11315 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11316 iRegIorL2I src1, iRegIorL2I src2,
11317 immI src3) %{
11318 match(Set dst (SubI src1 (RShiftI src2 src3)));
11319
11320 ins_cost(1.9 * INSN_COST);
11321 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11322
11323 ins_encode %{
11324 __ subw(as_Register($dst$$reg),
11325 as_Register($src1$$reg),
11326 as_Register($src2$$reg),
11327 Assembler::ASR,
11328 $src3$$constant & 0x1f);
11329 %}
11330
11331 ins_pipe(ialu_reg_reg_shift);
11332 %}
11333
11334 // This pattern is automatically generated from aarch64_ad.m4
11335 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11336 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11337 iRegL src1, iRegL src2,
11338 immI src3) %{
11339 match(Set dst (SubL src1 (RShiftL src2 src3)));
11340
11341 ins_cost(1.9 * INSN_COST);
11342 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11343
11344 ins_encode %{
11345 __ sub(as_Register($dst$$reg),
11346 as_Register($src1$$reg),
11347 as_Register($src2$$reg),
11348 Assembler::ASR,
11349 $src3$$constant & 0x3f);
11350 %}
11351
11352 ins_pipe(ialu_reg_reg_shift);
11353 %}
11354
11355 // This pattern is automatically generated from aarch64_ad.m4
11356 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11357 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11358 iRegIorL2I src1, iRegIorL2I src2,
11359 immI src3) %{
11360 match(Set dst (SubI src1 (LShiftI src2 src3)));
11361
11362 ins_cost(1.9 * INSN_COST);
11363 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11364
11365 ins_encode %{
11366 __ subw(as_Register($dst$$reg),
11367 as_Register($src1$$reg),
11368 as_Register($src2$$reg),
11369 Assembler::LSL,
11370 $src3$$constant & 0x1f);
11371 %}
11372
11373 ins_pipe(ialu_reg_reg_shift);
11374 %}
11375
11376 // This pattern is automatically generated from aarch64_ad.m4
11377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11378 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11379 iRegL src1, iRegL src2,
11380 immI src3) %{
11381 match(Set dst (SubL src1 (LShiftL src2 src3)));
11382
11383 ins_cost(1.9 * INSN_COST);
11384 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11385
11386 ins_encode %{
11387 __ sub(as_Register($dst$$reg),
11388 as_Register($src1$$reg),
11389 as_Register($src2$$reg),
11390 Assembler::LSL,
11391 $src3$$constant & 0x3f);
11392 %}
11393
11394 ins_pipe(ialu_reg_reg_shift);
11395 %}
11396
11397 // This pattern is automatically generated from aarch64_ad.m4
11398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11399
11400 // Shift Left followed by Shift Right.
11401 // This idiom is used by the compiler for the i2b bytecode etc.
11402 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11403 %{
11404 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11405 ins_cost(INSN_COST * 2);
11406 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11407 ins_encode %{
11408 int lshift = $lshift_count$$constant & 63;
11409 int rshift = $rshift_count$$constant & 63;
11410 int s = 63 - lshift;
11411 int r = (rshift - lshift) & 63;
11412 __ sbfm(as_Register($dst$$reg),
11413 as_Register($src$$reg),
11414 r, s);
11415 %}
11416
11417 ins_pipe(ialu_reg_shift);
11418 %}
11419
11420 // This pattern is automatically generated from aarch64_ad.m4
11421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11422
11423 // Shift Left followed by Shift Right.
11424 // This idiom is used by the compiler for the i2b bytecode etc.
11425 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11426 %{
11427 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11428 ins_cost(INSN_COST * 2);
11429 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11430 ins_encode %{
11431 int lshift = $lshift_count$$constant & 31;
11432 int rshift = $rshift_count$$constant & 31;
11433 int s = 31 - lshift;
11434 int r = (rshift - lshift) & 31;
11435 __ sbfmw(as_Register($dst$$reg),
11436 as_Register($src$$reg),
11437 r, s);
11438 %}
11439
11440 ins_pipe(ialu_reg_shift);
11441 %}
11442
11443 // This pattern is automatically generated from aarch64_ad.m4
11444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11445
11446 // Shift Left followed by Shift Right.
11447 // This idiom is used by the compiler for the i2b bytecode etc.
11448 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11449 %{
11450 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11451 ins_cost(INSN_COST * 2);
11452 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11453 ins_encode %{
11454 int lshift = $lshift_count$$constant & 63;
11455 int rshift = $rshift_count$$constant & 63;
11456 int s = 63 - lshift;
11457 int r = (rshift - lshift) & 63;
11458 __ ubfm(as_Register($dst$$reg),
11459 as_Register($src$$reg),
11460 r, s);
11461 %}
11462
11463 ins_pipe(ialu_reg_shift);
11464 %}
11465
11466 // This pattern is automatically generated from aarch64_ad.m4
11467 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11468
11469 // Shift Left followed by Shift Right.
11470 // This idiom is used by the compiler for the i2b bytecode etc.
11471 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11472 %{
11473 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11474 ins_cost(INSN_COST * 2);
11475 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11476 ins_encode %{
11477 int lshift = $lshift_count$$constant & 31;
11478 int rshift = $rshift_count$$constant & 31;
11479 int s = 31 - lshift;
11480 int r = (rshift - lshift) & 31;
11481 __ ubfmw(as_Register($dst$$reg),
11482 as_Register($src$$reg),
11483 r, s);
11484 %}
11485
11486 ins_pipe(ialu_reg_shift);
11487 %}
11488
11489 // Bitfield extract with shift & mask
11490
11491 // This pattern is automatically generated from aarch64_ad.m4
11492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11493 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11494 %{
11495 match(Set dst (AndI (URShiftI src rshift) mask));
11496 // Make sure we are not going to exceed what ubfxw can do.
11497 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11498
11499 ins_cost(INSN_COST);
11500 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11501 ins_encode %{
11502 int rshift = $rshift$$constant & 31;
11503 intptr_t mask = $mask$$constant;
11504 int width = exact_log2(mask+1);
11505 __ ubfxw(as_Register($dst$$reg),
11506 as_Register($src$$reg), rshift, width);
11507 %}
11508 ins_pipe(ialu_reg_shift);
11509 %}
11510
11511 // This pattern is automatically generated from aarch64_ad.m4
11512 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11513 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11514 %{
11515 match(Set dst (AndL (URShiftL src rshift) mask));
11516 // Make sure we are not going to exceed what ubfx can do.
11517 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11518
11519 ins_cost(INSN_COST);
11520 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11521 ins_encode %{
11522 int rshift = $rshift$$constant & 63;
11523 intptr_t mask = $mask$$constant;
11524 int width = exact_log2_long(mask+1);
11525 __ ubfx(as_Register($dst$$reg),
11526 as_Register($src$$reg), rshift, width);
11527 %}
11528 ins_pipe(ialu_reg_shift);
11529 %}
11530
11531
11532 // This pattern is automatically generated from aarch64_ad.m4
11533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11534
11535 // We can use ubfx when extending an And with a mask when we know mask
11536 // is positive. We know that because immI_bitmask guarantees it.
11537 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11538 %{
11539 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11540 // Make sure we are not going to exceed what ubfxw can do.
11541 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11542
11543 ins_cost(INSN_COST * 2);
11544 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11545 ins_encode %{
11546 int rshift = $rshift$$constant & 31;
11547 intptr_t mask = $mask$$constant;
11548 int width = exact_log2(mask+1);
11549 __ ubfx(as_Register($dst$$reg),
11550 as_Register($src$$reg), rshift, width);
11551 %}
11552 ins_pipe(ialu_reg_shift);
11553 %}
11554
11555
11556 // This pattern is automatically generated from aarch64_ad.m4
11557 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11558
11559 // We can use ubfiz when masking by a positive number and then left shifting the result.
11560 // We know that the mask is positive because immI_bitmask guarantees it.
11561 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11562 %{
11563 match(Set dst (LShiftI (AndI src mask) lshift));
11564 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11565
11566 ins_cost(INSN_COST);
11567 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11568 ins_encode %{
11569 int lshift = $lshift$$constant & 31;
11570 intptr_t mask = $mask$$constant;
11571 int width = exact_log2(mask+1);
11572 __ ubfizw(as_Register($dst$$reg),
11573 as_Register($src$$reg), lshift, width);
11574 %}
11575 ins_pipe(ialu_reg_shift);
11576 %}
11577
11578 // This pattern is automatically generated from aarch64_ad.m4
11579 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11580
11581 // We can use ubfiz when masking by a positive number and then left shifting the result.
11582 // We know that the mask is positive because immL_bitmask guarantees it.
11583 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11584 %{
11585 match(Set dst (LShiftL (AndL src mask) lshift));
11586 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11587
11588 ins_cost(INSN_COST);
11589 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11590 ins_encode %{
11591 int lshift = $lshift$$constant & 63;
11592 intptr_t mask = $mask$$constant;
11593 int width = exact_log2_long(mask+1);
11594 __ ubfiz(as_Register($dst$$reg),
11595 as_Register($src$$reg), lshift, width);
11596 %}
11597 ins_pipe(ialu_reg_shift);
11598 %}
11599
11600 // This pattern is automatically generated from aarch64_ad.m4
11601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11602
11603 // We can use ubfiz when masking by a positive number and then left shifting the result.
11604 // We know that the mask is positive because immI_bitmask guarantees it.
11605 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11606 %{
11607 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11608 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11609
11610 ins_cost(INSN_COST);
11611 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11612 ins_encode %{
11613 int lshift = $lshift$$constant & 31;
11614 intptr_t mask = $mask$$constant;
11615 int width = exact_log2(mask+1);
11616 __ ubfizw(as_Register($dst$$reg),
11617 as_Register($src$$reg), lshift, width);
11618 %}
11619 ins_pipe(ialu_reg_shift);
11620 %}
11621
11622 // This pattern is automatically generated from aarch64_ad.m4
11623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11624
11625 // We can use ubfiz when masking by a positive number and then left shifting the result.
11626 // We know that the mask is positive because immL_bitmask guarantees it.
11627 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11628 %{
11629 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11630 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11631
11632 ins_cost(INSN_COST);
11633 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11634 ins_encode %{
11635 int lshift = $lshift$$constant & 63;
11636 intptr_t mask = $mask$$constant;
11637 int width = exact_log2_long(mask+1);
11638 __ ubfiz(as_Register($dst$$reg),
11639 as_Register($src$$reg), lshift, width);
11640 %}
11641 ins_pipe(ialu_reg_shift);
11642 %}
11643
11644
11645 // This pattern is automatically generated from aarch64_ad.m4
11646 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11647
11648 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11649 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11650 %{
11651 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11652 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11653
11654 ins_cost(INSN_COST);
11655 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11656 ins_encode %{
11657 int lshift = $lshift$$constant & 63;
11658 intptr_t mask = $mask$$constant;
11659 int width = exact_log2(mask+1);
11660 __ ubfiz(as_Register($dst$$reg),
11661 as_Register($src$$reg), lshift, width);
11662 %}
11663 ins_pipe(ialu_reg_shift);
11664 %}
11665
11666 // This pattern is automatically generated from aarch64_ad.m4
11667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11668
11669 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11670 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11671 %{
11672 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11673 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11674
11675 ins_cost(INSN_COST);
11676 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11677 ins_encode %{
11678 int lshift = $lshift$$constant & 31;
11679 intptr_t mask = $mask$$constant;
11680 int width = exact_log2(mask+1);
11681 __ ubfiz(as_Register($dst$$reg),
11682 as_Register($src$$reg), lshift, width);
11683 %}
11684 ins_pipe(ialu_reg_shift);
11685 %}
11686
11687 // This pattern is automatically generated from aarch64_ad.m4
11688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11689
11690 // Can skip int2long conversions after AND with small bitmask
11691 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11692 %{
11693 match(Set dst (ConvI2L (AndI src msk)));
11694 ins_cost(INSN_COST);
11695 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11696 ins_encode %{
11697 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11698 %}
11699 ins_pipe(ialu_reg_shift);
11700 %}
11701
11702
11703 // Rotations
11704
11705 // This pattern is automatically generated from aarch64_ad.m4
11706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11707 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11708 %{
11709 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11710 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11711
11712 ins_cost(INSN_COST);
11713 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11714
11715 ins_encode %{
11716 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11717 $rshift$$constant & 63);
11718 %}
11719 ins_pipe(ialu_reg_reg_extr);
11720 %}
11721
11722
11723 // This pattern is automatically generated from aarch64_ad.m4
11724 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11725 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11726 %{
11727 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11728 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11729
11730 ins_cost(INSN_COST);
11731 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11732
11733 ins_encode %{
11734 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11735 $rshift$$constant & 31);
11736 %}
11737 ins_pipe(ialu_reg_reg_extr);
11738 %}
11739
11740
11741 // This pattern is automatically generated from aarch64_ad.m4
11742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11743 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11744 %{
11745 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11746 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11747
11748 ins_cost(INSN_COST);
11749 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11750
11751 ins_encode %{
11752 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11753 $rshift$$constant & 63);
11754 %}
11755 ins_pipe(ialu_reg_reg_extr);
11756 %}
11757
11758
11759 // This pattern is automatically generated from aarch64_ad.m4
11760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11761 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11762 %{
11763 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11764 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11765
11766 ins_cost(INSN_COST);
11767 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11768
11769 ins_encode %{
11770 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11771 $rshift$$constant & 31);
11772 %}
11773 ins_pipe(ialu_reg_reg_extr);
11774 %}
11775
11776 // This pattern is automatically generated from aarch64_ad.m4
11777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11778 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11779 %{
11780 match(Set dst (RotateRight src shift));
11781
11782 ins_cost(INSN_COST);
11783 format %{ "ror $dst, $src, $shift" %}
11784
11785 ins_encode %{
11786 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11787 $shift$$constant & 0x1f);
11788 %}
11789 ins_pipe(ialu_reg_reg_vshift);
11790 %}
11791
11792 // This pattern is automatically generated from aarch64_ad.m4
11793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11794 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11795 %{
11796 match(Set dst (RotateRight src shift));
11797
11798 ins_cost(INSN_COST);
11799 format %{ "ror $dst, $src, $shift" %}
11800
11801 ins_encode %{
11802 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11803 $shift$$constant & 0x3f);
11804 %}
11805 ins_pipe(ialu_reg_reg_vshift);
11806 %}
11807
11808 // This pattern is automatically generated from aarch64_ad.m4
11809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11810 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11811 %{
11812 match(Set dst (RotateRight src shift));
11813
11814 ins_cost(INSN_COST);
11815 format %{ "ror $dst, $src, $shift" %}
11816
11817 ins_encode %{
11818 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11819 %}
11820 ins_pipe(ialu_reg_reg_vshift);
11821 %}
11822
11823 // This pattern is automatically generated from aarch64_ad.m4
11824 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11825 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11826 %{
11827 match(Set dst (RotateRight src shift));
11828
11829 ins_cost(INSN_COST);
11830 format %{ "ror $dst, $src, $shift" %}
11831
11832 ins_encode %{
11833 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11834 %}
11835 ins_pipe(ialu_reg_reg_vshift);
11836 %}
11837
11838 // This pattern is automatically generated from aarch64_ad.m4
11839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11840 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11841 %{
11842 match(Set dst (RotateLeft src shift));
11843
11844 ins_cost(INSN_COST);
11845 format %{ "rol $dst, $src, $shift" %}
11846
11847 ins_encode %{
11848 __ subw(rscratch1, zr, as_Register($shift$$reg));
11849 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11850 %}
11851 ins_pipe(ialu_reg_reg_vshift);
11852 %}
11853
11854 // This pattern is automatically generated from aarch64_ad.m4
11855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11856 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11857 %{
11858 match(Set dst (RotateLeft src shift));
11859
11860 ins_cost(INSN_COST);
11861 format %{ "rol $dst, $src, $shift" %}
11862
11863 ins_encode %{
11864 __ subw(rscratch1, zr, as_Register($shift$$reg));
11865 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11866 %}
11867 ins_pipe(ialu_reg_reg_vshift);
11868 %}
11869
11870
11871 // Add/subtract (extended)
11872
11873 // This pattern is automatically generated from aarch64_ad.m4
11874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11875 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11876 %{
11877 match(Set dst (AddL src1 (ConvI2L src2)));
11878 ins_cost(INSN_COST);
11879 format %{ "add $dst, $src1, $src2, sxtw" %}
11880
11881 ins_encode %{
11882 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11883 as_Register($src2$$reg), ext::sxtw);
11884 %}
11885 ins_pipe(ialu_reg_reg);
11886 %}
11887
11888 // This pattern is automatically generated from aarch64_ad.m4
11889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11890 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11891 %{
11892 match(Set dst (SubL src1 (ConvI2L src2)));
11893 ins_cost(INSN_COST);
11894 format %{ "sub $dst, $src1, $src2, sxtw" %}
11895
11896 ins_encode %{
11897 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11898 as_Register($src2$$reg), ext::sxtw);
11899 %}
11900 ins_pipe(ialu_reg_reg);
11901 %}
11902
11903 // This pattern is automatically generated from aarch64_ad.m4
11904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11905 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11906 %{
11907 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11908 ins_cost(INSN_COST);
11909 format %{ "add $dst, $src1, $src2, sxth" %}
11910
11911 ins_encode %{
11912 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11913 as_Register($src2$$reg), ext::sxth);
11914 %}
11915 ins_pipe(ialu_reg_reg);
11916 %}
11917
11918 // This pattern is automatically generated from aarch64_ad.m4
11919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11920 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11921 %{
11922 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11923 ins_cost(INSN_COST);
11924 format %{ "add $dst, $src1, $src2, sxtb" %}
11925
11926 ins_encode %{
11927 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11928 as_Register($src2$$reg), ext::sxtb);
11929 %}
11930 ins_pipe(ialu_reg_reg);
11931 %}
11932
11933 // This pattern is automatically generated from aarch64_ad.m4
11934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11935 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11936 %{
11937 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11938 ins_cost(INSN_COST);
11939 format %{ "add $dst, $src1, $src2, uxtb" %}
11940
11941 ins_encode %{
11942 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11943 as_Register($src2$$reg), ext::uxtb);
11944 %}
11945 ins_pipe(ialu_reg_reg);
11946 %}
11947
11948 // This pattern is automatically generated from aarch64_ad.m4
11949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11950 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11951 %{
11952 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11953 ins_cost(INSN_COST);
11954 format %{ "add $dst, $src1, $src2, sxth" %}
11955
11956 ins_encode %{
11957 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11958 as_Register($src2$$reg), ext::sxth);
11959 %}
11960 ins_pipe(ialu_reg_reg);
11961 %}
11962
11963 // This pattern is automatically generated from aarch64_ad.m4
11964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11965 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11966 %{
11967 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11968 ins_cost(INSN_COST);
11969 format %{ "add $dst, $src1, $src2, sxtw" %}
11970
11971 ins_encode %{
11972 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11973 as_Register($src2$$reg), ext::sxtw);
11974 %}
11975 ins_pipe(ialu_reg_reg);
11976 %}
11977
11978 // This pattern is automatically generated from aarch64_ad.m4
11979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11980 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11981 %{
11982 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11983 ins_cost(INSN_COST);
11984 format %{ "add $dst, $src1, $src2, sxtb" %}
11985
11986 ins_encode %{
11987 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11988 as_Register($src2$$reg), ext::sxtb);
11989 %}
11990 ins_pipe(ialu_reg_reg);
11991 %}
11992
11993 // This pattern is automatically generated from aarch64_ad.m4
11994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11995 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11996 %{
11997 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11998 ins_cost(INSN_COST);
11999 format %{ "add $dst, $src1, $src2, uxtb" %}
12000
12001 ins_encode %{
12002 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12003 as_Register($src2$$reg), ext::uxtb);
12004 %}
12005 ins_pipe(ialu_reg_reg);
12006 %}
12007
12008 // This pattern is automatically generated from aarch64_ad.m4
12009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12010 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12011 %{
12012 match(Set dst (AddI src1 (AndI src2 mask)));
12013 ins_cost(INSN_COST);
12014 format %{ "addw $dst, $src1, $src2, uxtb" %}
12015
12016 ins_encode %{
12017 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12018 as_Register($src2$$reg), ext::uxtb);
12019 %}
12020 ins_pipe(ialu_reg_reg);
12021 %}
12022
12023 // This pattern is automatically generated from aarch64_ad.m4
12024 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12025 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12026 %{
12027 match(Set dst (AddI src1 (AndI src2 mask)));
12028 ins_cost(INSN_COST);
12029 format %{ "addw $dst, $src1, $src2, uxth" %}
12030
12031 ins_encode %{
12032 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12033 as_Register($src2$$reg), ext::uxth);
12034 %}
12035 ins_pipe(ialu_reg_reg);
12036 %}
12037
12038 // This pattern is automatically generated from aarch64_ad.m4
12039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12040 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12041 %{
12042 match(Set dst (AddL src1 (AndL src2 mask)));
12043 ins_cost(INSN_COST);
12044 format %{ "add $dst, $src1, $src2, uxtb" %}
12045
12046 ins_encode %{
12047 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12048 as_Register($src2$$reg), ext::uxtb);
12049 %}
12050 ins_pipe(ialu_reg_reg);
12051 %}
12052
12053 // This pattern is automatically generated from aarch64_ad.m4
12054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12055 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12056 %{
12057 match(Set dst (AddL src1 (AndL src2 mask)));
12058 ins_cost(INSN_COST);
12059 format %{ "add $dst, $src1, $src2, uxth" %}
12060
12061 ins_encode %{
12062 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12063 as_Register($src2$$reg), ext::uxth);
12064 %}
12065 ins_pipe(ialu_reg_reg);
12066 %}
12067
12068 // This pattern is automatically generated from aarch64_ad.m4
12069 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12070 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12071 %{
12072 match(Set dst (AddL src1 (AndL src2 mask)));
12073 ins_cost(INSN_COST);
12074 format %{ "add $dst, $src1, $src2, uxtw" %}
12075
12076 ins_encode %{
12077 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12078 as_Register($src2$$reg), ext::uxtw);
12079 %}
12080 ins_pipe(ialu_reg_reg);
12081 %}
12082
12083 // This pattern is automatically generated from aarch64_ad.m4
12084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12085 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12086 %{
12087 match(Set dst (SubI src1 (AndI src2 mask)));
12088 ins_cost(INSN_COST);
12089 format %{ "subw $dst, $src1, $src2, uxtb" %}
12090
12091 ins_encode %{
12092 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12093 as_Register($src2$$reg), ext::uxtb);
12094 %}
12095 ins_pipe(ialu_reg_reg);
12096 %}
12097
12098 // This pattern is automatically generated from aarch64_ad.m4
12099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12100 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12101 %{
12102 match(Set dst (SubI src1 (AndI src2 mask)));
12103 ins_cost(INSN_COST);
12104 format %{ "subw $dst, $src1, $src2, uxth" %}
12105
12106 ins_encode %{
12107 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12108 as_Register($src2$$reg), ext::uxth);
12109 %}
12110 ins_pipe(ialu_reg_reg);
12111 %}
12112
12113 // This pattern is automatically generated from aarch64_ad.m4
12114 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12115 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12116 %{
12117 match(Set dst (SubL src1 (AndL src2 mask)));
12118 ins_cost(INSN_COST);
12119 format %{ "sub $dst, $src1, $src2, uxtb" %}
12120
12121 ins_encode %{
12122 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12123 as_Register($src2$$reg), ext::uxtb);
12124 %}
12125 ins_pipe(ialu_reg_reg);
12126 %}
12127
12128 // This pattern is automatically generated from aarch64_ad.m4
12129 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12130 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12131 %{
12132 match(Set dst (SubL src1 (AndL src2 mask)));
12133 ins_cost(INSN_COST);
12134 format %{ "sub $dst, $src1, $src2, uxth" %}
12135
12136 ins_encode %{
12137 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12138 as_Register($src2$$reg), ext::uxth);
12139 %}
12140 ins_pipe(ialu_reg_reg);
12141 %}
12142
12143 // This pattern is automatically generated from aarch64_ad.m4
12144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12145 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12146 %{
12147 match(Set dst (SubL src1 (AndL src2 mask)));
12148 ins_cost(INSN_COST);
12149 format %{ "sub $dst, $src1, $src2, uxtw" %}
12150
12151 ins_encode %{
12152 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12153 as_Register($src2$$reg), ext::uxtw);
12154 %}
12155 ins_pipe(ialu_reg_reg);
12156 %}
12157
12158
12159 // This pattern is automatically generated from aarch64_ad.m4
12160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12161 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12162 %{
12163 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12164 ins_cost(1.9 * INSN_COST);
12165 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12166
12167 ins_encode %{
12168 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12169 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12170 %}
12171 ins_pipe(ialu_reg_reg_shift);
12172 %}
12173
12174 // This pattern is automatically generated from aarch64_ad.m4
12175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12176 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12177 %{
12178 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12179 ins_cost(1.9 * INSN_COST);
12180 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12181
12182 ins_encode %{
12183 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12184 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12185 %}
12186 ins_pipe(ialu_reg_reg_shift);
12187 %}
12188
12189 // This pattern is automatically generated from aarch64_ad.m4
12190 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12191 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12192 %{
12193 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12194 ins_cost(1.9 * INSN_COST);
12195 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12196
12197 ins_encode %{
12198 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12199 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12200 %}
12201 ins_pipe(ialu_reg_reg_shift);
12202 %}
12203
12204 // This pattern is automatically generated from aarch64_ad.m4
12205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12206 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12207 %{
12208 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12209 ins_cost(1.9 * INSN_COST);
12210 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12211
12212 ins_encode %{
12213 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12214 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12215 %}
12216 ins_pipe(ialu_reg_reg_shift);
12217 %}
12218
12219 // This pattern is automatically generated from aarch64_ad.m4
12220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12221 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12222 %{
12223 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12224 ins_cost(1.9 * INSN_COST);
12225 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12226
12227 ins_encode %{
12228 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12229 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12230 %}
12231 ins_pipe(ialu_reg_reg_shift);
12232 %}
12233
12234 // This pattern is automatically generated from aarch64_ad.m4
12235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12236 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12237 %{
12238 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12239 ins_cost(1.9 * INSN_COST);
12240 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12241
12242 ins_encode %{
12243 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12244 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12245 %}
12246 ins_pipe(ialu_reg_reg_shift);
12247 %}
12248
12249 // This pattern is automatically generated from aarch64_ad.m4
12250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12251 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12252 %{
12253 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12254 ins_cost(1.9 * INSN_COST);
12255 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12256
12257 ins_encode %{
12258 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12259 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12260 %}
12261 ins_pipe(ialu_reg_reg_shift);
12262 %}
12263
12264 // This pattern is automatically generated from aarch64_ad.m4
12265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12266 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12267 %{
12268 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12269 ins_cost(1.9 * INSN_COST);
12270 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12271
12272 ins_encode %{
12273 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12274 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12275 %}
12276 ins_pipe(ialu_reg_reg_shift);
12277 %}
12278
12279 // This pattern is automatically generated from aarch64_ad.m4
12280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12281 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12282 %{
12283 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12284 ins_cost(1.9 * INSN_COST);
12285 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12286
12287 ins_encode %{
12288 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12289 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12290 %}
12291 ins_pipe(ialu_reg_reg_shift);
12292 %}
12293
12294 // This pattern is automatically generated from aarch64_ad.m4
12295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12296 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12297 %{
12298 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12299 ins_cost(1.9 * INSN_COST);
12300 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12301
12302 ins_encode %{
12303 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12304 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12305 %}
12306 ins_pipe(ialu_reg_reg_shift);
12307 %}
12308
12309 // This pattern is automatically generated from aarch64_ad.m4
12310 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12311 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12312 %{
12313 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12314 ins_cost(1.9 * INSN_COST);
12315 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12316
12317 ins_encode %{
12318 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12319 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12320 %}
12321 ins_pipe(ialu_reg_reg_shift);
12322 %}
12323
12324 // This pattern is automatically generated from aarch64_ad.m4
12325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12326 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12327 %{
12328 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12329 ins_cost(1.9 * INSN_COST);
12330 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12331
12332 ins_encode %{
12333 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12334 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12335 %}
12336 ins_pipe(ialu_reg_reg_shift);
12337 %}
12338
12339 // This pattern is automatically generated from aarch64_ad.m4
12340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12341 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12342 %{
12343 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12344 ins_cost(1.9 * INSN_COST);
12345 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12346
12347 ins_encode %{
12348 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12349 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12350 %}
12351 ins_pipe(ialu_reg_reg_shift);
12352 %}
12353
12354 // This pattern is automatically generated from aarch64_ad.m4
12355 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12356 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12357 %{
12358 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12359 ins_cost(1.9 * INSN_COST);
12360 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12361
12362 ins_encode %{
12363 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12364 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12365 %}
12366 ins_pipe(ialu_reg_reg_shift);
12367 %}
12368
12369 // This pattern is automatically generated from aarch64_ad.m4
12370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12371 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12372 %{
12373 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12374 ins_cost(1.9 * INSN_COST);
12375 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12376
12377 ins_encode %{
12378 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12379 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12380 %}
12381 ins_pipe(ialu_reg_reg_shift);
12382 %}
12383
12384 // This pattern is automatically generated from aarch64_ad.m4
12385 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12386 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12387 %{
12388 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12389 ins_cost(1.9 * INSN_COST);
12390 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12391
12392 ins_encode %{
12393 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12394 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12395 %}
12396 ins_pipe(ialu_reg_reg_shift);
12397 %}
12398
12399 // This pattern is automatically generated from aarch64_ad.m4
12400 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12401 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12402 %{
12403 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12404 ins_cost(1.9 * INSN_COST);
12405 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12406
12407 ins_encode %{
12408 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12409 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12410 %}
12411 ins_pipe(ialu_reg_reg_shift);
12412 %}
12413
12414 // This pattern is automatically generated from aarch64_ad.m4
12415 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12416 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12417 %{
12418 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12419 ins_cost(1.9 * INSN_COST);
12420 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12421
12422 ins_encode %{
12423 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12424 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12425 %}
12426 ins_pipe(ialu_reg_reg_shift);
12427 %}
12428
12429 // This pattern is automatically generated from aarch64_ad.m4
12430 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12431 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12432 %{
12433 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12434 ins_cost(1.9 * INSN_COST);
12435 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12436
12437 ins_encode %{
12438 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12439 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12440 %}
12441 ins_pipe(ialu_reg_reg_shift);
12442 %}
12443
12444 // This pattern is automatically generated from aarch64_ad.m4
12445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12446 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12447 %{
12448 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12449 ins_cost(1.9 * INSN_COST);
12450 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12451
12452 ins_encode %{
12453 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12454 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12455 %}
12456 ins_pipe(ialu_reg_reg_shift);
12457 %}
12458
12459 // This pattern is automatically generated from aarch64_ad.m4
12460 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12461 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12462 %{
12463 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12464 ins_cost(1.9 * INSN_COST);
12465 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12466
12467 ins_encode %{
12468 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12469 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12470 %}
12471 ins_pipe(ialu_reg_reg_shift);
12472 %}
12473
12474 // This pattern is automatically generated from aarch64_ad.m4
12475 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12476 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12477 %{
12478 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12479 ins_cost(1.9 * INSN_COST);
12480 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12481
12482 ins_encode %{
12483 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12484 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12485 %}
12486 ins_pipe(ialu_reg_reg_shift);
12487 %}
12488
12489 // This pattern is automatically generated from aarch64_ad.m4
12490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12491 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12492 %{
12493 effect(DEF dst, USE src1, USE src2, USE cr);
12494 ins_cost(INSN_COST * 2);
12495 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12496
12497 ins_encode %{
12498 __ cselw($dst$$Register,
12499 $src1$$Register,
12500 $src2$$Register,
12501 Assembler::LT);
12502 %}
12503 ins_pipe(icond_reg_reg);
12504 %}
12505
12506 // This pattern is automatically generated from aarch64_ad.m4
12507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12508 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12509 %{
12510 effect(DEF dst, USE src1, USE src2, USE cr);
12511 ins_cost(INSN_COST * 2);
12512 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12513
12514 ins_encode %{
12515 __ cselw($dst$$Register,
12516 $src1$$Register,
12517 $src2$$Register,
12518 Assembler::GT);
12519 %}
12520 ins_pipe(icond_reg_reg);
12521 %}
12522
12523 // This pattern is automatically generated from aarch64_ad.m4
12524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12525 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12526 %{
12527 effect(DEF dst, USE src1, USE cr);
12528 ins_cost(INSN_COST * 2);
12529 format %{ "cselw $dst, $src1, zr lt\t" %}
12530
12531 ins_encode %{
12532 __ cselw($dst$$Register,
12533 $src1$$Register,
12534 zr,
12535 Assembler::LT);
12536 %}
12537 ins_pipe(icond_reg);
12538 %}
12539
12540 // This pattern is automatically generated from aarch64_ad.m4
12541 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12542 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12543 %{
12544 effect(DEF dst, USE src1, USE cr);
12545 ins_cost(INSN_COST * 2);
12546 format %{ "cselw $dst, $src1, zr gt\t" %}
12547
12548 ins_encode %{
12549 __ cselw($dst$$Register,
12550 $src1$$Register,
12551 zr,
12552 Assembler::GT);
12553 %}
12554 ins_pipe(icond_reg);
12555 %}
12556
12557 // This pattern is automatically generated from aarch64_ad.m4
12558 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12559 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12560 %{
12561 effect(DEF dst, USE src1, USE cr);
12562 ins_cost(INSN_COST * 2);
12563 format %{ "csincw $dst, $src1, zr le\t" %}
12564
12565 ins_encode %{
12566 __ csincw($dst$$Register,
12567 $src1$$Register,
12568 zr,
12569 Assembler::LE);
12570 %}
12571 ins_pipe(icond_reg);
12572 %}
12573
12574 // This pattern is automatically generated from aarch64_ad.m4
12575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12576 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12577 %{
12578 effect(DEF dst, USE src1, USE cr);
12579 ins_cost(INSN_COST * 2);
12580 format %{ "csincw $dst, $src1, zr gt\t" %}
12581
12582 ins_encode %{
12583 __ csincw($dst$$Register,
12584 $src1$$Register,
12585 zr,
12586 Assembler::GT);
12587 %}
12588 ins_pipe(icond_reg);
12589 %}
12590
12591 // This pattern is automatically generated from aarch64_ad.m4
12592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12593 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12594 %{
12595 effect(DEF dst, USE src1, USE cr);
12596 ins_cost(INSN_COST * 2);
12597 format %{ "csinvw $dst, $src1, zr lt\t" %}
12598
12599 ins_encode %{
12600 __ csinvw($dst$$Register,
12601 $src1$$Register,
12602 zr,
12603 Assembler::LT);
12604 %}
12605 ins_pipe(icond_reg);
12606 %}
12607
12608 // This pattern is automatically generated from aarch64_ad.m4
12609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12610 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12611 %{
12612 effect(DEF dst, USE src1, USE cr);
12613 ins_cost(INSN_COST * 2);
12614 format %{ "csinvw $dst, $src1, zr ge\t" %}
12615
12616 ins_encode %{
12617 __ csinvw($dst$$Register,
12618 $src1$$Register,
12619 zr,
12620 Assembler::GE);
12621 %}
12622 ins_pipe(icond_reg);
12623 %}
12624
12625 // This pattern is automatically generated from aarch64_ad.m4
12626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12627 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12628 %{
12629 match(Set dst (MinI src imm));
12630 ins_cost(INSN_COST * 3);
12631 expand %{
12632 rFlagsReg cr;
12633 compI_reg_imm0(cr, src);
12634 cmovI_reg_imm0_lt(dst, src, cr);
12635 %}
12636 %}
12637
12638 // This pattern is automatically generated from aarch64_ad.m4
12639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12640 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12641 %{
12642 match(Set dst (MinI imm src));
12643 ins_cost(INSN_COST * 3);
12644 expand %{
12645 rFlagsReg cr;
12646 compI_reg_imm0(cr, src);
12647 cmovI_reg_imm0_lt(dst, src, cr);
12648 %}
12649 %}
12650
12651 // This pattern is automatically generated from aarch64_ad.m4
12652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12653 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12654 %{
12655 match(Set dst (MinI src imm));
12656 ins_cost(INSN_COST * 3);
12657 expand %{
12658 rFlagsReg cr;
12659 compI_reg_imm0(cr, src);
12660 cmovI_reg_imm1_le(dst, src, cr);
12661 %}
12662 %}
12663
12664 // This pattern is automatically generated from aarch64_ad.m4
12665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12666 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12667 %{
12668 match(Set dst (MinI imm src));
12669 ins_cost(INSN_COST * 3);
12670 expand %{
12671 rFlagsReg cr;
12672 compI_reg_imm0(cr, src);
12673 cmovI_reg_imm1_le(dst, src, cr);
12674 %}
12675 %}
12676
12677 // This pattern is automatically generated from aarch64_ad.m4
12678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12679 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12680 %{
12681 match(Set dst (MinI src imm));
12682 ins_cost(INSN_COST * 3);
12683 expand %{
12684 rFlagsReg cr;
12685 compI_reg_imm0(cr, src);
12686 cmovI_reg_immM1_lt(dst, src, cr);
12687 %}
12688 %}
12689
12690 // This pattern is automatically generated from aarch64_ad.m4
12691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12692 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12693 %{
12694 match(Set dst (MinI imm src));
12695 ins_cost(INSN_COST * 3);
12696 expand %{
12697 rFlagsReg cr;
12698 compI_reg_imm0(cr, src);
12699 cmovI_reg_immM1_lt(dst, src, cr);
12700 %}
12701 %}
12702
12703 // This pattern is automatically generated from aarch64_ad.m4
12704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12705 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12706 %{
12707 match(Set dst (MaxI src imm));
12708 ins_cost(INSN_COST * 3);
12709 expand %{
12710 rFlagsReg cr;
12711 compI_reg_imm0(cr, src);
12712 cmovI_reg_imm0_gt(dst, src, cr);
12713 %}
12714 %}
12715
12716 // This pattern is automatically generated from aarch64_ad.m4
12717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12718 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12719 %{
12720 match(Set dst (MaxI imm src));
12721 ins_cost(INSN_COST * 3);
12722 expand %{
12723 rFlagsReg cr;
12724 compI_reg_imm0(cr, src);
12725 cmovI_reg_imm0_gt(dst, src, cr);
12726 %}
12727 %}
12728
12729 // This pattern is automatically generated from aarch64_ad.m4
12730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12731 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12732 %{
12733 match(Set dst (MaxI src imm));
12734 ins_cost(INSN_COST * 3);
12735 expand %{
12736 rFlagsReg cr;
12737 compI_reg_imm0(cr, src);
12738 cmovI_reg_imm1_gt(dst, src, cr);
12739 %}
12740 %}
12741
12742 // This pattern is automatically generated from aarch64_ad.m4
12743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12744 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12745 %{
12746 match(Set dst (MaxI imm src));
12747 ins_cost(INSN_COST * 3);
12748 expand %{
12749 rFlagsReg cr;
12750 compI_reg_imm0(cr, src);
12751 cmovI_reg_imm1_gt(dst, src, cr);
12752 %}
12753 %}
12754
12755 // This pattern is automatically generated from aarch64_ad.m4
12756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12757 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12758 %{
12759 match(Set dst (MaxI src imm));
12760 ins_cost(INSN_COST * 3);
12761 expand %{
12762 rFlagsReg cr;
12763 compI_reg_imm0(cr, src);
12764 cmovI_reg_immM1_ge(dst, src, cr);
12765 %}
12766 %}
12767
12768 // This pattern is automatically generated from aarch64_ad.m4
12769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12770 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12771 %{
12772 match(Set dst (MaxI imm src));
12773 ins_cost(INSN_COST * 3);
12774 expand %{
12775 rFlagsReg cr;
12776 compI_reg_imm0(cr, src);
12777 cmovI_reg_immM1_ge(dst, src, cr);
12778 %}
12779 %}
12780
12781 // This pattern is automatically generated from aarch64_ad.m4
12782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12783 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12784 %{
12785 match(Set dst (ReverseI src));
12786 ins_cost(INSN_COST);
12787 format %{ "rbitw $dst, $src" %}
12788 ins_encode %{
12789 __ rbitw($dst$$Register, $src$$Register);
12790 %}
12791 ins_pipe(ialu_reg);
12792 %}
12793
12794 // This pattern is automatically generated from aarch64_ad.m4
12795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12796 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12797 %{
12798 match(Set dst (ReverseL src));
12799 ins_cost(INSN_COST);
12800 format %{ "rbit $dst, $src" %}
12801 ins_encode %{
12802 __ rbit($dst$$Register, $src$$Register);
12803 %}
12804 ins_pipe(ialu_reg);
12805 %}
12806
12807
12808 // END This section of the file is automatically generated. Do not edit --------------
12809
12810
12811 // ============================================================================
12812 // Floating Point Arithmetic Instructions
12813
12814 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12815 match(Set dst (AddHF src1 src2));
12816 format %{ "faddh $dst, $src1, $src2" %}
12817 ins_encode %{
12818 __ faddh($dst$$FloatRegister,
12819 $src1$$FloatRegister,
12820 $src2$$FloatRegister);
12821 %}
12822 ins_pipe(fp_dop_reg_reg_s);
12823 %}
12824
12825 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12826 match(Set dst (AddF src1 src2));
12827
12828 ins_cost(INSN_COST * 5);
12829 format %{ "fadds $dst, $src1, $src2" %}
12830
12831 ins_encode %{
12832 __ fadds(as_FloatRegister($dst$$reg),
12833 as_FloatRegister($src1$$reg),
12834 as_FloatRegister($src2$$reg));
12835 %}
12836
12837 ins_pipe(fp_dop_reg_reg_s);
12838 %}
12839
12840 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12841 match(Set dst (AddD src1 src2));
12842
12843 ins_cost(INSN_COST * 5);
12844 format %{ "faddd $dst, $src1, $src2" %}
12845
12846 ins_encode %{
12847 __ faddd(as_FloatRegister($dst$$reg),
12848 as_FloatRegister($src1$$reg),
12849 as_FloatRegister($src2$$reg));
12850 %}
12851
12852 ins_pipe(fp_dop_reg_reg_d);
12853 %}
12854
12855 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12856 match(Set dst (SubHF src1 src2));
12857 format %{ "fsubh $dst, $src1, $src2" %}
12858 ins_encode %{
12859 __ fsubh($dst$$FloatRegister,
12860 $src1$$FloatRegister,
12861 $src2$$FloatRegister);
12862 %}
12863 ins_pipe(fp_dop_reg_reg_s);
12864 %}
12865
12866 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12867 match(Set dst (SubF src1 src2));
12868
12869 ins_cost(INSN_COST * 5);
12870 format %{ "fsubs $dst, $src1, $src2" %}
12871
12872 ins_encode %{
12873 __ fsubs(as_FloatRegister($dst$$reg),
12874 as_FloatRegister($src1$$reg),
12875 as_FloatRegister($src2$$reg));
12876 %}
12877
12878 ins_pipe(fp_dop_reg_reg_s);
12879 %}
12880
12881 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12882 match(Set dst (SubD src1 src2));
12883
12884 ins_cost(INSN_COST * 5);
12885 format %{ "fsubd $dst, $src1, $src2" %}
12886
12887 ins_encode %{
12888 __ fsubd(as_FloatRegister($dst$$reg),
12889 as_FloatRegister($src1$$reg),
12890 as_FloatRegister($src2$$reg));
12891 %}
12892
12893 ins_pipe(fp_dop_reg_reg_d);
12894 %}
12895
12896 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12897 match(Set dst (MulHF src1 src2));
12898 format %{ "fmulh $dst, $src1, $src2" %}
12899 ins_encode %{
12900 __ fmulh($dst$$FloatRegister,
12901 $src1$$FloatRegister,
12902 $src2$$FloatRegister);
12903 %}
12904 ins_pipe(fp_dop_reg_reg_s);
12905 %}
12906
12907 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12908 match(Set dst (MulF src1 src2));
12909
12910 ins_cost(INSN_COST * 6);
12911 format %{ "fmuls $dst, $src1, $src2" %}
12912
12913 ins_encode %{
12914 __ fmuls(as_FloatRegister($dst$$reg),
12915 as_FloatRegister($src1$$reg),
12916 as_FloatRegister($src2$$reg));
12917 %}
12918
12919 ins_pipe(fp_dop_reg_reg_s);
12920 %}
12921
12922 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12923 match(Set dst (MulD src1 src2));
12924
12925 ins_cost(INSN_COST * 6);
12926 format %{ "fmuld $dst, $src1, $src2" %}
12927
12928 ins_encode %{
12929 __ fmuld(as_FloatRegister($dst$$reg),
12930 as_FloatRegister($src1$$reg),
12931 as_FloatRegister($src2$$reg));
12932 %}
12933
12934 ins_pipe(fp_dop_reg_reg_d);
12935 %}
12936
12937 // src1 * src2 + src3 (half-precision float)
12938 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12939 match(Set dst (FmaHF src3 (Binary src1 src2)));
12940 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12941 ins_encode %{
12942 assert(UseFMA, "Needs FMA instructions support.");
12943 __ fmaddh($dst$$FloatRegister,
12944 $src1$$FloatRegister,
12945 $src2$$FloatRegister,
12946 $src3$$FloatRegister);
12947 %}
12948 ins_pipe(pipe_class_default);
12949 %}
12950
12951 // src1 * src2 + src3
12952 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12953 match(Set dst (FmaF src3 (Binary src1 src2)));
12954
12955 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12956
12957 ins_encode %{
12958 assert(UseFMA, "Needs FMA instructions support.");
12959 __ fmadds(as_FloatRegister($dst$$reg),
12960 as_FloatRegister($src1$$reg),
12961 as_FloatRegister($src2$$reg),
12962 as_FloatRegister($src3$$reg));
12963 %}
12964
12965 ins_pipe(pipe_class_default);
12966 %}
12967
12968 // src1 * src2 + src3
12969 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12970 match(Set dst (FmaD src3 (Binary src1 src2)));
12971
12972 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12973
12974 ins_encode %{
12975 assert(UseFMA, "Needs FMA instructions support.");
12976 __ fmaddd(as_FloatRegister($dst$$reg),
12977 as_FloatRegister($src1$$reg),
12978 as_FloatRegister($src2$$reg),
12979 as_FloatRegister($src3$$reg));
12980 %}
12981
12982 ins_pipe(pipe_class_default);
12983 %}
12984
12985 // src1 * (-src2) + src3
12986 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12987 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12988 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12989
12990 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12991
12992 ins_encode %{
12993 assert(UseFMA, "Needs FMA instructions support.");
12994 __ fmsubs(as_FloatRegister($dst$$reg),
12995 as_FloatRegister($src1$$reg),
12996 as_FloatRegister($src2$$reg),
12997 as_FloatRegister($src3$$reg));
12998 %}
12999
13000 ins_pipe(pipe_class_default);
13001 %}
13002
13003 // src1 * (-src2) + src3
13004 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13005 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13006 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13007
13008 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13009
13010 ins_encode %{
13011 assert(UseFMA, "Needs FMA instructions support.");
13012 __ fmsubd(as_FloatRegister($dst$$reg),
13013 as_FloatRegister($src1$$reg),
13014 as_FloatRegister($src2$$reg),
13015 as_FloatRegister($src3$$reg));
13016 %}
13017
13018 ins_pipe(pipe_class_default);
13019 %}
13020
13021 // src1 * (-src2) - src3
13022 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13023 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13024 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13025
13026 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13027
13028 ins_encode %{
13029 assert(UseFMA, "Needs FMA instructions support.");
13030 __ fnmadds(as_FloatRegister($dst$$reg),
13031 as_FloatRegister($src1$$reg),
13032 as_FloatRegister($src2$$reg),
13033 as_FloatRegister($src3$$reg));
13034 %}
13035
13036 ins_pipe(pipe_class_default);
13037 %}
13038
13039 // src1 * (-src2) - src3
13040 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13041 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13042 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13043
13044 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13045
13046 ins_encode %{
13047 assert(UseFMA, "Needs FMA instructions support.");
13048 __ fnmaddd(as_FloatRegister($dst$$reg),
13049 as_FloatRegister($src1$$reg),
13050 as_FloatRegister($src2$$reg),
13051 as_FloatRegister($src3$$reg));
13052 %}
13053
13054 ins_pipe(pipe_class_default);
13055 %}
13056
13057 // src1 * src2 - src3
13058 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13059 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13060
13061 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13062
13063 ins_encode %{
13064 assert(UseFMA, "Needs FMA instructions support.");
13065 __ fnmsubs(as_FloatRegister($dst$$reg),
13066 as_FloatRegister($src1$$reg),
13067 as_FloatRegister($src2$$reg),
13068 as_FloatRegister($src3$$reg));
13069 %}
13070
13071 ins_pipe(pipe_class_default);
13072 %}
13073
13074 // src1 * src2 - src3
13075 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13076 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13077
13078 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13079
13080 ins_encode %{
13081 assert(UseFMA, "Needs FMA instructions support.");
13082 // n.b. insn name should be fnmsubd
13083 __ fnmsub(as_FloatRegister($dst$$reg),
13084 as_FloatRegister($src1$$reg),
13085 as_FloatRegister($src2$$reg),
13086 as_FloatRegister($src3$$reg));
13087 %}
13088
13089 ins_pipe(pipe_class_default);
13090 %}
13091
13092 // Math.max(HH)H (half-precision float)
13093 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13094 match(Set dst (MaxHF src1 src2));
13095 format %{ "fmaxh $dst, $src1, $src2" %}
13096 ins_encode %{
13097 __ fmaxh($dst$$FloatRegister,
13098 $src1$$FloatRegister,
13099 $src2$$FloatRegister);
13100 %}
13101 ins_pipe(fp_dop_reg_reg_s);
13102 %}
13103
13104 // Math.min(HH)H (half-precision float)
13105 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13106 match(Set dst (MinHF src1 src2));
13107 format %{ "fminh $dst, $src1, $src2" %}
13108 ins_encode %{
13109 __ fminh($dst$$FloatRegister,
13110 $src1$$FloatRegister,
13111 $src2$$FloatRegister);
13112 %}
13113 ins_pipe(fp_dop_reg_reg_s);
13114 %}
13115
13116 // Math.max(FF)F
13117 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13118 match(Set dst (MaxF src1 src2));
13119
13120 format %{ "fmaxs $dst, $src1, $src2" %}
13121 ins_encode %{
13122 __ fmaxs(as_FloatRegister($dst$$reg),
13123 as_FloatRegister($src1$$reg),
13124 as_FloatRegister($src2$$reg));
13125 %}
13126
13127 ins_pipe(fp_dop_reg_reg_s);
13128 %}
13129
13130 // Math.min(FF)F
13131 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13132 match(Set dst (MinF src1 src2));
13133
13134 format %{ "fmins $dst, $src1, $src2" %}
13135 ins_encode %{
13136 __ fmins(as_FloatRegister($dst$$reg),
13137 as_FloatRegister($src1$$reg),
13138 as_FloatRegister($src2$$reg));
13139 %}
13140
13141 ins_pipe(fp_dop_reg_reg_s);
13142 %}
13143
13144 // Math.max(DD)D
13145 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13146 match(Set dst (MaxD src1 src2));
13147
13148 format %{ "fmaxd $dst, $src1, $src2" %}
13149 ins_encode %{
13150 __ fmaxd(as_FloatRegister($dst$$reg),
13151 as_FloatRegister($src1$$reg),
13152 as_FloatRegister($src2$$reg));
13153 %}
13154
13155 ins_pipe(fp_dop_reg_reg_d);
13156 %}
13157
13158 // Math.min(DD)D
13159 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13160 match(Set dst (MinD src1 src2));
13161
13162 format %{ "fmind $dst, $src1, $src2" %}
13163 ins_encode %{
13164 __ fmind(as_FloatRegister($dst$$reg),
13165 as_FloatRegister($src1$$reg),
13166 as_FloatRegister($src2$$reg));
13167 %}
13168
13169 ins_pipe(fp_dop_reg_reg_d);
13170 %}
13171
13172 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13173 match(Set dst (DivHF src1 src2));
13174 format %{ "fdivh $dst, $src1, $src2" %}
13175 ins_encode %{
13176 __ fdivh($dst$$FloatRegister,
13177 $src1$$FloatRegister,
13178 $src2$$FloatRegister);
13179 %}
13180 ins_pipe(fp_div_s);
13181 %}
13182
13183 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13184 match(Set dst (DivF src1 src2));
13185
13186 ins_cost(INSN_COST * 18);
13187 format %{ "fdivs $dst, $src1, $src2" %}
13188
13189 ins_encode %{
13190 __ fdivs(as_FloatRegister($dst$$reg),
13191 as_FloatRegister($src1$$reg),
13192 as_FloatRegister($src2$$reg));
13193 %}
13194
13195 ins_pipe(fp_div_s);
13196 %}
13197
13198 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13199 match(Set dst (DivD src1 src2));
13200
13201 ins_cost(INSN_COST * 32);
13202 format %{ "fdivd $dst, $src1, $src2" %}
13203
13204 ins_encode %{
13205 __ fdivd(as_FloatRegister($dst$$reg),
13206 as_FloatRegister($src1$$reg),
13207 as_FloatRegister($src2$$reg));
13208 %}
13209
13210 ins_pipe(fp_div_d);
13211 %}
13212
13213 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13214 match(Set dst (NegF src));
13215
13216 ins_cost(INSN_COST * 3);
13217 format %{ "fneg $dst, $src" %}
13218
13219 ins_encode %{
13220 __ fnegs(as_FloatRegister($dst$$reg),
13221 as_FloatRegister($src$$reg));
13222 %}
13223
13224 ins_pipe(fp_uop_s);
13225 %}
13226
13227 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13228 match(Set dst (NegD src));
13229
13230 ins_cost(INSN_COST * 3);
13231 format %{ "fnegd $dst, $src" %}
13232
13233 ins_encode %{
13234 __ fnegd(as_FloatRegister($dst$$reg),
13235 as_FloatRegister($src$$reg));
13236 %}
13237
13238 ins_pipe(fp_uop_d);
13239 %}
13240
13241 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13242 %{
13243 match(Set dst (AbsI src));
13244
13245 effect(KILL cr);
13246 ins_cost(INSN_COST * 2);
13247 format %{ "cmpw $src, zr\n\t"
13248 "cnegw $dst, $src, Assembler::LT\t# int abs"
13249 %}
13250
13251 ins_encode %{
13252 __ cmpw(as_Register($src$$reg), zr);
13253 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13254 %}
13255 ins_pipe(pipe_class_default);
13256 %}
13257
13258 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13259 %{
13260 match(Set dst (AbsL src));
13261
13262 effect(KILL cr);
13263 ins_cost(INSN_COST * 2);
13264 format %{ "cmp $src, zr\n\t"
13265 "cneg $dst, $src, Assembler::LT\t# long abs"
13266 %}
13267
13268 ins_encode %{
13269 __ cmp(as_Register($src$$reg), zr);
13270 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13271 %}
13272 ins_pipe(pipe_class_default);
13273 %}
13274
13275 instruct absF_reg(vRegF dst, vRegF src) %{
13276 match(Set dst (AbsF src));
13277
13278 ins_cost(INSN_COST * 3);
13279 format %{ "fabss $dst, $src" %}
13280 ins_encode %{
13281 __ fabss(as_FloatRegister($dst$$reg),
13282 as_FloatRegister($src$$reg));
13283 %}
13284
13285 ins_pipe(fp_uop_s);
13286 %}
13287
13288 instruct absD_reg(vRegD dst, vRegD src) %{
13289 match(Set dst (AbsD src));
13290
13291 ins_cost(INSN_COST * 3);
13292 format %{ "fabsd $dst, $src" %}
13293 ins_encode %{
13294 __ fabsd(as_FloatRegister($dst$$reg),
13295 as_FloatRegister($src$$reg));
13296 %}
13297
13298 ins_pipe(fp_uop_d);
13299 %}
13300
13301 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13302 match(Set dst (AbsF (SubF src1 src2)));
13303
13304 ins_cost(INSN_COST * 3);
13305 format %{ "fabds $dst, $src1, $src2" %}
13306 ins_encode %{
13307 __ fabds(as_FloatRegister($dst$$reg),
13308 as_FloatRegister($src1$$reg),
13309 as_FloatRegister($src2$$reg));
13310 %}
13311
13312 ins_pipe(fp_uop_s);
13313 %}
13314
13315 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13316 match(Set dst (AbsD (SubD src1 src2)));
13317
13318 ins_cost(INSN_COST * 3);
13319 format %{ "fabdd $dst, $src1, $src2" %}
13320 ins_encode %{
13321 __ fabdd(as_FloatRegister($dst$$reg),
13322 as_FloatRegister($src1$$reg),
13323 as_FloatRegister($src2$$reg));
13324 %}
13325
13326 ins_pipe(fp_uop_d);
13327 %}
13328
13329 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13330 match(Set dst (SqrtD src));
13331
13332 ins_cost(INSN_COST * 50);
13333 format %{ "fsqrtd $dst, $src" %}
13334 ins_encode %{
13335 __ fsqrtd(as_FloatRegister($dst$$reg),
13336 as_FloatRegister($src$$reg));
13337 %}
13338
13339 ins_pipe(fp_div_s);
13340 %}
13341
13342 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13343 match(Set dst (SqrtF src));
13344
13345 ins_cost(INSN_COST * 50);
13346 format %{ "fsqrts $dst, $src" %}
13347 ins_encode %{
13348 __ fsqrts(as_FloatRegister($dst$$reg),
13349 as_FloatRegister($src$$reg));
13350 %}
13351
13352 ins_pipe(fp_div_d);
13353 %}
13354
13355 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13356 match(Set dst (SqrtHF src));
13357 format %{ "fsqrth $dst, $src" %}
13358 ins_encode %{
13359 __ fsqrth($dst$$FloatRegister,
13360 $src$$FloatRegister);
13361 %}
13362 ins_pipe(fp_div_s);
13363 %}
13364
13365 // Math.rint, floor, ceil
13366 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13367 match(Set dst (RoundDoubleMode src rmode));
13368 format %{ "frint $dst, $src, $rmode" %}
13369 ins_encode %{
13370 switch ($rmode$$constant) {
13371 case RoundDoubleModeNode::rmode_rint:
13372 __ frintnd(as_FloatRegister($dst$$reg),
13373 as_FloatRegister($src$$reg));
13374 break;
13375 case RoundDoubleModeNode::rmode_floor:
13376 __ frintmd(as_FloatRegister($dst$$reg),
13377 as_FloatRegister($src$$reg));
13378 break;
13379 case RoundDoubleModeNode::rmode_ceil:
13380 __ frintpd(as_FloatRegister($dst$$reg),
13381 as_FloatRegister($src$$reg));
13382 break;
13383 }
13384 %}
13385 ins_pipe(fp_uop_d);
13386 %}
13387
13388 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13389 match(Set dst (CopySignD src1 (Binary src2 zero)));
13390 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13391 format %{ "CopySignD $dst $src1 $src2" %}
13392 ins_encode %{
13393 FloatRegister dst = as_FloatRegister($dst$$reg),
13394 src1 = as_FloatRegister($src1$$reg),
13395 src2 = as_FloatRegister($src2$$reg),
13396 zero = as_FloatRegister($zero$$reg);
13397 __ fnegd(dst, zero);
13398 __ bsl(dst, __ T8B, src2, src1);
13399 %}
13400 ins_pipe(fp_uop_d);
13401 %}
13402
13403 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13404 match(Set dst (CopySignF src1 src2));
13405 effect(TEMP_DEF dst, USE src1, USE src2);
13406 format %{ "CopySignF $dst $src1 $src2" %}
13407 ins_encode %{
13408 FloatRegister dst = as_FloatRegister($dst$$reg),
13409 src1 = as_FloatRegister($src1$$reg),
13410 src2 = as_FloatRegister($src2$$reg);
13411 __ movi(dst, __ T2S, 0x80, 24);
13412 __ bsl(dst, __ T8B, src2, src1);
13413 %}
13414 ins_pipe(fp_uop_d);
13415 %}
13416
13417 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13418 match(Set dst (SignumD src (Binary zero one)));
13419 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13420 format %{ "signumD $dst, $src" %}
13421 ins_encode %{
13422 FloatRegister src = as_FloatRegister($src$$reg),
13423 dst = as_FloatRegister($dst$$reg),
13424 zero = as_FloatRegister($zero$$reg),
13425 one = as_FloatRegister($one$$reg);
13426 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13427 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13428 // Bit selection instruction gets bit from "one" for each enabled bit in
13429 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13430 // NaN the whole "src" will be copied because "dst" is zero. For all other
13431 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13432 // from "src", and all other bits are copied from 1.0.
13433 __ bsl(dst, __ T8B, one, src);
13434 %}
13435 ins_pipe(fp_uop_d);
13436 %}
13437
13438 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13439 match(Set dst (SignumF src (Binary zero one)));
13440 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13441 format %{ "signumF $dst, $src" %}
13442 ins_encode %{
13443 FloatRegister src = as_FloatRegister($src$$reg),
13444 dst = as_FloatRegister($dst$$reg),
13445 zero = as_FloatRegister($zero$$reg),
13446 one = as_FloatRegister($one$$reg);
13447 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13448 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13449 // Bit selection instruction gets bit from "one" for each enabled bit in
13450 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13451 // NaN the whole "src" will be copied because "dst" is zero. For all other
13452 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13453 // from "src", and all other bits are copied from 1.0.
13454 __ bsl(dst, __ T8B, one, src);
13455 %}
13456 ins_pipe(fp_uop_d);
13457 %}
13458
13459 instruct onspinwait() %{
13460 match(OnSpinWait);
13461 ins_cost(INSN_COST);
13462
13463 format %{ "onspinwait" %}
13464
13465 ins_encode %{
13466 __ spin_wait();
13467 %}
13468 ins_pipe(pipe_class_empty);
13469 %}
13470
13471 // ============================================================================
13472 // Logical Instructions
13473
13474 // Integer Logical Instructions
13475
13476 // And Instructions
13477
13478
13479 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13480 match(Set dst (AndI src1 src2));
13481
13482 format %{ "andw $dst, $src1, $src2\t# int" %}
13483
13484 ins_cost(INSN_COST);
13485 ins_encode %{
13486 __ andw(as_Register($dst$$reg),
13487 as_Register($src1$$reg),
13488 as_Register($src2$$reg));
13489 %}
13490
13491 ins_pipe(ialu_reg_reg);
13492 %}
13493
13494 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13495 match(Set dst (AndI src1 src2));
13496
13497 format %{ "andsw $dst, $src1, $src2\t# int" %}
13498
13499 ins_cost(INSN_COST);
13500 ins_encode %{
13501 __ andw(as_Register($dst$$reg),
13502 as_Register($src1$$reg),
13503 (uint64_t)($src2$$constant));
13504 %}
13505
13506 ins_pipe(ialu_reg_imm);
13507 %}
13508
13509 // Or Instructions
13510
13511 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13512 match(Set dst (OrI src1 src2));
13513
13514 format %{ "orrw $dst, $src1, $src2\t# int" %}
13515
13516 ins_cost(INSN_COST);
13517 ins_encode %{
13518 __ orrw(as_Register($dst$$reg),
13519 as_Register($src1$$reg),
13520 as_Register($src2$$reg));
13521 %}
13522
13523 ins_pipe(ialu_reg_reg);
13524 %}
13525
13526 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13527 match(Set dst (OrI src1 src2));
13528
13529 format %{ "orrw $dst, $src1, $src2\t# int" %}
13530
13531 ins_cost(INSN_COST);
13532 ins_encode %{
13533 __ orrw(as_Register($dst$$reg),
13534 as_Register($src1$$reg),
13535 (uint64_t)($src2$$constant));
13536 %}
13537
13538 ins_pipe(ialu_reg_imm);
13539 %}
13540
13541 // Xor Instructions
13542
13543 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13544 match(Set dst (XorI src1 src2));
13545
13546 format %{ "eorw $dst, $src1, $src2\t# int" %}
13547
13548 ins_cost(INSN_COST);
13549 ins_encode %{
13550 __ eorw(as_Register($dst$$reg),
13551 as_Register($src1$$reg),
13552 as_Register($src2$$reg));
13553 %}
13554
13555 ins_pipe(ialu_reg_reg);
13556 %}
13557
13558 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13559 match(Set dst (XorI src1 src2));
13560
13561 format %{ "eorw $dst, $src1, $src2\t# int" %}
13562
13563 ins_cost(INSN_COST);
13564 ins_encode %{
13565 __ eorw(as_Register($dst$$reg),
13566 as_Register($src1$$reg),
13567 (uint64_t)($src2$$constant));
13568 %}
13569
13570 ins_pipe(ialu_reg_imm);
13571 %}
13572
13573 // Long Logical Instructions
13574 // TODO
13575
13576 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13577 match(Set dst (AndL src1 src2));
13578
13579 format %{ "and $dst, $src1, $src2\t# int" %}
13580
13581 ins_cost(INSN_COST);
13582 ins_encode %{
13583 __ andr(as_Register($dst$$reg),
13584 as_Register($src1$$reg),
13585 as_Register($src2$$reg));
13586 %}
13587
13588 ins_pipe(ialu_reg_reg);
13589 %}
13590
13591 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13592 match(Set dst (AndL src1 src2));
13593
13594 format %{ "and $dst, $src1, $src2\t# int" %}
13595
13596 ins_cost(INSN_COST);
13597 ins_encode %{
13598 __ andr(as_Register($dst$$reg),
13599 as_Register($src1$$reg),
13600 (uint64_t)($src2$$constant));
13601 %}
13602
13603 ins_pipe(ialu_reg_imm);
13604 %}
13605
13606 // Or Instructions
13607
13608 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13609 match(Set dst (OrL src1 src2));
13610
13611 format %{ "orr $dst, $src1, $src2\t# int" %}
13612
13613 ins_cost(INSN_COST);
13614 ins_encode %{
13615 __ orr(as_Register($dst$$reg),
13616 as_Register($src1$$reg),
13617 as_Register($src2$$reg));
13618 %}
13619
13620 ins_pipe(ialu_reg_reg);
13621 %}
13622
13623 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13624 match(Set dst (OrL src1 src2));
13625
13626 format %{ "orr $dst, $src1, $src2\t# int" %}
13627
13628 ins_cost(INSN_COST);
13629 ins_encode %{
13630 __ orr(as_Register($dst$$reg),
13631 as_Register($src1$$reg),
13632 (uint64_t)($src2$$constant));
13633 %}
13634
13635 ins_pipe(ialu_reg_imm);
13636 %}
13637
13638 // Xor Instructions
13639
13640 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13641 match(Set dst (XorL src1 src2));
13642
13643 format %{ "eor $dst, $src1, $src2\t# int" %}
13644
13645 ins_cost(INSN_COST);
13646 ins_encode %{
13647 __ eor(as_Register($dst$$reg),
13648 as_Register($src1$$reg),
13649 as_Register($src2$$reg));
13650 %}
13651
13652 ins_pipe(ialu_reg_reg);
13653 %}
13654
13655 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13656 match(Set dst (XorL src1 src2));
13657
13658 ins_cost(INSN_COST);
13659 format %{ "eor $dst, $src1, $src2\t# int" %}
13660
13661 ins_encode %{
13662 __ eor(as_Register($dst$$reg),
13663 as_Register($src1$$reg),
13664 (uint64_t)($src2$$constant));
13665 %}
13666
13667 ins_pipe(ialu_reg_imm);
13668 %}
13669
13670 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13671 %{
13672 match(Set dst (ConvI2L src));
13673
13674 ins_cost(INSN_COST);
13675 format %{ "sxtw $dst, $src\t# i2l" %}
13676 ins_encode %{
13677 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13678 %}
13679 ins_pipe(ialu_reg_shift);
13680 %}
13681
13682 // this pattern occurs in bigmath arithmetic
13683 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13684 %{
13685 match(Set dst (AndL (ConvI2L src) mask));
13686
13687 ins_cost(INSN_COST);
13688 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13689 ins_encode %{
13690 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13691 %}
13692
13693 ins_pipe(ialu_reg_shift);
13694 %}
13695
13696 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13697 match(Set dst (ConvL2I src));
13698
13699 ins_cost(INSN_COST);
13700 format %{ "movw $dst, $src \t// l2i" %}
13701
13702 ins_encode %{
13703 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13704 %}
13705
13706 ins_pipe(ialu_reg);
13707 %}
13708
13709 instruct convD2F_reg(vRegF dst, vRegD src) %{
13710 match(Set dst (ConvD2F src));
13711
13712 ins_cost(INSN_COST * 5);
13713 format %{ "fcvtd $dst, $src \t// d2f" %}
13714
13715 ins_encode %{
13716 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13717 %}
13718
13719 ins_pipe(fp_d2f);
13720 %}
13721
13722 instruct convF2D_reg(vRegD dst, vRegF src) %{
13723 match(Set dst (ConvF2D src));
13724
13725 ins_cost(INSN_COST * 5);
13726 format %{ "fcvts $dst, $src \t// f2d" %}
13727
13728 ins_encode %{
13729 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13730 %}
13731
13732 ins_pipe(fp_f2d);
13733 %}
13734
13735 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13736 match(Set dst (ConvF2I src));
13737
13738 ins_cost(INSN_COST * 5);
13739 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13740
13741 ins_encode %{
13742 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13743 %}
13744
13745 ins_pipe(fp_f2i);
13746 %}
13747
13748 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13749 match(Set dst (ConvF2L src));
13750
13751 ins_cost(INSN_COST * 5);
13752 format %{ "fcvtzs $dst, $src \t// f2l" %}
13753
13754 ins_encode %{
13755 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13756 %}
13757
13758 ins_pipe(fp_f2l);
13759 %}
13760
13761 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13762 match(Set dst (ConvF2HF src));
13763 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13764 "smov $dst, $tmp\t# move result from $tmp to $dst"
13765 %}
13766 effect(TEMP tmp);
13767 ins_encode %{
13768 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13769 %}
13770 ins_pipe(pipe_slow);
13771 %}
13772
13773 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13774 match(Set dst (ConvHF2F src));
13775 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13776 "fcvt $dst, $tmp\t# convert half to single precision"
13777 %}
13778 effect(TEMP tmp);
13779 ins_encode %{
13780 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13781 %}
13782 ins_pipe(pipe_slow);
13783 %}
13784
13785 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13786 match(Set dst (ConvI2F src));
13787
13788 ins_cost(INSN_COST * 5);
13789 format %{ "scvtfws $dst, $src \t// i2f" %}
13790
13791 ins_encode %{
13792 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13793 %}
13794
13795 ins_pipe(fp_i2f);
13796 %}
13797
13798 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13799 match(Set dst (ConvL2F src));
13800
13801 ins_cost(INSN_COST * 5);
13802 format %{ "scvtfs $dst, $src \t// l2f" %}
13803
13804 ins_encode %{
13805 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13806 %}
13807
13808 ins_pipe(fp_l2f);
13809 %}
13810
13811 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13812 match(Set dst (ConvD2I src));
13813
13814 ins_cost(INSN_COST * 5);
13815 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13816
13817 ins_encode %{
13818 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13819 %}
13820
13821 ins_pipe(fp_d2i);
13822 %}
13823
13824 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13825 match(Set dst (ConvD2L src));
13826
13827 ins_cost(INSN_COST * 5);
13828 format %{ "fcvtzd $dst, $src \t// d2l" %}
13829
13830 ins_encode %{
13831 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13832 %}
13833
13834 ins_pipe(fp_d2l);
13835 %}
13836
13837 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13838 match(Set dst (ConvI2D src));
13839
13840 ins_cost(INSN_COST * 5);
13841 format %{ "scvtfwd $dst, $src \t// i2d" %}
13842
13843 ins_encode %{
13844 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13845 %}
13846
13847 ins_pipe(fp_i2d);
13848 %}
13849
13850 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13851 match(Set dst (ConvL2D src));
13852
13853 ins_cost(INSN_COST * 5);
13854 format %{ "scvtfd $dst, $src \t// l2d" %}
13855
13856 ins_encode %{
13857 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13858 %}
13859
13860 ins_pipe(fp_l2d);
13861 %}
13862
13863 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13864 %{
13865 match(Set dst (RoundD src));
13866 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13867 format %{ "java_round_double $dst,$src"%}
13868 ins_encode %{
13869 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13870 as_FloatRegister($ftmp$$reg));
13871 %}
13872 ins_pipe(pipe_slow);
13873 %}
13874
13875 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13876 %{
13877 match(Set dst (RoundF src));
13878 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13879 format %{ "java_round_float $dst,$src"%}
13880 ins_encode %{
13881 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13882 as_FloatRegister($ftmp$$reg));
13883 %}
13884 ins_pipe(pipe_slow);
13885 %}
13886
13887 // stack <-> reg and reg <-> reg shuffles with no conversion
13888
13889 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13890
13891 match(Set dst (MoveF2I src));
13892
13893 effect(DEF dst, USE src);
13894
13895 ins_cost(4 * INSN_COST);
13896
13897 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13898
13899 ins_encode %{
13900 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13901 %}
13902
13903 ins_pipe(iload_reg_reg);
13904
13905 %}
13906
13907 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13908
13909 match(Set dst (MoveI2F src));
13910
13911 effect(DEF dst, USE src);
13912
13913 ins_cost(4 * INSN_COST);
13914
13915 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13916
13917 ins_encode %{
13918 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13919 %}
13920
13921 ins_pipe(pipe_class_memory);
13922
13923 %}
13924
13925 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13926
13927 match(Set dst (MoveD2L src));
13928
13929 effect(DEF dst, USE src);
13930
13931 ins_cost(4 * INSN_COST);
13932
13933 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13934
13935 ins_encode %{
13936 __ ldr($dst$$Register, Address(sp, $src$$disp));
13937 %}
13938
13939 ins_pipe(iload_reg_reg);
13940
13941 %}
13942
13943 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13944
13945 match(Set dst (MoveL2D src));
13946
13947 effect(DEF dst, USE src);
13948
13949 ins_cost(4 * INSN_COST);
13950
13951 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13952
13953 ins_encode %{
13954 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13955 %}
13956
13957 ins_pipe(pipe_class_memory);
13958
13959 %}
13960
13961 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13962
13963 match(Set dst (MoveF2I src));
13964
13965 effect(DEF dst, USE src);
13966
13967 ins_cost(INSN_COST);
13968
13969 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13970
13971 ins_encode %{
13972 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13973 %}
13974
13975 ins_pipe(pipe_class_memory);
13976
13977 %}
13978
13979 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13980
13981 match(Set dst (MoveI2F src));
13982
13983 effect(DEF dst, USE src);
13984
13985 ins_cost(INSN_COST);
13986
13987 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13988
13989 ins_encode %{
13990 __ strw($src$$Register, Address(sp, $dst$$disp));
13991 %}
13992
13993 ins_pipe(istore_reg_reg);
13994
13995 %}
13996
13997 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13998
13999 match(Set dst (MoveD2L src));
14000
14001 effect(DEF dst, USE src);
14002
14003 ins_cost(INSN_COST);
14004
14005 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14006
14007 ins_encode %{
14008 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14009 %}
14010
14011 ins_pipe(pipe_class_memory);
14012
14013 %}
14014
14015 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14016
14017 match(Set dst (MoveL2D src));
14018
14019 effect(DEF dst, USE src);
14020
14021 ins_cost(INSN_COST);
14022
14023 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14024
14025 ins_encode %{
14026 __ str($src$$Register, Address(sp, $dst$$disp));
14027 %}
14028
14029 ins_pipe(istore_reg_reg);
14030
14031 %}
14032
14033 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14034
14035 match(Set dst (MoveF2I src));
14036
14037 effect(DEF dst, USE src);
14038
14039 ins_cost(INSN_COST);
14040
14041 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14042
14043 ins_encode %{
14044 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14045 %}
14046
14047 ins_pipe(fp_f2i);
14048
14049 %}
14050
14051 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14052
14053 match(Set dst (MoveI2F src));
14054
14055 effect(DEF dst, USE src);
14056
14057 ins_cost(INSN_COST);
14058
14059 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14060
14061 ins_encode %{
14062 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14063 %}
14064
14065 ins_pipe(fp_i2f);
14066
14067 %}
14068
14069 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14070
14071 match(Set dst (MoveD2L src));
14072
14073 effect(DEF dst, USE src);
14074
14075 ins_cost(INSN_COST);
14076
14077 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14078
14079 ins_encode %{
14080 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14081 %}
14082
14083 ins_pipe(fp_d2l);
14084
14085 %}
14086
14087 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14088
14089 match(Set dst (MoveL2D src));
14090
14091 effect(DEF dst, USE src);
14092
14093 ins_cost(INSN_COST);
14094
14095 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14096
14097 ins_encode %{
14098 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14099 %}
14100
14101 ins_pipe(fp_l2d);
14102
14103 %}
14104
14105 // ============================================================================
14106 // clearing of an array
14107
14108 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14109 %{
14110 match(Set dummy (ClearArray (Binary cnt base) zero));
14111 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14112
14113 ins_cost(4 * INSN_COST);
14114 format %{ "ClearArray $cnt, $base" %}
14115
14116 ins_encode %{
14117 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14118 if (tpc == nullptr) {
14119 ciEnv::current()->record_failure("CodeCache is full");
14120 return;
14121 }
14122 %}
14123
14124 ins_pipe(pipe_class_memory);
14125 %}
14126
14127 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14128 %{
14129 predicate(((ClearArrayNode*)n)->word_copy_only());
14130 match(Set dummy (ClearArray (Binary cnt base) val));
14131 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14132
14133 ins_cost(4 * INSN_COST);
14134 format %{ "ClearArray $cnt, $base, $val" %}
14135
14136 ins_encode %{
14137 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14138 %}
14139
14140 ins_pipe(pipe_class_memory);
14141 %}
14142
14143 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14144 %{
14145 predicate((uint64_t)n->in(2)->get_long()
14146 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14147 && !((ClearArrayNode*)n)->word_copy_only());
14148 match(Set dummy (ClearArray cnt base));
14149 effect(TEMP temp, USE_KILL base, KILL cr);
14150
14151 ins_cost(4 * INSN_COST);
14152 format %{ "ClearArray $cnt, $base" %}
14153
14154 ins_encode %{
14155 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14156 if (tpc == nullptr) {
14157 ciEnv::current()->record_failure("CodeCache is full");
14158 return;
14159 }
14160 %}
14161
14162 ins_pipe(pipe_class_memory);
14163 %}
14164
14165 // ============================================================================
14166 // Overflow Math Instructions
14167
14168 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14169 %{
14170 match(Set cr (OverflowAddI op1 op2));
14171
14172 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14173 ins_cost(INSN_COST);
14174 ins_encode %{
14175 __ cmnw($op1$$Register, $op2$$Register);
14176 %}
14177
14178 ins_pipe(icmp_reg_reg);
14179 %}
14180
14181 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14182 %{
14183 match(Set cr (OverflowAddI op1 op2));
14184
14185 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14186 ins_cost(INSN_COST);
14187 ins_encode %{
14188 __ cmnw($op1$$Register, $op2$$constant);
14189 %}
14190
14191 ins_pipe(icmp_reg_imm);
14192 %}
14193
14194 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14195 %{
14196 match(Set cr (OverflowAddL op1 op2));
14197
14198 format %{ "cmn $op1, $op2\t# overflow check long" %}
14199 ins_cost(INSN_COST);
14200 ins_encode %{
14201 __ cmn($op1$$Register, $op2$$Register);
14202 %}
14203
14204 ins_pipe(icmp_reg_reg);
14205 %}
14206
14207 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14208 %{
14209 match(Set cr (OverflowAddL op1 op2));
14210
14211 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14212 ins_cost(INSN_COST);
14213 ins_encode %{
14214 __ adds(zr, $op1$$Register, $op2$$constant);
14215 %}
14216
14217 ins_pipe(icmp_reg_imm);
14218 %}
14219
14220 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14221 %{
14222 match(Set cr (OverflowSubI op1 op2));
14223
14224 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14225 ins_cost(INSN_COST);
14226 ins_encode %{
14227 __ cmpw($op1$$Register, $op2$$Register);
14228 %}
14229
14230 ins_pipe(icmp_reg_reg);
14231 %}
14232
14233 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14234 %{
14235 match(Set cr (OverflowSubI op1 op2));
14236
14237 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14238 ins_cost(INSN_COST);
14239 ins_encode %{
14240 __ cmpw($op1$$Register, $op2$$constant);
14241 %}
14242
14243 ins_pipe(icmp_reg_imm);
14244 %}
14245
14246 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14247 %{
14248 match(Set cr (OverflowSubL op1 op2));
14249
14250 format %{ "cmp $op1, $op2\t# overflow check long" %}
14251 ins_cost(INSN_COST);
14252 ins_encode %{
14253 __ cmp($op1$$Register, $op2$$Register);
14254 %}
14255
14256 ins_pipe(icmp_reg_reg);
14257 %}
14258
14259 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14260 %{
14261 match(Set cr (OverflowSubL op1 op2));
14262
14263 format %{ "cmp $op1, $op2\t# overflow check long" %}
14264 ins_cost(INSN_COST);
14265 ins_encode %{
14266 __ subs(zr, $op1$$Register, $op2$$constant);
14267 %}
14268
14269 ins_pipe(icmp_reg_imm);
14270 %}
14271
14272 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14273 %{
14274 match(Set cr (OverflowSubI zero op1));
14275
14276 format %{ "cmpw zr, $op1\t# overflow check int" %}
14277 ins_cost(INSN_COST);
14278 ins_encode %{
14279 __ cmpw(zr, $op1$$Register);
14280 %}
14281
14282 ins_pipe(icmp_reg_imm);
14283 %}
14284
14285 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14286 %{
14287 match(Set cr (OverflowSubL zero op1));
14288
14289 format %{ "cmp zr, $op1\t# overflow check long" %}
14290 ins_cost(INSN_COST);
14291 ins_encode %{
14292 __ cmp(zr, $op1$$Register);
14293 %}
14294
14295 ins_pipe(icmp_reg_imm);
14296 %}
14297
14298 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14299 %{
14300 match(Set cr (OverflowMulI op1 op2));
14301
14302 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14303 "cmp rscratch1, rscratch1, sxtw\n\t"
14304 "movw rscratch1, #0x80000000\n\t"
14305 "cselw rscratch1, rscratch1, zr, NE\n\t"
14306 "cmpw rscratch1, #1" %}
14307 ins_cost(5 * INSN_COST);
14308 ins_encode %{
14309 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14310 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14311 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14312 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14313 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14314 %}
14315
14316 ins_pipe(pipe_slow);
14317 %}
14318
14319 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14320 %{
14321 match(If cmp (OverflowMulI op1 op2));
14322 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14323 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14324 effect(USE labl, KILL cr);
14325
14326 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14327 "cmp rscratch1, rscratch1, sxtw\n\t"
14328 "b$cmp $labl" %}
14329 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14330 ins_encode %{
14331 Label* L = $labl$$label;
14332 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14333 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14334 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14335 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14336 %}
14337
14338 ins_pipe(pipe_serial);
14339 %}
14340
14341 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14342 %{
14343 match(Set cr (OverflowMulL op1 op2));
14344
14345 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14346 "smulh rscratch2, $op1, $op2\n\t"
14347 "cmp rscratch2, rscratch1, ASR #63\n\t"
14348 "movw rscratch1, #0x80000000\n\t"
14349 "cselw rscratch1, rscratch1, zr, NE\n\t"
14350 "cmpw rscratch1, #1" %}
14351 ins_cost(6 * INSN_COST);
14352 ins_encode %{
14353 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14354 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14355 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14356 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14357 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14358 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14359 %}
14360
14361 ins_pipe(pipe_slow);
14362 %}
14363
14364 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14365 %{
14366 match(If cmp (OverflowMulL op1 op2));
14367 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14368 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14369 effect(USE labl, KILL cr);
14370
14371 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14372 "smulh rscratch2, $op1, $op2\n\t"
14373 "cmp rscratch2, rscratch1, ASR #63\n\t"
14374 "b$cmp $labl" %}
14375 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14376 ins_encode %{
14377 Label* L = $labl$$label;
14378 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14379 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14380 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14381 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14382 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14383 %}
14384
14385 ins_pipe(pipe_serial);
14386 %}
14387
14388 // ============================================================================
14389 // Compare Instructions
14390
14391 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14392 %{
14393 match(Set cr (CmpI op1 op2));
14394
14395 effect(DEF cr, USE op1, USE op2);
14396
14397 ins_cost(INSN_COST);
14398 format %{ "cmpw $op1, $op2" %}
14399
14400 ins_encode(aarch64_enc_cmpw(op1, op2));
14401
14402 ins_pipe(icmp_reg_reg);
14403 %}
14404
14405 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14406 %{
14407 match(Set cr (CmpI op1 zero));
14408
14409 effect(DEF cr, USE op1);
14410
14411 ins_cost(INSN_COST);
14412 format %{ "cmpw $op1, 0" %}
14413
14414 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14415
14416 ins_pipe(icmp_reg_imm);
14417 %}
14418
14419 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14420 %{
14421 match(Set cr (CmpI op1 op2));
14422
14423 effect(DEF cr, USE op1);
14424
14425 ins_cost(INSN_COST);
14426 format %{ "cmpw $op1, $op2" %}
14427
14428 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14429
14430 ins_pipe(icmp_reg_imm);
14431 %}
14432
14433 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14434 %{
14435 match(Set cr (CmpI op1 op2));
14436
14437 effect(DEF cr, USE op1);
14438
14439 ins_cost(INSN_COST * 2);
14440 format %{ "cmpw $op1, $op2" %}
14441
14442 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14443
14444 ins_pipe(icmp_reg_imm);
14445 %}
14446
14447 // Unsigned compare Instructions; really, same as signed compare
14448 // except it should only be used to feed an If or a CMovI which takes a
14449 // cmpOpU.
14450
14451 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14452 %{
14453 match(Set cr (CmpU op1 op2));
14454
14455 effect(DEF cr, USE op1, USE op2);
14456
14457 ins_cost(INSN_COST);
14458 format %{ "cmpw $op1, $op2\t# unsigned" %}
14459
14460 ins_encode(aarch64_enc_cmpw(op1, op2));
14461
14462 ins_pipe(icmp_reg_reg);
14463 %}
14464
14465 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14466 %{
14467 match(Set cr (CmpU op1 zero));
14468
14469 effect(DEF cr, USE op1);
14470
14471 ins_cost(INSN_COST);
14472 format %{ "cmpw $op1, #0\t# unsigned" %}
14473
14474 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14475
14476 ins_pipe(icmp_reg_imm);
14477 %}
14478
14479 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14480 %{
14481 match(Set cr (CmpU op1 op2));
14482
14483 effect(DEF cr, USE op1);
14484
14485 ins_cost(INSN_COST);
14486 format %{ "cmpw $op1, $op2\t# unsigned" %}
14487
14488 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14489
14490 ins_pipe(icmp_reg_imm);
14491 %}
14492
14493 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14494 %{
14495 match(Set cr (CmpU op1 op2));
14496
14497 effect(DEF cr, USE op1);
14498
14499 ins_cost(INSN_COST * 2);
14500 format %{ "cmpw $op1, $op2\t# unsigned" %}
14501
14502 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14503
14504 ins_pipe(icmp_reg_imm);
14505 %}
14506
14507 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14508 %{
14509 match(Set cr (CmpL op1 op2));
14510
14511 effect(DEF cr, USE op1, USE op2);
14512
14513 ins_cost(INSN_COST);
14514 format %{ "cmp $op1, $op2" %}
14515
14516 ins_encode(aarch64_enc_cmp(op1, op2));
14517
14518 ins_pipe(icmp_reg_reg);
14519 %}
14520
14521 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14522 %{
14523 match(Set cr (CmpL op1 zero));
14524
14525 effect(DEF cr, USE op1);
14526
14527 ins_cost(INSN_COST);
14528 format %{ "tst $op1" %}
14529
14530 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14531
14532 ins_pipe(icmp_reg_imm);
14533 %}
14534
14535 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14536 %{
14537 match(Set cr (CmpL op1 op2));
14538
14539 effect(DEF cr, USE op1);
14540
14541 ins_cost(INSN_COST);
14542 format %{ "cmp $op1, $op2" %}
14543
14544 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14545
14546 ins_pipe(icmp_reg_imm);
14547 %}
14548
14549 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14550 %{
14551 match(Set cr (CmpL op1 op2));
14552
14553 effect(DEF cr, USE op1);
14554
14555 ins_cost(INSN_COST * 2);
14556 format %{ "cmp $op1, $op2" %}
14557
14558 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14559
14560 ins_pipe(icmp_reg_imm);
14561 %}
14562
14563 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14564 %{
14565 match(Set cr (CmpUL op1 op2));
14566
14567 effect(DEF cr, USE op1, USE op2);
14568
14569 ins_cost(INSN_COST);
14570 format %{ "cmp $op1, $op2" %}
14571
14572 ins_encode(aarch64_enc_cmp(op1, op2));
14573
14574 ins_pipe(icmp_reg_reg);
14575 %}
14576
14577 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14578 %{
14579 match(Set cr (CmpUL op1 zero));
14580
14581 effect(DEF cr, USE op1);
14582
14583 ins_cost(INSN_COST);
14584 format %{ "tst $op1" %}
14585
14586 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14587
14588 ins_pipe(icmp_reg_imm);
14589 %}
14590
14591 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14592 %{
14593 match(Set cr (CmpUL op1 op2));
14594
14595 effect(DEF cr, USE op1);
14596
14597 ins_cost(INSN_COST);
14598 format %{ "cmp $op1, $op2" %}
14599
14600 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14601
14602 ins_pipe(icmp_reg_imm);
14603 %}
14604
14605 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14606 %{
14607 match(Set cr (CmpUL op1 op2));
14608
14609 effect(DEF cr, USE op1);
14610
14611 ins_cost(INSN_COST * 2);
14612 format %{ "cmp $op1, $op2" %}
14613
14614 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14615
14616 ins_pipe(icmp_reg_imm);
14617 %}
14618
14619 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14620 %{
14621 match(Set cr (CmpP op1 op2));
14622
14623 effect(DEF cr, USE op1, USE op2);
14624
14625 ins_cost(INSN_COST);
14626 format %{ "cmp $op1, $op2\t // ptr" %}
14627
14628 ins_encode(aarch64_enc_cmpp(op1, op2));
14629
14630 ins_pipe(icmp_reg_reg);
14631 %}
14632
14633 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14634 %{
14635 match(Set cr (CmpN op1 op2));
14636
14637 effect(DEF cr, USE op1, USE op2);
14638
14639 ins_cost(INSN_COST);
14640 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14641
14642 ins_encode(aarch64_enc_cmpn(op1, op2));
14643
14644 ins_pipe(icmp_reg_reg);
14645 %}
14646
14647 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14648 %{
14649 match(Set cr (CmpP op1 zero));
14650
14651 effect(DEF cr, USE op1, USE zero);
14652
14653 ins_cost(INSN_COST);
14654 format %{ "cmp $op1, 0\t // ptr" %}
14655
14656 ins_encode(aarch64_enc_testp(op1));
14657
14658 ins_pipe(icmp_reg_imm);
14659 %}
14660
14661 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14662 %{
14663 match(Set cr (CmpN op1 zero));
14664
14665 effect(DEF cr, USE op1, USE zero);
14666
14667 ins_cost(INSN_COST);
14668 format %{ "cmp $op1, 0\t // compressed ptr" %}
14669
14670 ins_encode(aarch64_enc_testn(op1));
14671
14672 ins_pipe(icmp_reg_imm);
14673 %}
14674
14675 // FP comparisons
14676 //
14677 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14678 // using normal cmpOp. See declaration of rFlagsReg for details.
14679
14680 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14681 %{
14682 match(Set cr (CmpF src1 src2));
14683
14684 ins_cost(3 * INSN_COST);
14685 format %{ "fcmps $src1, $src2" %}
14686
14687 ins_encode %{
14688 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14689 %}
14690
14691 ins_pipe(pipe_class_compare);
14692 %}
14693
14694 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14695 %{
14696 match(Set cr (CmpF src1 src2));
14697
14698 ins_cost(3 * INSN_COST);
14699 format %{ "fcmps $src1, 0.0" %}
14700
14701 ins_encode %{
14702 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14703 %}
14704
14705 ins_pipe(pipe_class_compare);
14706 %}
14707 // FROM HERE
14708
14709 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14710 %{
14711 match(Set cr (CmpD src1 src2));
14712
14713 ins_cost(3 * INSN_COST);
14714 format %{ "fcmpd $src1, $src2" %}
14715
14716 ins_encode %{
14717 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14718 %}
14719
14720 ins_pipe(pipe_class_compare);
14721 %}
14722
14723 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14724 %{
14725 match(Set cr (CmpD src1 src2));
14726
14727 ins_cost(3 * INSN_COST);
14728 format %{ "fcmpd $src1, 0.0" %}
14729
14730 ins_encode %{
14731 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14732 %}
14733
14734 ins_pipe(pipe_class_compare);
14735 %}
14736
14737 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14738 %{
14739 match(Set dst (CmpF3 src1 src2));
14740 effect(KILL cr);
14741
14742 ins_cost(5 * INSN_COST);
14743 format %{ "fcmps $src1, $src2\n\t"
14744 "csinvw($dst, zr, zr, eq\n\t"
14745 "csnegw($dst, $dst, $dst, lt)"
14746 %}
14747
14748 ins_encode %{
14749 Label done;
14750 FloatRegister s1 = as_FloatRegister($src1$$reg);
14751 FloatRegister s2 = as_FloatRegister($src2$$reg);
14752 Register d = as_Register($dst$$reg);
14753 __ fcmps(s1, s2);
14754 // installs 0 if EQ else -1
14755 __ csinvw(d, zr, zr, Assembler::EQ);
14756 // keeps -1 if less or unordered else installs 1
14757 __ csnegw(d, d, d, Assembler::LT);
14758 __ bind(done);
14759 %}
14760
14761 ins_pipe(pipe_class_default);
14762
14763 %}
14764
14765 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14766 %{
14767 match(Set dst (CmpD3 src1 src2));
14768 effect(KILL cr);
14769
14770 ins_cost(5 * INSN_COST);
14771 format %{ "fcmpd $src1, $src2\n\t"
14772 "csinvw($dst, zr, zr, eq\n\t"
14773 "csnegw($dst, $dst, $dst, lt)"
14774 %}
14775
14776 ins_encode %{
14777 Label done;
14778 FloatRegister s1 = as_FloatRegister($src1$$reg);
14779 FloatRegister s2 = as_FloatRegister($src2$$reg);
14780 Register d = as_Register($dst$$reg);
14781 __ fcmpd(s1, s2);
14782 // installs 0 if EQ else -1
14783 __ csinvw(d, zr, zr, Assembler::EQ);
14784 // keeps -1 if less or unordered else installs 1
14785 __ csnegw(d, d, d, Assembler::LT);
14786 __ bind(done);
14787 %}
14788 ins_pipe(pipe_class_default);
14789
14790 %}
14791
14792 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14793 %{
14794 match(Set dst (CmpF3 src1 zero));
14795 effect(KILL cr);
14796
14797 ins_cost(5 * INSN_COST);
14798 format %{ "fcmps $src1, 0.0\n\t"
14799 "csinvw($dst, zr, zr, eq\n\t"
14800 "csnegw($dst, $dst, $dst, lt)"
14801 %}
14802
14803 ins_encode %{
14804 Label done;
14805 FloatRegister s1 = as_FloatRegister($src1$$reg);
14806 Register d = as_Register($dst$$reg);
14807 __ fcmps(s1, 0.0);
14808 // installs 0 if EQ else -1
14809 __ csinvw(d, zr, zr, Assembler::EQ);
14810 // keeps -1 if less or unordered else installs 1
14811 __ csnegw(d, d, d, Assembler::LT);
14812 __ bind(done);
14813 %}
14814
14815 ins_pipe(pipe_class_default);
14816
14817 %}
14818
14819 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14820 %{
14821 match(Set dst (CmpD3 src1 zero));
14822 effect(KILL cr);
14823
14824 ins_cost(5 * INSN_COST);
14825 format %{ "fcmpd $src1, 0.0\n\t"
14826 "csinvw($dst, zr, zr, eq\n\t"
14827 "csnegw($dst, $dst, $dst, lt)"
14828 %}
14829
14830 ins_encode %{
14831 Label done;
14832 FloatRegister s1 = as_FloatRegister($src1$$reg);
14833 Register d = as_Register($dst$$reg);
14834 __ fcmpd(s1, 0.0);
14835 // installs 0 if EQ else -1
14836 __ csinvw(d, zr, zr, Assembler::EQ);
14837 // keeps -1 if less or unordered else installs 1
14838 __ csnegw(d, d, d, Assembler::LT);
14839 __ bind(done);
14840 %}
14841 ins_pipe(pipe_class_default);
14842
14843 %}
14844
14845 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14846 %{
14847 match(Set dst (CmpLTMask p q));
14848 effect(KILL cr);
14849
14850 ins_cost(3 * INSN_COST);
14851
14852 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14853 "csetw $dst, lt\n\t"
14854 "subw $dst, zr, $dst"
14855 %}
14856
14857 ins_encode %{
14858 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14859 __ csetw(as_Register($dst$$reg), Assembler::LT);
14860 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14861 %}
14862
14863 ins_pipe(ialu_reg_reg);
14864 %}
14865
14866 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14867 %{
14868 match(Set dst (CmpLTMask src zero));
14869 effect(KILL cr);
14870
14871 ins_cost(INSN_COST);
14872
14873 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14874
14875 ins_encode %{
14876 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14877 %}
14878
14879 ins_pipe(ialu_reg_shift);
14880 %}
14881
14882 // ============================================================================
14883 // Max and Min
14884
14885 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14886
14887 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14888 %{
14889 effect(DEF cr, USE src);
14890 ins_cost(INSN_COST);
14891 format %{ "cmpw $src, 0" %}
14892
14893 ins_encode %{
14894 __ cmpw($src$$Register, 0);
14895 %}
14896 ins_pipe(icmp_reg_imm);
14897 %}
14898
14899 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14900 %{
14901 match(Set dst (MinI src1 src2));
14902 ins_cost(INSN_COST * 3);
14903
14904 expand %{
14905 rFlagsReg cr;
14906 compI_reg_reg(cr, src1, src2);
14907 cmovI_reg_reg_lt(dst, src1, src2, cr);
14908 %}
14909 %}
14910
14911 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14912 %{
14913 match(Set dst (MaxI src1 src2));
14914 ins_cost(INSN_COST * 3);
14915
14916 expand %{
14917 rFlagsReg cr;
14918 compI_reg_reg(cr, src1, src2);
14919 cmovI_reg_reg_gt(dst, src1, src2, cr);
14920 %}
14921 %}
14922
14923
14924 // ============================================================================
14925 // Branch Instructions
14926
14927 // Direct Branch.
14928 instruct branch(label lbl)
14929 %{
14930 match(Goto);
14931
14932 effect(USE lbl);
14933
14934 ins_cost(BRANCH_COST);
14935 format %{ "b $lbl" %}
14936
14937 ins_encode(aarch64_enc_b(lbl));
14938
14939 ins_pipe(pipe_branch);
14940 %}
14941
14942 // Conditional Near Branch
14943 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14944 %{
14945 // Same match rule as `branchConFar'.
14946 match(If cmp cr);
14947
14948 effect(USE lbl);
14949
14950 ins_cost(BRANCH_COST);
14951 // If set to 1 this indicates that the current instruction is a
14952 // short variant of a long branch. This avoids using this
14953 // instruction in first-pass matching. It will then only be used in
14954 // the `Shorten_branches' pass.
14955 // ins_short_branch(1);
14956 format %{ "b$cmp $lbl" %}
14957
14958 ins_encode(aarch64_enc_br_con(cmp, lbl));
14959
14960 ins_pipe(pipe_branch_cond);
14961 %}
14962
14963 // Conditional Near Branch Unsigned
14964 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14965 %{
14966 // Same match rule as `branchConFar'.
14967 match(If cmp cr);
14968
14969 effect(USE lbl);
14970
14971 ins_cost(BRANCH_COST);
14972 // If set to 1 this indicates that the current instruction is a
14973 // short variant of a long branch. This avoids using this
14974 // instruction in first-pass matching. It will then only be used in
14975 // the `Shorten_branches' pass.
14976 // ins_short_branch(1);
14977 format %{ "b$cmp $lbl\t# unsigned" %}
14978
14979 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14980
14981 ins_pipe(pipe_branch_cond);
14982 %}
14983
14984 // Make use of CBZ and CBNZ. These instructions, as well as being
14985 // shorter than (cmp; branch), have the additional benefit of not
14986 // killing the flags.
14987
14988 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14989 match(If cmp (CmpI op1 op2));
14990 effect(USE labl);
14991
14992 ins_cost(BRANCH_COST);
14993 format %{ "cbw$cmp $op1, $labl" %}
14994 ins_encode %{
14995 Label* L = $labl$$label;
14996 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14997 if (cond == Assembler::EQ)
14998 __ cbzw($op1$$Register, *L);
14999 else
15000 __ cbnzw($op1$$Register, *L);
15001 %}
15002 ins_pipe(pipe_cmp_branch);
15003 %}
15004
15005 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15006 match(If cmp (CmpL op1 op2));
15007 effect(USE labl);
15008
15009 ins_cost(BRANCH_COST);
15010 format %{ "cb$cmp $op1, $labl" %}
15011 ins_encode %{
15012 Label* L = $labl$$label;
15013 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15014 if (cond == Assembler::EQ)
15015 __ cbz($op1$$Register, *L);
15016 else
15017 __ cbnz($op1$$Register, *L);
15018 %}
15019 ins_pipe(pipe_cmp_branch);
15020 %}
15021
15022 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15023 match(If cmp (CmpP op1 op2));
15024 effect(USE labl);
15025
15026 ins_cost(BRANCH_COST);
15027 format %{ "cb$cmp $op1, $labl" %}
15028 ins_encode %{
15029 Label* L = $labl$$label;
15030 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15031 if (cond == Assembler::EQ)
15032 __ cbz($op1$$Register, *L);
15033 else
15034 __ cbnz($op1$$Register, *L);
15035 %}
15036 ins_pipe(pipe_cmp_branch);
15037 %}
15038
15039 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15040 match(If cmp (CmpN op1 op2));
15041 effect(USE labl);
15042
15043 ins_cost(BRANCH_COST);
15044 format %{ "cbw$cmp $op1, $labl" %}
15045 ins_encode %{
15046 Label* L = $labl$$label;
15047 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15048 if (cond == Assembler::EQ)
15049 __ cbzw($op1$$Register, *L);
15050 else
15051 __ cbnzw($op1$$Register, *L);
15052 %}
15053 ins_pipe(pipe_cmp_branch);
15054 %}
15055
15056 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15057 match(If cmp (CmpP (DecodeN oop) zero));
15058 effect(USE labl);
15059
15060 ins_cost(BRANCH_COST);
15061 format %{ "cb$cmp $oop, $labl" %}
15062 ins_encode %{
15063 Label* L = $labl$$label;
15064 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15065 if (cond == Assembler::EQ)
15066 __ cbzw($oop$$Register, *L);
15067 else
15068 __ cbnzw($oop$$Register, *L);
15069 %}
15070 ins_pipe(pipe_cmp_branch);
15071 %}
15072
15073 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15074 match(If cmp (CmpU op1 op2));
15075 effect(USE labl);
15076
15077 ins_cost(BRANCH_COST);
15078 format %{ "cbw$cmp $op1, $labl" %}
15079 ins_encode %{
15080 Label* L = $labl$$label;
15081 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15082 if (cond == Assembler::EQ || cond == Assembler::LS) {
15083 __ cbzw($op1$$Register, *L);
15084 } else {
15085 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15086 __ cbnzw($op1$$Register, *L);
15087 }
15088 %}
15089 ins_pipe(pipe_cmp_branch);
15090 %}
15091
15092 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15093 match(If cmp (CmpUL op1 op2));
15094 effect(USE labl);
15095
15096 ins_cost(BRANCH_COST);
15097 format %{ "cb$cmp $op1, $labl" %}
15098 ins_encode %{
15099 Label* L = $labl$$label;
15100 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15101 if (cond == Assembler::EQ || cond == Assembler::LS) {
15102 __ cbz($op1$$Register, *L);
15103 } else {
15104 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15105 __ cbnz($op1$$Register, *L);
15106 }
15107 %}
15108 ins_pipe(pipe_cmp_branch);
15109 %}
15110
15111 // Test bit and Branch
15112
15113 // Patterns for short (< 32KiB) variants
15114 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15115 match(If cmp (CmpL op1 op2));
15116 effect(USE labl);
15117
15118 ins_cost(BRANCH_COST);
15119 format %{ "cb$cmp $op1, $labl # long" %}
15120 ins_encode %{
15121 Label* L = $labl$$label;
15122 Assembler::Condition cond =
15123 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15124 __ tbr(cond, $op1$$Register, 63, *L);
15125 %}
15126 ins_pipe(pipe_cmp_branch);
15127 ins_short_branch(1);
15128 %}
15129
15130 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15131 match(If cmp (CmpI op1 op2));
15132 effect(USE labl);
15133
15134 ins_cost(BRANCH_COST);
15135 format %{ "cb$cmp $op1, $labl # int" %}
15136 ins_encode %{
15137 Label* L = $labl$$label;
15138 Assembler::Condition cond =
15139 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15140 __ tbr(cond, $op1$$Register, 31, *L);
15141 %}
15142 ins_pipe(pipe_cmp_branch);
15143 ins_short_branch(1);
15144 %}
15145
15146 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15147 match(If cmp (CmpL (AndL op1 op2) op3));
15148 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15149 effect(USE labl);
15150
15151 ins_cost(BRANCH_COST);
15152 format %{ "tb$cmp $op1, $op2, $labl" %}
15153 ins_encode %{
15154 Label* L = $labl$$label;
15155 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15156 int bit = exact_log2_long($op2$$constant);
15157 __ tbr(cond, $op1$$Register, bit, *L);
15158 %}
15159 ins_pipe(pipe_cmp_branch);
15160 ins_short_branch(1);
15161 %}
15162
15163 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15164 match(If cmp (CmpI (AndI op1 op2) op3));
15165 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15166 effect(USE labl);
15167
15168 ins_cost(BRANCH_COST);
15169 format %{ "tb$cmp $op1, $op2, $labl" %}
15170 ins_encode %{
15171 Label* L = $labl$$label;
15172 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15173 int bit = exact_log2((juint)$op2$$constant);
15174 __ tbr(cond, $op1$$Register, bit, *L);
15175 %}
15176 ins_pipe(pipe_cmp_branch);
15177 ins_short_branch(1);
15178 %}
15179
15180 // And far variants
15181 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15182 match(If cmp (CmpL op1 op2));
15183 effect(USE labl);
15184
15185 ins_cost(BRANCH_COST);
15186 format %{ "cb$cmp $op1, $labl # long" %}
15187 ins_encode %{
15188 Label* L = $labl$$label;
15189 Assembler::Condition cond =
15190 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15191 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15192 %}
15193 ins_pipe(pipe_cmp_branch);
15194 %}
15195
15196 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15197 match(If cmp (CmpI op1 op2));
15198 effect(USE labl);
15199
15200 ins_cost(BRANCH_COST);
15201 format %{ "cb$cmp $op1, $labl # int" %}
15202 ins_encode %{
15203 Label* L = $labl$$label;
15204 Assembler::Condition cond =
15205 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15206 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15207 %}
15208 ins_pipe(pipe_cmp_branch);
15209 %}
15210
15211 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15212 match(If cmp (CmpL (AndL op1 op2) op3));
15213 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15214 effect(USE labl);
15215
15216 ins_cost(BRANCH_COST);
15217 format %{ "tb$cmp $op1, $op2, $labl" %}
15218 ins_encode %{
15219 Label* L = $labl$$label;
15220 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15221 int bit = exact_log2_long($op2$$constant);
15222 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15223 %}
15224 ins_pipe(pipe_cmp_branch);
15225 %}
15226
15227 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15228 match(If cmp (CmpI (AndI op1 op2) op3));
15229 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15230 effect(USE labl);
15231
15232 ins_cost(BRANCH_COST);
15233 format %{ "tb$cmp $op1, $op2, $labl" %}
15234 ins_encode %{
15235 Label* L = $labl$$label;
15236 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15237 int bit = exact_log2((juint)$op2$$constant);
15238 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15239 %}
15240 ins_pipe(pipe_cmp_branch);
15241 %}
15242
15243 // Test bits
15244
15245 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15246 match(Set cr (CmpL (AndL op1 op2) op3));
15247 predicate(Assembler::operand_valid_for_logical_immediate
15248 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15249
15250 ins_cost(INSN_COST);
15251 format %{ "tst $op1, $op2 # long" %}
15252 ins_encode %{
15253 __ tst($op1$$Register, $op2$$constant);
15254 %}
15255 ins_pipe(ialu_reg_reg);
15256 %}
15257
15258 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15259 match(Set cr (CmpI (AndI op1 op2) op3));
15260 predicate(Assembler::operand_valid_for_logical_immediate
15261 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15262
15263 ins_cost(INSN_COST);
15264 format %{ "tst $op1, $op2 # int" %}
15265 ins_encode %{
15266 __ tstw($op1$$Register, $op2$$constant);
15267 %}
15268 ins_pipe(ialu_reg_reg);
15269 %}
15270
15271 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15272 match(Set cr (CmpL (AndL op1 op2) op3));
15273
15274 ins_cost(INSN_COST);
15275 format %{ "tst $op1, $op2 # long" %}
15276 ins_encode %{
15277 __ tst($op1$$Register, $op2$$Register);
15278 %}
15279 ins_pipe(ialu_reg_reg);
15280 %}
15281
15282 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15283 match(Set cr (CmpI (AndI op1 op2) op3));
15284
15285 ins_cost(INSN_COST);
15286 format %{ "tstw $op1, $op2 # int" %}
15287 ins_encode %{
15288 __ tstw($op1$$Register, $op2$$Register);
15289 %}
15290 ins_pipe(ialu_reg_reg);
15291 %}
15292
15293
15294 // Conditional Far Branch
15295 // Conditional Far Branch Unsigned
15296 // TODO: fixme
15297
15298 // counted loop end branch near
15299 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15300 %{
15301 match(CountedLoopEnd cmp cr);
15302
15303 effect(USE lbl);
15304
15305 ins_cost(BRANCH_COST);
15306 // short variant.
15307 // ins_short_branch(1);
15308 format %{ "b$cmp $lbl \t// counted loop end" %}
15309
15310 ins_encode(aarch64_enc_br_con(cmp, lbl));
15311
15312 ins_pipe(pipe_branch);
15313 %}
15314
15315 // counted loop end branch far
15316 // TODO: fixme
15317
15318 // ============================================================================
15319 // inlined locking and unlocking
15320
15321 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15322 %{
15323 match(Set cr (FastLock object box));
15324 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15325
15326 ins_cost(5 * INSN_COST);
15327 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15328
15329 ins_encode %{
15330 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15331 %}
15332
15333 ins_pipe(pipe_serial);
15334 %}
15335
15336 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15337 %{
15338 match(Set cr (FastUnlock object box));
15339 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15340
15341 ins_cost(5 * INSN_COST);
15342 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15343
15344 ins_encode %{
15345 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15346 %}
15347
15348 ins_pipe(pipe_serial);
15349 %}
15350
15351 // ============================================================================
15352 // Safepoint Instructions
15353
15354 // TODO
15355 // provide a near and far version of this code
15356
15357 instruct safePoint(rFlagsReg cr, iRegP poll)
15358 %{
15359 match(SafePoint poll);
15360 effect(KILL cr);
15361
15362 format %{
15363 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15364 %}
15365 ins_encode %{
15366 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15367 %}
15368 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15369 %}
15370
15371
15372 // ============================================================================
15373 // Procedure Call/Return Instructions
15374
15375 // Call Java Static Instruction
15376
15377 instruct CallStaticJavaDirect(method meth)
15378 %{
15379 match(CallStaticJava);
15380
15381 effect(USE meth);
15382
15383 ins_cost(CALL_COST);
15384
15385 format %{ "call,static $meth \t// ==> " %}
15386
15387 ins_encode(aarch64_enc_java_static_call(meth),
15388 aarch64_enc_call_epilog);
15389
15390 ins_pipe(pipe_class_call);
15391 %}
15392
15393 // TO HERE
15394
15395 // Call Java Dynamic Instruction
15396 instruct CallDynamicJavaDirect(method meth)
15397 %{
15398 match(CallDynamicJava);
15399
15400 effect(USE meth);
15401
15402 ins_cost(CALL_COST);
15403
15404 format %{ "CALL,dynamic $meth \t// ==> " %}
15405
15406 ins_encode(aarch64_enc_java_dynamic_call(meth),
15407 aarch64_enc_call_epilog);
15408
15409 ins_pipe(pipe_class_call);
15410 %}
15411
15412 // Call Runtime Instruction
15413
15414 instruct CallRuntimeDirect(method meth)
15415 %{
15416 match(CallRuntime);
15417
15418 effect(USE meth);
15419
15420 ins_cost(CALL_COST);
15421
15422 format %{ "CALL, runtime $meth" %}
15423
15424 ins_encode( aarch64_enc_java_to_runtime(meth) );
15425
15426 ins_pipe(pipe_class_call);
15427 %}
15428
15429 // Call Runtime Instruction
15430
15431 instruct CallLeafDirect(method meth)
15432 %{
15433 match(CallLeaf);
15434
15435 effect(USE meth);
15436
15437 ins_cost(CALL_COST);
15438
15439 format %{ "CALL, runtime leaf $meth" %}
15440
15441 ins_encode( aarch64_enc_java_to_runtime(meth) );
15442
15443 ins_pipe(pipe_class_call);
15444 %}
15445
15446 // Call Runtime Instruction without safepoint and with vector arguments
15447 instruct CallLeafDirectVector(method meth)
15448 %{
15449 match(CallLeafVector);
15450
15451 effect(USE meth);
15452
15453 ins_cost(CALL_COST);
15454
15455 format %{ "CALL, runtime leaf vector $meth" %}
15456
15457 ins_encode(aarch64_enc_java_to_runtime(meth));
15458
15459 ins_pipe(pipe_class_call);
15460 %}
15461
15462 // Call Runtime Instruction
15463
15464 // entry point is null, target holds the address to call
15465 instruct CallLeafNoFPIndirect(iRegP target)
15466 %{
15467 predicate(n->as_Call()->entry_point() == nullptr);
15468
15469 match(CallLeafNoFP target);
15470
15471 ins_cost(CALL_COST);
15472
15473 format %{ "CALL, runtime leaf nofp indirect $target" %}
15474
15475 ins_encode %{
15476 __ blr($target$$Register);
15477 %}
15478
15479 ins_pipe(pipe_class_call);
15480 %}
15481
15482 instruct CallLeafNoFPDirect(method meth)
15483 %{
15484 predicate(n->as_Call()->entry_point() != nullptr);
15485
15486 match(CallLeafNoFP);
15487
15488 effect(USE meth);
15489
15490 ins_cost(CALL_COST);
15491
15492 format %{ "CALL, runtime leaf nofp $meth" %}
15493
15494 ins_encode( aarch64_enc_java_to_runtime(meth) );
15495
15496 ins_pipe(pipe_class_call);
15497 %}
15498
15499 // Tail Call; Jump from runtime stub to Java code.
15500 // Also known as an 'interprocedural jump'.
15501 // Target of jump will eventually return to caller.
15502 // TailJump below removes the return address.
15503 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15504 // emitted just above the TailCall which has reset rfp to the caller state.
15505 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15506 %{
15507 match(TailCall jump_target method_ptr);
15508
15509 ins_cost(CALL_COST);
15510
15511 format %{ "br $jump_target\t# $method_ptr holds method" %}
15512
15513 ins_encode(aarch64_enc_tail_call(jump_target));
15514
15515 ins_pipe(pipe_class_call);
15516 %}
15517
15518 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15519 %{
15520 match(TailJump jump_target ex_oop);
15521
15522 ins_cost(CALL_COST);
15523
15524 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15525
15526 ins_encode(aarch64_enc_tail_jmp(jump_target));
15527
15528 ins_pipe(pipe_class_call);
15529 %}
15530
15531 // Forward exception.
15532 instruct ForwardExceptionjmp()
15533 %{
15534 match(ForwardException);
15535 ins_cost(CALL_COST);
15536
15537 format %{ "b forward_exception_stub" %}
15538 ins_encode %{
15539 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15540 %}
15541 ins_pipe(pipe_class_call);
15542 %}
15543
15544 // Create exception oop: created by stack-crawling runtime code.
15545 // Created exception is now available to this handler, and is setup
15546 // just prior to jumping to this handler. No code emitted.
15547 // TODO check
15548 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15549 instruct CreateException(iRegP_R0 ex_oop)
15550 %{
15551 match(Set ex_oop (CreateEx));
15552
15553 format %{ " -- \t// exception oop; no code emitted" %}
15554
15555 size(0);
15556
15557 ins_encode( /*empty*/ );
15558
15559 ins_pipe(pipe_class_empty);
15560 %}
15561
15562 // Rethrow exception: The exception oop will come in the first
15563 // argument position. Then JUMP (not call) to the rethrow stub code.
15564 instruct RethrowException() %{
15565 match(Rethrow);
15566 ins_cost(CALL_COST);
15567
15568 format %{ "b rethrow_stub" %}
15569
15570 ins_encode( aarch64_enc_rethrow() );
15571
15572 ins_pipe(pipe_class_call);
15573 %}
15574
15575
15576 // Return Instruction
15577 // epilog node loads ret address into lr as part of frame pop
15578 instruct Ret()
15579 %{
15580 match(Return);
15581
15582 format %{ "ret\t// return register" %}
15583
15584 ins_encode( aarch64_enc_ret() );
15585
15586 ins_pipe(pipe_branch);
15587 %}
15588
15589 // Die now.
15590 instruct ShouldNotReachHere() %{
15591 match(Halt);
15592
15593 ins_cost(CALL_COST);
15594 format %{ "ShouldNotReachHere" %}
15595
15596 ins_encode %{
15597 if (is_reachable()) {
15598 const char* str = __ code_string(_halt_reason);
15599 __ stop(str);
15600 }
15601 %}
15602
15603 ins_pipe(pipe_class_default);
15604 %}
15605
15606 // ============================================================================
15607 // Partial Subtype Check
15608 //
15609 // superklass array for an instance of the superklass. Set a hidden
15610 // internal cache on a hit (cache is checked with exposed code in
15611 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15612 // encoding ALSO sets flags.
15613
15614 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15615 %{
15616 match(Set result (PartialSubtypeCheck sub super));
15617 predicate(!UseSecondarySupersTable);
15618 effect(KILL cr, KILL temp);
15619
15620 ins_cost(20 * INSN_COST); // slightly larger than the next version
15621 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15622
15623 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15624
15625 opcode(0x1); // Force zero of result reg on hit
15626
15627 ins_pipe(pipe_class_memory);
15628 %}
15629
15630 // Two versions of partialSubtypeCheck, both used when we need to
15631 // search for a super class in the secondary supers array. The first
15632 // is used when we don't know _a priori_ the class being searched
15633 // for. The second, far more common, is used when we do know: this is
15634 // used for instanceof, checkcast, and any case where C2 can determine
15635 // it by constant propagation.
15636
15637 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15638 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15639 rFlagsReg cr)
15640 %{
15641 match(Set result (PartialSubtypeCheck sub super));
15642 predicate(UseSecondarySupersTable);
15643 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15644
15645 ins_cost(10 * INSN_COST); // slightly larger than the next version
15646 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15647
15648 ins_encode %{
15649 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15650 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15651 $vtemp$$FloatRegister,
15652 $result$$Register, /*L_success*/nullptr);
15653 %}
15654
15655 ins_pipe(pipe_class_memory);
15656 %}
15657
15658 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15659 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15660 rFlagsReg cr)
15661 %{
15662 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15663 predicate(UseSecondarySupersTable);
15664 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15665
15666 ins_cost(5 * INSN_COST); // smaller than the next version
15667 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15668
15669 ins_encode %{
15670 bool success = false;
15671 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15672 if (InlineSecondarySupersTest) {
15673 success =
15674 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15675 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15676 $vtemp$$FloatRegister,
15677 $result$$Register,
15678 super_klass_slot);
15679 } else {
15680 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15681 success = (call != nullptr);
15682 }
15683 if (!success) {
15684 ciEnv::current()->record_failure("CodeCache is full");
15685 return;
15686 }
15687 %}
15688
15689 ins_pipe(pipe_class_memory);
15690 %}
15691
15692 // Intrisics for String.compareTo()
15693
15694 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15695 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15696 %{
15697 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15698 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15699 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15700
15701 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15702 ins_encode %{
15703 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15704 __ string_compare($str1$$Register, $str2$$Register,
15705 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15706 $tmp1$$Register, $tmp2$$Register,
15707 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15708 %}
15709 ins_pipe(pipe_class_memory);
15710 %}
15711
15712 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15713 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15714 %{
15715 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15716 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15717 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15718
15719 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15720 ins_encode %{
15721 __ string_compare($str1$$Register, $str2$$Register,
15722 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15723 $tmp1$$Register, $tmp2$$Register,
15724 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15725 %}
15726 ins_pipe(pipe_class_memory);
15727 %}
15728
15729 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15730 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15731 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15732 %{
15733 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15734 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15735 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15736 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15737
15738 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15739 ins_encode %{
15740 __ string_compare($str1$$Register, $str2$$Register,
15741 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15742 $tmp1$$Register, $tmp2$$Register,
15743 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15744 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15745 %}
15746 ins_pipe(pipe_class_memory);
15747 %}
15748
15749 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15750 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15751 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15752 %{
15753 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15754 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15755 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15756 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15757
15758 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15759 ins_encode %{
15760 __ string_compare($str1$$Register, $str2$$Register,
15761 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15762 $tmp1$$Register, $tmp2$$Register,
15763 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15764 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15765 %}
15766 ins_pipe(pipe_class_memory);
15767 %}
15768
15769 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15770 // these string_compare variants as NEON register type for convenience so that the prototype of
15771 // string_compare can be shared with all variants.
15772
15773 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15774 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15775 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15776 pRegGov_P1 pgtmp2, rFlagsReg cr)
15777 %{
15778 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15779 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15780 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15781 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15782
15783 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15784 ins_encode %{
15785 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15786 __ string_compare($str1$$Register, $str2$$Register,
15787 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15788 $tmp1$$Register, $tmp2$$Register,
15789 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15790 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15791 StrIntrinsicNode::LL);
15792 %}
15793 ins_pipe(pipe_class_memory);
15794 %}
15795
15796 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15797 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15798 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15799 pRegGov_P1 pgtmp2, rFlagsReg cr)
15800 %{
15801 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15802 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15803 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15804 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15805
15806 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15807 ins_encode %{
15808 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15809 __ string_compare($str1$$Register, $str2$$Register,
15810 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15811 $tmp1$$Register, $tmp2$$Register,
15812 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15813 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15814 StrIntrinsicNode::LU);
15815 %}
15816 ins_pipe(pipe_class_memory);
15817 %}
15818
15819 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15820 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15821 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15822 pRegGov_P1 pgtmp2, rFlagsReg cr)
15823 %{
15824 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15825 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15826 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15827 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15828
15829 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15830 ins_encode %{
15831 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15832 __ string_compare($str1$$Register, $str2$$Register,
15833 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15834 $tmp1$$Register, $tmp2$$Register,
15835 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15836 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15837 StrIntrinsicNode::UL);
15838 %}
15839 ins_pipe(pipe_class_memory);
15840 %}
15841
15842 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15843 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15844 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15845 pRegGov_P1 pgtmp2, rFlagsReg cr)
15846 %{
15847 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15848 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15849 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15850 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15851
15852 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15853 ins_encode %{
15854 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15855 __ string_compare($str1$$Register, $str2$$Register,
15856 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15857 $tmp1$$Register, $tmp2$$Register,
15858 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15859 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15860 StrIntrinsicNode::UU);
15861 %}
15862 ins_pipe(pipe_class_memory);
15863 %}
15864
15865 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15866 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15867 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15868 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15869 %{
15870 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15871 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15872 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15873 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15874 TEMP vtmp0, TEMP vtmp1, KILL cr);
15875 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15876 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15877
15878 ins_encode %{
15879 __ string_indexof($str1$$Register, $str2$$Register,
15880 $cnt1$$Register, $cnt2$$Register,
15881 $tmp1$$Register, $tmp2$$Register,
15882 $tmp3$$Register, $tmp4$$Register,
15883 $tmp5$$Register, $tmp6$$Register,
15884 -1, $result$$Register, StrIntrinsicNode::UU);
15885 %}
15886 ins_pipe(pipe_class_memory);
15887 %}
15888
15889 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15890 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15891 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15892 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15893 %{
15894 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15895 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15896 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15897 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15898 TEMP vtmp0, TEMP vtmp1, KILL cr);
15899 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15900 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15901
15902 ins_encode %{
15903 __ string_indexof($str1$$Register, $str2$$Register,
15904 $cnt1$$Register, $cnt2$$Register,
15905 $tmp1$$Register, $tmp2$$Register,
15906 $tmp3$$Register, $tmp4$$Register,
15907 $tmp5$$Register, $tmp6$$Register,
15908 -1, $result$$Register, StrIntrinsicNode::LL);
15909 %}
15910 ins_pipe(pipe_class_memory);
15911 %}
15912
15913 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15914 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15915 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15916 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15917 %{
15918 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15919 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15920 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15921 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15922 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15923 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15924 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15925
15926 ins_encode %{
15927 __ string_indexof($str1$$Register, $str2$$Register,
15928 $cnt1$$Register, $cnt2$$Register,
15929 $tmp1$$Register, $tmp2$$Register,
15930 $tmp3$$Register, $tmp4$$Register,
15931 $tmp5$$Register, $tmp6$$Register,
15932 -1, $result$$Register, StrIntrinsicNode::UL);
15933 %}
15934 ins_pipe(pipe_class_memory);
15935 %}
15936
15937 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15938 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15939 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15940 %{
15941 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15942 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15943 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15944 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15945 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15946 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15947
15948 ins_encode %{
15949 int icnt2 = (int)$int_cnt2$$constant;
15950 __ string_indexof($str1$$Register, $str2$$Register,
15951 $cnt1$$Register, zr,
15952 $tmp1$$Register, $tmp2$$Register,
15953 $tmp3$$Register, $tmp4$$Register, zr, zr,
15954 icnt2, $result$$Register, StrIntrinsicNode::UU);
15955 %}
15956 ins_pipe(pipe_class_memory);
15957 %}
15958
15959 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15960 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15961 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15962 %{
15963 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15964 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15965 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15966 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15967 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15968 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15969
15970 ins_encode %{
15971 int icnt2 = (int)$int_cnt2$$constant;
15972 __ string_indexof($str1$$Register, $str2$$Register,
15973 $cnt1$$Register, zr,
15974 $tmp1$$Register, $tmp2$$Register,
15975 $tmp3$$Register, $tmp4$$Register, zr, zr,
15976 icnt2, $result$$Register, StrIntrinsicNode::LL);
15977 %}
15978 ins_pipe(pipe_class_memory);
15979 %}
15980
15981 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15982 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15983 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15984 %{
15985 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15986 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15987 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15988 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15989 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15990 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15991
15992 ins_encode %{
15993 int icnt2 = (int)$int_cnt2$$constant;
15994 __ string_indexof($str1$$Register, $str2$$Register,
15995 $cnt1$$Register, zr,
15996 $tmp1$$Register, $tmp2$$Register,
15997 $tmp3$$Register, $tmp4$$Register, zr, zr,
15998 icnt2, $result$$Register, StrIntrinsicNode::UL);
15999 %}
16000 ins_pipe(pipe_class_memory);
16001 %}
16002
16003 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16004 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16005 iRegINoSp tmp3, rFlagsReg cr)
16006 %{
16007 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16008 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16009 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16010 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16011
16012 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16013
16014 ins_encode %{
16015 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16016 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16017 $tmp3$$Register);
16018 %}
16019 ins_pipe(pipe_class_memory);
16020 %}
16021
16022 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16023 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16024 iRegINoSp tmp3, rFlagsReg cr)
16025 %{
16026 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16027 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16028 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16029 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16030
16031 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16032
16033 ins_encode %{
16034 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16035 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16036 $tmp3$$Register);
16037 %}
16038 ins_pipe(pipe_class_memory);
16039 %}
16040
16041 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16042 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16043 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16044 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16045 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16046 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16047 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16048 ins_encode %{
16049 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16050 $result$$Register, $ztmp1$$FloatRegister,
16051 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16052 $ptmp$$PRegister, true /* isL */);
16053 %}
16054 ins_pipe(pipe_class_memory);
16055 %}
16056
16057 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16058 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16059 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16060 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16061 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16062 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16063 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16064 ins_encode %{
16065 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16066 $result$$Register, $ztmp1$$FloatRegister,
16067 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16068 $ptmp$$PRegister, false /* isL */);
16069 %}
16070 ins_pipe(pipe_class_memory);
16071 %}
16072
16073 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16074 iRegI_R0 result, rFlagsReg cr)
16075 %{
16076 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16077 match(Set result (StrEquals (Binary str1 str2) cnt));
16078 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16079
16080 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16081 ins_encode %{
16082 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16083 __ string_equals($str1$$Register, $str2$$Register,
16084 $result$$Register, $cnt$$Register);
16085 %}
16086 ins_pipe(pipe_class_memory);
16087 %}
16088
16089 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16090 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16091 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16092 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16093 iRegP_R10 tmp, rFlagsReg cr)
16094 %{
16095 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16096 match(Set result (AryEq ary1 ary2));
16097 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16098 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16099 TEMP vtmp6, TEMP vtmp7, KILL cr);
16100
16101 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16102 ins_encode %{
16103 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16104 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16105 $result$$Register, $tmp$$Register, 1);
16106 if (tpc == nullptr) {
16107 ciEnv::current()->record_failure("CodeCache is full");
16108 return;
16109 }
16110 %}
16111 ins_pipe(pipe_class_memory);
16112 %}
16113
16114 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16115 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16116 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16117 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16118 iRegP_R10 tmp, rFlagsReg cr)
16119 %{
16120 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16121 match(Set result (AryEq ary1 ary2));
16122 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16123 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16124 TEMP vtmp6, TEMP vtmp7, KILL cr);
16125
16126 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16127 ins_encode %{
16128 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16129 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16130 $result$$Register, $tmp$$Register, 2);
16131 if (tpc == nullptr) {
16132 ciEnv::current()->record_failure("CodeCache is full");
16133 return;
16134 }
16135 %}
16136 ins_pipe(pipe_class_memory);
16137 %}
16138
16139 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16140 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16141 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16142 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16143 %{
16144 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16145 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16146 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16147
16148 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16149 ins_encode %{
16150 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16151 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16152 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16153 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16154 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16155 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16156 (BasicType)$basic_type$$constant);
16157 if (tpc == nullptr) {
16158 ciEnv::current()->record_failure("CodeCache is full");
16159 return;
16160 }
16161 %}
16162 ins_pipe(pipe_class_memory);
16163 %}
16164
16165 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16166 %{
16167 match(Set result (CountPositives ary1 len));
16168 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16169 format %{ "count positives byte[] $ary1,$len -> $result" %}
16170 ins_encode %{
16171 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16172 if (tpc == nullptr) {
16173 ciEnv::current()->record_failure("CodeCache is full");
16174 return;
16175 }
16176 %}
16177 ins_pipe( pipe_slow );
16178 %}
16179
16180 // fast char[] to byte[] compression
16181 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16182 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16183 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16184 iRegI_R0 result, rFlagsReg cr)
16185 %{
16186 match(Set result (StrCompressedCopy src (Binary dst len)));
16187 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16188 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16189
16190 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16191 ins_encode %{
16192 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16193 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16194 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16195 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16196 %}
16197 ins_pipe(pipe_slow);
16198 %}
16199
16200 // fast byte[] to char[] inflation
16201 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16202 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16203 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16204 %{
16205 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16206 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16207 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16208 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16209
16210 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16211 ins_encode %{
16212 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16213 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16214 $vtmp2$$FloatRegister, $tmp$$Register);
16215 if (tpc == nullptr) {
16216 ciEnv::current()->record_failure("CodeCache is full");
16217 return;
16218 }
16219 %}
16220 ins_pipe(pipe_class_memory);
16221 %}
16222
16223 // encode char[] to byte[] in ISO_8859_1
16224 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16225 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16226 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16227 iRegI_R0 result, rFlagsReg cr)
16228 %{
16229 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16230 match(Set result (EncodeISOArray src (Binary dst len)));
16231 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16232 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16233
16234 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16235 ins_encode %{
16236 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16237 $result$$Register, false,
16238 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16239 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16240 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16241 %}
16242 ins_pipe(pipe_class_memory);
16243 %}
16244
16245 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16246 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16247 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16248 iRegI_R0 result, rFlagsReg cr)
16249 %{
16250 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16251 match(Set result (EncodeISOArray src (Binary dst len)));
16252 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16253 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16254
16255 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16256 ins_encode %{
16257 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16258 $result$$Register, true,
16259 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16260 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16261 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16262 %}
16263 ins_pipe(pipe_class_memory);
16264 %}
16265
16266 //----------------------------- CompressBits/ExpandBits ------------------------
16267
16268 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16269 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16270 match(Set dst (CompressBits src mask));
16271 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16272 format %{ "mov $tsrc, $src\n\t"
16273 "mov $tmask, $mask\n\t"
16274 "bext $tdst, $tsrc, $tmask\n\t"
16275 "mov $dst, $tdst"
16276 %}
16277 ins_encode %{
16278 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16279 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16280 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16281 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16282 %}
16283 ins_pipe(pipe_slow);
16284 %}
16285
16286 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16287 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16288 match(Set dst (CompressBits (LoadI mem) mask));
16289 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16290 format %{ "ldrs $tsrc, $mem\n\t"
16291 "ldrs $tmask, $mask\n\t"
16292 "bext $tdst, $tsrc, $tmask\n\t"
16293 "mov $dst, $tdst"
16294 %}
16295 ins_encode %{
16296 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16297 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16298 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16299 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16300 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16301 %}
16302 ins_pipe(pipe_slow);
16303 %}
16304
16305 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16306 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16307 match(Set dst (CompressBits src mask));
16308 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16309 format %{ "mov $tsrc, $src\n\t"
16310 "mov $tmask, $mask\n\t"
16311 "bext $tdst, $tsrc, $tmask\n\t"
16312 "mov $dst, $tdst"
16313 %}
16314 ins_encode %{
16315 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16316 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16317 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16318 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16319 %}
16320 ins_pipe(pipe_slow);
16321 %}
16322
16323 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16324 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16325 match(Set dst (CompressBits (LoadL mem) mask));
16326 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16327 format %{ "ldrd $tsrc, $mem\n\t"
16328 "ldrd $tmask, $mask\n\t"
16329 "bext $tdst, $tsrc, $tmask\n\t"
16330 "mov $dst, $tdst"
16331 %}
16332 ins_encode %{
16333 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16334 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16335 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16336 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16337 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16338 %}
16339 ins_pipe(pipe_slow);
16340 %}
16341
16342 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16343 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16344 match(Set dst (ExpandBits src mask));
16345 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16346 format %{ "mov $tsrc, $src\n\t"
16347 "mov $tmask, $mask\n\t"
16348 "bdep $tdst, $tsrc, $tmask\n\t"
16349 "mov $dst, $tdst"
16350 %}
16351 ins_encode %{
16352 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16353 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16354 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16355 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16356 %}
16357 ins_pipe(pipe_slow);
16358 %}
16359
16360 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16361 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16362 match(Set dst (ExpandBits (LoadI mem) mask));
16363 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16364 format %{ "ldrs $tsrc, $mem\n\t"
16365 "ldrs $tmask, $mask\n\t"
16366 "bdep $tdst, $tsrc, $tmask\n\t"
16367 "mov $dst, $tdst"
16368 %}
16369 ins_encode %{
16370 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16371 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16372 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16373 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16374 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16375 %}
16376 ins_pipe(pipe_slow);
16377 %}
16378
16379 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16380 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16381 match(Set dst (ExpandBits src mask));
16382 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16383 format %{ "mov $tsrc, $src\n\t"
16384 "mov $tmask, $mask\n\t"
16385 "bdep $tdst, $tsrc, $tmask\n\t"
16386 "mov $dst, $tdst"
16387 %}
16388 ins_encode %{
16389 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16390 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16391 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16392 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16393 %}
16394 ins_pipe(pipe_slow);
16395 %}
16396
16397
16398 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16399 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16400 match(Set dst (ExpandBits (LoadL mem) mask));
16401 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16402 format %{ "ldrd $tsrc, $mem\n\t"
16403 "ldrd $tmask, $mask\n\t"
16404 "bdep $tdst, $tsrc, $tmask\n\t"
16405 "mov $dst, $tdst"
16406 %}
16407 ins_encode %{
16408 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16409 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16410 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16411 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16412 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16413 %}
16414 ins_pipe(pipe_slow);
16415 %}
16416
16417 //----------------------------- Reinterpret ----------------------------------
16418 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16419 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16420 match(Set dst (ReinterpretHF2S src));
16421 format %{ "reinterpretHF2S $dst, $src" %}
16422 ins_encode %{
16423 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16424 %}
16425 ins_pipe(pipe_slow);
16426 %}
16427
16428 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16429 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16430 match(Set dst (ReinterpretS2HF src));
16431 format %{ "reinterpretS2HF $dst, $src" %}
16432 ins_encode %{
16433 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16434 %}
16435 ins_pipe(pipe_slow);
16436 %}
16437
16438 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16439 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16440 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16441 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16442 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16443 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16444 // can be omitted in this pattern, resulting in -
16445 // fcvt $dst, $src // Convert float to half-precision float
16446 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16447 %{
16448 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16449 format %{ "convF2HFAndS2HF $dst, $src" %}
16450 ins_encode %{
16451 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16452 %}
16453 ins_pipe(pipe_slow);
16454 %}
16455
16456 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16457 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16458 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16459 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16460 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16461 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16462 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16463 // resulting in -
16464 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16465 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16466 %{
16467 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16468 format %{ "convHF2SAndHF2F $dst, $src" %}
16469 ins_encode %{
16470 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16471 %}
16472 ins_pipe(pipe_slow);
16473 %}
16474
16475 // ============================================================================
16476 // This name is KNOWN by the ADLC and cannot be changed.
16477 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16478 // for this guy.
16479 instruct tlsLoadP(thread_RegP dst)
16480 %{
16481 match(Set dst (ThreadLocal));
16482
16483 ins_cost(0);
16484
16485 format %{ " -- \t// $dst=Thread::current(), empty" %}
16486
16487 size(0);
16488
16489 ins_encode( /*empty*/ );
16490
16491 ins_pipe(pipe_class_empty);
16492 %}
16493
16494 //----------PEEPHOLE RULES-----------------------------------------------------
16495 // These must follow all instruction definitions as they use the names
16496 // defined in the instructions definitions.
16497 //
16498 // peepmatch ( root_instr_name [preceding_instruction]* );
16499 //
16500 // peepconstraint %{
16501 // (instruction_number.operand_name relational_op instruction_number.operand_name
16502 // [, ...] );
16503 // // instruction numbers are zero-based using left to right order in peepmatch
16504 //
16505 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16506 // // provide an instruction_number.operand_name for each operand that appears
16507 // // in the replacement instruction's match rule
16508 //
16509 // ---------VM FLAGS---------------------------------------------------------
16510 //
16511 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16512 //
16513 // Each peephole rule is given an identifying number starting with zero and
16514 // increasing by one in the order seen by the parser. An individual peephole
16515 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16516 // on the command-line.
16517 //
16518 // ---------CURRENT LIMITATIONS----------------------------------------------
16519 //
16520 // Only match adjacent instructions in same basic block
16521 // Only equality constraints
16522 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16523 // Only one replacement instruction
16524 //
16525 // ---------EXAMPLE----------------------------------------------------------
16526 //
16527 // // pertinent parts of existing instructions in architecture description
16528 // instruct movI(iRegINoSp dst, iRegI src)
16529 // %{
16530 // match(Set dst (CopyI src));
16531 // %}
16532 //
16533 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16534 // %{
16535 // match(Set dst (AddI dst src));
16536 // effect(KILL cr);
16537 // %}
16538 //
16539 // // Change (inc mov) to lea
16540 // peephole %{
16541 // // increment preceded by register-register move
16542 // peepmatch ( incI_iReg movI );
16543 // // require that the destination register of the increment
16544 // // match the destination register of the move
16545 // peepconstraint ( 0.dst == 1.dst );
16546 // // construct a replacement instruction that sets
16547 // // the destination to ( move's source register + one )
16548 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16549 // %}
16550 //
16551
16552 // Implementation no longer uses movX instructions since
16553 // machine-independent system no longer uses CopyX nodes.
16554 //
16555 // peephole
16556 // %{
16557 // peepmatch (incI_iReg movI);
16558 // peepconstraint (0.dst == 1.dst);
16559 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16560 // %}
16561
16562 // peephole
16563 // %{
16564 // peepmatch (decI_iReg movI);
16565 // peepconstraint (0.dst == 1.dst);
16566 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16567 // %}
16568
16569 // peephole
16570 // %{
16571 // peepmatch (addI_iReg_imm movI);
16572 // peepconstraint (0.dst == 1.dst);
16573 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16574 // %}
16575
16576 // peephole
16577 // %{
16578 // peepmatch (incL_iReg movL);
16579 // peepconstraint (0.dst == 1.dst);
16580 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16581 // %}
16582
16583 // peephole
16584 // %{
16585 // peepmatch (decL_iReg movL);
16586 // peepconstraint (0.dst == 1.dst);
16587 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16588 // %}
16589
16590 // peephole
16591 // %{
16592 // peepmatch (addL_iReg_imm movL);
16593 // peepconstraint (0.dst == 1.dst);
16594 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16595 // %}
16596
16597 // peephole
16598 // %{
16599 // peepmatch (addP_iReg_imm movP);
16600 // peepconstraint (0.dst == 1.dst);
16601 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16602 // %}
16603
16604 // // Change load of spilled value to only a spill
16605 // instruct storeI(memory mem, iRegI src)
16606 // %{
16607 // match(Set mem (StoreI mem src));
16608 // %}
16609 //
16610 // instruct loadI(iRegINoSp dst, memory mem)
16611 // %{
16612 // match(Set dst (LoadI mem));
16613 // %}
16614 //
16615
16616 //----------SMARTSPILL RULES---------------------------------------------------
16617 // These must follow all instruction definitions as they use the names
16618 // defined in the instructions definitions.
16619
16620 // Local Variables:
16621 // mode: c++
16622 // End: