1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BootTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else if (_entry_point == nullptr) {
1694 // See CallLeafNoFPIndirect
1695 return 1 * NativeInstruction::instruction_size;
1696 } else {
1697 return 6 * NativeInstruction::instruction_size;
1698 }
1699 }
1700
1701 //=============================================================================
1702
1703 #ifndef PRODUCT
1704 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1705 st->print("BREAKPOINT");
1706 }
1707 #endif
1708
1709 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1710 __ brk(0);
1711 }
1712
1713 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1714 return MachNode::size(ra_);
1715 }
1716
1717 //=============================================================================
1718
1719 #ifndef PRODUCT
1720 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1721 st->print("nop \t# %d bytes pad for loops and calls", _count);
1722 }
1723 #endif
1724
1725 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1726 for (int i = 0; i < _count; i++) {
1727 __ nop();
1728 }
1729 }
1730
1731 uint MachNopNode::size(PhaseRegAlloc*) const {
1732 return _count * NativeInstruction::instruction_size;
1733 }
1734
1735 //=============================================================================
1736 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1737
1738 int ConstantTable::calculate_table_base_offset() const {
1739 return 0; // absolute addressing, no offset
1740 }
1741
1742 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1743 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1744 ShouldNotReachHere();
1745 }
1746
1747 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1748 // Empty encoding
1749 }
1750
1751 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1752 return 0;
1753 }
1754
1755 #ifndef PRODUCT
1756 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1757 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1758 }
1759 #endif
1760
1761 #ifndef PRODUCT
1762 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1763 Compile* C = ra_->C;
1764
1765 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1766
1767 if (C->output()->need_stack_bang(framesize))
1768 st->print("# stack bang size=%d\n\t", framesize);
1769
1770 if (VM_Version::use_rop_protection()) {
1771 st->print("ldr zr, [lr]\n\t");
1772 st->print("paciaz\n\t");
1773 }
1774 if (framesize < ((1 << 9) + 2 * wordSize)) {
1775 st->print("sub sp, sp, #%d\n\t", framesize);
1776 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1777 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1778 } else {
1779 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1780 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1781 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1782 st->print("sub sp, sp, rscratch1");
1783 }
1784 if (C->stub_function() == nullptr) {
1785 st->print("\n\t");
1786 st->print("ldr rscratch1, [guard]\n\t");
1787 st->print("dmb ishld\n\t");
1788 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1789 st->print("cmp rscratch1, rscratch2\n\t");
1790 st->print("b.eq skip");
1791 st->print("\n\t");
1792 st->print("blr #nmethod_entry_barrier_stub\n\t");
1793 st->print("b skip\n\t");
1794 st->print("guard: int\n\t");
1795 st->print("\n\t");
1796 st->print("skip:\n\t");
1797 }
1798 }
1799 #endif
1800
1801 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1802 Compile* C = ra_->C;
1803
1804
1805 __ verified_entry(C, 0);
1806
1807 if (C->stub_function() == nullptr) {
1808 __ entry_barrier();
1809 }
1810
1811 if (!Compile::current()->output()->in_scratch_emit_size()) {
1812 __ bind(*_verified_entry);
1813 }
1814
1815 if (VerifyStackAtCalls) {
1816 Unimplemented();
1817 }
1818
1819 C->output()->set_frame_complete(__ offset());
1820
1821 if (C->has_mach_constant_base_node()) {
1822 // NOTE: We set the table base offset here because users might be
1823 // emitted before MachConstantBaseNode.
1824 ConstantTable& constant_table = C->output()->constant_table();
1825 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1826 }
1827 }
1828
1829 int MachPrologNode::reloc() const
1830 {
1831 return 0;
1832 }
1833
1834 //=============================================================================
1835
1836 #ifndef PRODUCT
1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1838 Compile* C = ra_->C;
1839 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1840
1841 st->print("# pop frame %d\n\t",framesize);
1842
1843 if (framesize == 0) {
1844 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1845 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1846 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1847 st->print("add sp, sp, #%d\n\t", framesize);
1848 } else {
1849 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1850 st->print("add sp, sp, rscratch1\n\t");
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 }
1853 if (VM_Version::use_rop_protection()) {
1854 st->print("autiaz\n\t");
1855 st->print("ldr zr, [lr]\n\t");
1856 }
1857
1858 if (do_polling() && C->is_method_compilation()) {
1859 st->print("# test polling word\n\t");
1860 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1861 st->print("cmp sp, rscratch1\n\t");
1862 st->print("bhi #slow_path");
1863 }
1864 }
1865 #endif
1866
1867 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1868 Compile* C = ra_->C;
1869 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1870
1871 __ remove_frame(framesize, C->needs_stack_repair());
1872
1873 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1874 __ reserved_stack_check();
1875 }
1876
1877 if (do_polling() && C->is_method_compilation()) {
1878 Label dummy_label;
1879 Label* code_stub = &dummy_label;
1880 if (!C->output()->in_scratch_emit_size()) {
1881 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1882 C->output()->add_stub(stub);
1883 code_stub = &stub->entry();
1884 }
1885 __ relocate(relocInfo::poll_return_type);
1886 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1887 }
1888 }
1889
1890 int MachEpilogNode::reloc() const {
1891 // Return number of relocatable values contained in this instruction.
1892 return 1; // 1 for polling page.
1893 }
1894
1895 const Pipeline * MachEpilogNode::pipeline() const {
1896 return MachNode::pipeline_class();
1897 }
1898
1899 //=============================================================================
1900
1901 static enum RC rc_class(OptoReg::Name reg) {
1902
1903 if (reg == OptoReg::Bad) {
1904 return rc_bad;
1905 }
1906
1907 // we have 32 int registers * 2 halves
1908 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1909
1910 if (reg < slots_of_int_registers) {
1911 return rc_int;
1912 }
1913
1914 // we have 32 float register * 8 halves
1915 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1916 if (reg < slots_of_int_registers + slots_of_float_registers) {
1917 return rc_float;
1918 }
1919
1920 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1921 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1922 return rc_predicate;
1923 }
1924
1925 // Between predicate regs & stack is the flags.
1926 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1927
1928 return rc_stack;
1929 }
1930
1931 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1932 Compile* C = ra_->C;
1933
1934 // Get registers to move.
1935 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1936 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1937 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1938 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1939
1940 enum RC src_hi_rc = rc_class(src_hi);
1941 enum RC src_lo_rc = rc_class(src_lo);
1942 enum RC dst_hi_rc = rc_class(dst_hi);
1943 enum RC dst_lo_rc = rc_class(dst_lo);
1944
1945 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1946
1947 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1948 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1949 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1950 "expected aligned-adjacent pairs");
1951 }
1952
1953 if (src_lo == dst_lo && src_hi == dst_hi) {
1954 return 0; // Self copy, no move.
1955 }
1956
1957 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1958 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1959 int src_offset = ra_->reg2offset(src_lo);
1960 int dst_offset = ra_->reg2offset(dst_lo);
1961
1962 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1963 uint ireg = ideal_reg();
1964 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
1965 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
1966 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
1967 if (ireg == Op_VecA && masm) {
1968 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1969 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1970 // stack->stack
1971 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1972 sve_vector_reg_size_in_bytes);
1973 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1974 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1975 sve_vector_reg_size_in_bytes);
1976 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1977 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1978 sve_vector_reg_size_in_bytes);
1979 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1980 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1981 as_FloatRegister(Matcher::_regEncode[src_lo]),
1982 as_FloatRegister(Matcher::_regEncode[src_lo]));
1983 } else {
1984 ShouldNotReachHere();
1985 }
1986 } else if (masm) {
1987 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1988 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1989 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1990 // stack->stack
1991 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1992 if (ireg == Op_VecD) {
1993 __ unspill(rscratch1, true, src_offset);
1994 __ spill(rscratch1, true, dst_offset);
1995 } else {
1996 __ spill_copy128(src_offset, dst_offset);
1997 }
1998 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1999 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2000 ireg == Op_VecD ? __ T8B : __ T16B,
2001 as_FloatRegister(Matcher::_regEncode[src_lo]));
2002 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2003 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2004 ireg == Op_VecD ? __ D : __ Q,
2005 ra_->reg2offset(dst_lo));
2006 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2007 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2008 ireg == Op_VecD ? __ D : __ Q,
2009 ra_->reg2offset(src_lo));
2010 } else {
2011 ShouldNotReachHere();
2012 }
2013 }
2014 } else if (masm) {
2015 switch (src_lo_rc) {
2016 case rc_int:
2017 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2018 if (is64) {
2019 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2020 as_Register(Matcher::_regEncode[src_lo]));
2021 } else {
2022 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2023 as_Register(Matcher::_regEncode[src_lo]));
2024 }
2025 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2026 if (is64) {
2027 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2028 as_Register(Matcher::_regEncode[src_lo]));
2029 } else {
2030 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2031 as_Register(Matcher::_regEncode[src_lo]));
2032 }
2033 } else { // gpr --> stack spill
2034 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2035 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2036 }
2037 break;
2038 case rc_float:
2039 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2040 if (is64) {
2041 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2042 as_FloatRegister(Matcher::_regEncode[src_lo]));
2043 } else {
2044 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2045 as_FloatRegister(Matcher::_regEncode[src_lo]));
2046 }
2047 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2048 if (is64) {
2049 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 as_FloatRegister(Matcher::_regEncode[src_lo]));
2051 } else {
2052 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2053 as_FloatRegister(Matcher::_regEncode[src_lo]));
2054 }
2055 } else { // fpr --> stack spill
2056 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2057 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2058 is64 ? __ D : __ S, dst_offset);
2059 }
2060 break;
2061 case rc_stack:
2062 if (dst_lo_rc == rc_int) { // stack --> gpr load
2063 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2064 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2065 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2066 is64 ? __ D : __ S, src_offset);
2067 } else if (dst_lo_rc == rc_predicate) {
2068 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2069 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2070 } else { // stack --> stack copy
2071 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2072 if (ideal_reg() == Op_RegVectMask) {
2073 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2074 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2075 } else {
2076 __ unspill(rscratch1, is64, src_offset);
2077 __ spill(rscratch1, is64, dst_offset);
2078 }
2079 }
2080 break;
2081 case rc_predicate:
2082 if (dst_lo_rc == rc_predicate) {
2083 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2084 } else if (dst_lo_rc == rc_stack) {
2085 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2086 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2087 } else {
2088 assert(false, "bad src and dst rc_class combination.");
2089 ShouldNotReachHere();
2090 }
2091 break;
2092 default:
2093 assert(false, "bad rc_class for spill");
2094 ShouldNotReachHere();
2095 }
2096 }
2097
2098 if (st) {
2099 st->print("spill ");
2100 if (src_lo_rc == rc_stack) {
2101 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2102 } else {
2103 st->print("%s -> ", Matcher::regName[src_lo]);
2104 }
2105 if (dst_lo_rc == rc_stack) {
2106 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2107 } else {
2108 st->print("%s", Matcher::regName[dst_lo]);
2109 }
2110 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2111 int vsize = 0;
2112 switch (ideal_reg()) {
2113 case Op_VecD:
2114 vsize = 64;
2115 break;
2116 case Op_VecX:
2117 vsize = 128;
2118 break;
2119 case Op_VecA:
2120 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2121 break;
2122 default:
2123 assert(false, "bad register type for spill");
2124 ShouldNotReachHere();
2125 }
2126 st->print("\t# vector spill size = %d", vsize);
2127 } else if (ideal_reg() == Op_RegVectMask) {
2128 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2129 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2130 st->print("\t# predicate spill size = %d", vsize);
2131 } else {
2132 st->print("\t# spill size = %d", is64 ? 64 : 32);
2133 }
2134 }
2135
2136 return 0;
2137
2138 }
2139
2140 #ifndef PRODUCT
2141 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2142 if (!ra_)
2143 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2144 else
2145 implementation(nullptr, ra_, false, st);
2146 }
2147 #endif
2148
2149 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2150 implementation(masm, ra_, false, nullptr);
2151 }
2152
2153 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2154 return MachNode::size(ra_);
2155 }
2156
2157 //=============================================================================
2158
2159 #ifndef PRODUCT
2160 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2161 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2162 int reg = ra_->get_reg_first(this);
2163 st->print("add %s, rsp, #%d]\t# box lock",
2164 Matcher::regName[reg], offset);
2165 }
2166 #endif
2167
2168 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2169 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2170 int reg = ra_->get_encode(this);
2171
2172 // This add will handle any 24-bit signed offset. 24 bits allows an
2173 // 8 megabyte stack frame.
2174 __ add(as_Register(reg), sp, offset);
2175 }
2176
2177 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2178 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2179 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2180
2181 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2182 return NativeInstruction::instruction_size;
2183 } else {
2184 return 2 * NativeInstruction::instruction_size;
2185 }
2186 }
2187
2188 ///=============================================================================
2189 #ifndef PRODUCT
2190 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2191 {
2192 st->print_cr("# MachVEPNode");
2193 if (!_verified) {
2194 st->print_cr("\t load_class");
2195 } else {
2196 st->print_cr("\t unpack_inline_arg");
2197 }
2198 }
2199 #endif
2200
2201 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2202 {
2203 if (!_verified) {
2204 __ ic_check(1);
2205 } else {
2206 // TODO 8284443 Avoid creation of temporary frame
2207 if (ra_->C->stub_function() == nullptr) {
2208 __ verified_entry(ra_->C, 0);
2209 __ entry_barrier();
2210 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2211 __ remove_frame(framesize, false);
2212 }
2213 // Unpack inline type args passed as oop and then jump to
2214 // the verified entry point (skipping the unverified entry).
2215 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2216 // Emit code for verified entry and save increment for stack repair on return
2217 __ verified_entry(ra_->C, sp_inc);
2218 if (Compile::current()->output()->in_scratch_emit_size()) {
2219 Label dummy_verified_entry;
2220 __ b(dummy_verified_entry);
2221 } else {
2222 __ b(*_verified_entry);
2223 }
2224 }
2225 }
2226
2227 //=============================================================================
2228 #ifndef PRODUCT
2229 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2230 {
2231 st->print_cr("# MachUEPNode");
2232 if (UseCompressedClassPointers) {
2233 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2234 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2235 st->print_cr("\tcmpw rscratch1, r10");
2236 } else {
2237 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2238 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2239 st->print_cr("\tcmp rscratch1, r10");
2240 }
2241 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2242 }
2243 #endif
2244
2245 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2246 {
2247 __ ic_check(InteriorEntryAlignment);
2248 }
2249
2250 // REQUIRED EMIT CODE
2251
2252 //=============================================================================
2253
2254 // Emit deopt handler code.
2255 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2256 {
2257 // Note that the code buffer's insts_mark is always relative to insts.
2258 // That's why we must use the macroassembler to generate a handler.
2259 address base = __ start_a_stub(size_deopt_handler());
2260 if (base == nullptr) {
2261 ciEnv::current()->record_failure("CodeCache is full");
2262 return 0; // CodeBuffer::expand failed
2263 }
2264
2265 int offset = __ offset();
2266 Label start;
2267 __ bind(start);
2268 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2269
2270 int entry_offset = __ offset();
2271 __ b(start);
2272
2273 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2274 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2275 "out of bounds read in post-call NOP check");
2276 __ end_a_stub();
2277 return entry_offset;
2278 }
2279
2280 // REQUIRED MATCHER CODE
2281
2282 //=============================================================================
2283
2284 bool Matcher::match_rule_supported(int opcode) {
2285 if (!has_match_rule(opcode))
2286 return false;
2287
2288 switch (opcode) {
2289 case Op_OnSpinWait:
2290 return VM_Version::supports_on_spin_wait();
2291 case Op_CacheWB:
2292 case Op_CacheWBPreSync:
2293 case Op_CacheWBPostSync:
2294 if (!VM_Version::supports_data_cache_line_flush()) {
2295 return false;
2296 }
2297 break;
2298 case Op_ExpandBits:
2299 case Op_CompressBits:
2300 if (!VM_Version::supports_svebitperm()) {
2301 return false;
2302 }
2303 break;
2304 case Op_FmaF:
2305 case Op_FmaD:
2306 case Op_FmaVF:
2307 case Op_FmaVD:
2308 if (!UseFMA) {
2309 return false;
2310 }
2311 break;
2312 case Op_FmaHF:
2313 // UseFMA flag also needs to be checked along with FEAT_FP16
2314 if (!UseFMA || !is_feat_fp16_supported()) {
2315 return false;
2316 }
2317 break;
2318 case Op_AddHF:
2319 case Op_SubHF:
2320 case Op_MulHF:
2321 case Op_DivHF:
2322 case Op_MinHF:
2323 case Op_MaxHF:
2324 case Op_SqrtHF:
2325 // Half-precision floating point scalar operations require FEAT_FP16
2326 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2327 // features are supported.
2328 if (!is_feat_fp16_supported()) {
2329 return false;
2330 }
2331 break;
2332 }
2333
2334 return true; // Per default match rules are supported.
2335 }
2336
2337 const RegMask* Matcher::predicate_reg_mask(void) {
2338 return &_PR_REG_mask;
2339 }
2340
2341 bool Matcher::supports_vector_calling_convention(void) {
2342 return EnableVectorSupport;
2343 }
2344
2345 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2346 assert(EnableVectorSupport, "sanity");
2347 int lo = V0_num;
2348 int hi = V0_H_num;
2349 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2350 hi = V0_K_num;
2351 }
2352 return OptoRegPair(hi, lo);
2353 }
2354
2355 // Is this branch offset short enough that a short branch can be used?
2356 //
2357 // NOTE: If the platform does not provide any short branch variants, then
2358 // this method should return false for offset 0.
2359 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2360 // The passed offset is relative to address of the branch.
2361
2362 return (-32768 <= offset && offset < 32768);
2363 }
2364
2365 // Vector width in bytes.
2366 int Matcher::vector_width_in_bytes(BasicType bt) {
2367 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2368 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2369 // Minimum 2 values in vector
2370 if (size < 2*type2aelembytes(bt)) size = 0;
2371 // But never < 4
2372 if (size < 4) size = 0;
2373 return size;
2374 }
2375
2376 // Limits on vector size (number of elements) loaded into vector.
2377 int Matcher::max_vector_size(const BasicType bt) {
2378 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2379 }
2380
2381 int Matcher::min_vector_size(const BasicType bt) {
2382 // Usually, the shortest vector length supported by AArch64 ISA and
2383 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2384 // vectors in a few special cases.
2385 int size;
2386 switch(bt) {
2387 case T_BOOLEAN:
2388 // Load/store a vector mask with only 2 elements for vector types
2389 // such as "2I/2F/2L/2D".
2390 size = 2;
2391 break;
2392 case T_BYTE:
2393 // Generate a "4B" vector, to support vector cast between "8B/16B"
2394 // and "4S/4I/4L/4F/4D".
2395 size = 4;
2396 break;
2397 case T_SHORT:
2398 // Generate a "2S" vector, to support vector cast between "4S/8S"
2399 // and "2I/2L/2F/2D".
2400 size = 2;
2401 break;
2402 default:
2403 // Limit the min vector length to 64-bit.
2404 size = 8 / type2aelembytes(bt);
2405 // The number of elements in a vector should be at least 2.
2406 size = MAX2(size, 2);
2407 }
2408
2409 int max_size = max_vector_size(bt);
2410 return MIN2(size, max_size);
2411 }
2412
2413 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2414 return Matcher::max_vector_size(bt);
2415 }
2416
2417 // Actual max scalable vector register length.
2418 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2419 return Matcher::max_vector_size(bt);
2420 }
2421
2422 // Vector ideal reg.
2423 uint Matcher::vector_ideal_reg(int len) {
2424 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2425 return Op_VecA;
2426 }
2427 switch(len) {
2428 // For 16-bit/32-bit mask vector, reuse VecD.
2429 case 2:
2430 case 4:
2431 case 8: return Op_VecD;
2432 case 16: return Op_VecX;
2433 }
2434 ShouldNotReachHere();
2435 return 0;
2436 }
2437
2438 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2439 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2440 switch (ideal_reg) {
2441 case Op_VecA: return new vecAOper();
2442 case Op_VecD: return new vecDOper();
2443 case Op_VecX: return new vecXOper();
2444 }
2445 ShouldNotReachHere();
2446 return nullptr;
2447 }
2448
2449 bool Matcher::is_reg2reg_move(MachNode* m) {
2450 return false;
2451 }
2452
2453 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2454 return false;
2455 }
2456
2457 bool Matcher::is_generic_vector(MachOper* opnd) {
2458 return opnd->opcode() == VREG;
2459 }
2460
2461 // Return whether or not this register is ever used as an argument.
2462 // This function is used on startup to build the trampoline stubs in
2463 // generateOptoStub. Registers not mentioned will be killed by the VM
2464 // call in the trampoline, and arguments in those registers not be
2465 // available to the callee.
2466 bool Matcher::can_be_java_arg(int reg)
2467 {
2468 return
2469 reg == R0_num || reg == R0_H_num ||
2470 reg == R1_num || reg == R1_H_num ||
2471 reg == R2_num || reg == R2_H_num ||
2472 reg == R3_num || reg == R3_H_num ||
2473 reg == R4_num || reg == R4_H_num ||
2474 reg == R5_num || reg == R5_H_num ||
2475 reg == R6_num || reg == R6_H_num ||
2476 reg == R7_num || reg == R7_H_num ||
2477 reg == V0_num || reg == V0_H_num ||
2478 reg == V1_num || reg == V1_H_num ||
2479 reg == V2_num || reg == V2_H_num ||
2480 reg == V3_num || reg == V3_H_num ||
2481 reg == V4_num || reg == V4_H_num ||
2482 reg == V5_num || reg == V5_H_num ||
2483 reg == V6_num || reg == V6_H_num ||
2484 reg == V7_num || reg == V7_H_num;
2485 }
2486
2487 bool Matcher::is_spillable_arg(int reg)
2488 {
2489 return can_be_java_arg(reg);
2490 }
2491
2492 uint Matcher::int_pressure_limit()
2493 {
2494 // JDK-8183543: When taking the number of available registers as int
2495 // register pressure threshold, the jtreg test:
2496 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2497 // failed due to C2 compilation failure with
2498 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2499 //
2500 // A derived pointer is live at CallNode and then is flagged by RA
2501 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2502 // derived pointers and lastly fail to spill after reaching maximum
2503 // number of iterations. Lowering the default pressure threshold to
2504 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2505 // a high register pressure area of the code so that split_DEF can
2506 // generate DefinitionSpillCopy for the derived pointer.
2507 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2508 if (!PreserveFramePointer) {
2509 // When PreserveFramePointer is off, frame pointer is allocatable,
2510 // but different from other SOC registers, it is excluded from
2511 // fatproj's mask because its save type is No-Save. Decrease 1 to
2512 // ensure high pressure at fatproj when PreserveFramePointer is off.
2513 // See check_pressure_at_fatproj().
2514 default_int_pressure_threshold--;
2515 }
2516 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2517 }
2518
2519 uint Matcher::float_pressure_limit()
2520 {
2521 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2522 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2523 }
2524
2525 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2526 return false;
2527 }
2528
2529 const RegMask& Matcher::divI_proj_mask() {
2530 ShouldNotReachHere();
2531 return RegMask::EMPTY;
2532 }
2533
2534 // Register for MODI projection of divmodI.
2535 const RegMask& Matcher::modI_proj_mask() {
2536 ShouldNotReachHere();
2537 return RegMask::EMPTY;
2538 }
2539
2540 // Register for DIVL projection of divmodL.
2541 const RegMask& Matcher::divL_proj_mask() {
2542 ShouldNotReachHere();
2543 return RegMask::EMPTY;
2544 }
2545
2546 // Register for MODL projection of divmodL.
2547 const RegMask& Matcher::modL_proj_mask() {
2548 ShouldNotReachHere();
2549 return RegMask::EMPTY;
2550 }
2551
2552 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2553 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2554 Node* u = addp->fast_out(i);
2555 if (u->is_LoadStore()) {
2556 // On AArch64, LoadStoreNodes (i.e. compare and swap
2557 // instructions) only take register indirect as an operand, so
2558 // any attempt to use an AddPNode as an input to a LoadStoreNode
2559 // must fail.
2560 return false;
2561 }
2562 if (u->is_Mem()) {
2563 int opsize = u->as_Mem()->memory_size();
2564 assert(opsize > 0, "unexpected memory operand size");
2565 if (u->as_Mem()->memory_size() != (1<<shift)) {
2566 return false;
2567 }
2568 }
2569 }
2570 return true;
2571 }
2572
2573 // Convert BootTest condition to Assembler condition.
2574 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2575 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2576 Assembler::Condition result;
2577 switch(cond) {
2578 case BoolTest::eq:
2579 result = Assembler::EQ; break;
2580 case BoolTest::ne:
2581 result = Assembler::NE; break;
2582 case BoolTest::le:
2583 result = Assembler::LE; break;
2584 case BoolTest::ge:
2585 result = Assembler::GE; break;
2586 case BoolTest::lt:
2587 result = Assembler::LT; break;
2588 case BoolTest::gt:
2589 result = Assembler::GT; break;
2590 case BoolTest::ule:
2591 result = Assembler::LS; break;
2592 case BoolTest::uge:
2593 result = Assembler::HS; break;
2594 case BoolTest::ult:
2595 result = Assembler::LO; break;
2596 case BoolTest::ugt:
2597 result = Assembler::HI; break;
2598 case BoolTest::overflow:
2599 result = Assembler::VS; break;
2600 case BoolTest::no_overflow:
2601 result = Assembler::VC; break;
2602 default:
2603 ShouldNotReachHere();
2604 return Assembler::Condition(-1);
2605 }
2606
2607 // Check conversion
2608 if (cond & BoolTest::unsigned_compare) {
2609 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2610 } else {
2611 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2612 }
2613
2614 return result;
2615 }
2616
2617 // Binary src (Replicate con)
2618 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2619 if (n == nullptr || m == nullptr) {
2620 return false;
2621 }
2622
2623 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2624 return false;
2625 }
2626
2627 Node* imm_node = m->in(1);
2628 if (!imm_node->is_Con()) {
2629 return false;
2630 }
2631
2632 const Type* t = imm_node->bottom_type();
2633 if (!(t->isa_int() || t->isa_long())) {
2634 return false;
2635 }
2636
2637 switch (n->Opcode()) {
2638 case Op_AndV:
2639 case Op_OrV:
2640 case Op_XorV: {
2641 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2642 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2643 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2644 }
2645 case Op_AddVB:
2646 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2647 case Op_AddVS:
2648 case Op_AddVI:
2649 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2650 case Op_AddVL:
2651 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2652 default:
2653 return false;
2654 }
2655 }
2656
2657 // (XorV src (Replicate m1))
2658 // (XorVMask src (MaskAll m1))
2659 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2660 if (n != nullptr && m != nullptr) {
2661 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2662 VectorNode::is_all_ones_vector(m);
2663 }
2664 return false;
2665 }
2666
2667 // Should the matcher clone input 'm' of node 'n'?
2668 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2669 if (is_vshift_con_pattern(n, m) ||
2670 is_vector_bitwise_not_pattern(n, m) ||
2671 is_valid_sve_arith_imm_pattern(n, m) ||
2672 is_encode_and_store_pattern(n, m)) {
2673 mstack.push(m, Visit);
2674 return true;
2675 }
2676 return false;
2677 }
2678
2679 // Should the Matcher clone shifts on addressing modes, expecting them
2680 // to be subsumed into complex addressing expressions or compute them
2681 // into registers?
2682 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2683
2684 // Loads and stores with indirect memory input (e.g., volatile loads and
2685 // stores) do not subsume the input into complex addressing expressions. If
2686 // the addressing expression is input to at least one such load or store, do
2687 // not clone the addressing expression. Query needs_acquiring_load and
2688 // needs_releasing_store as a proxy for indirect memory input, as it is not
2689 // possible to directly query for indirect memory input at this stage.
2690 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2691 Node* n = m->fast_out(i);
2692 if (n->is_Load() && needs_acquiring_load(n)) {
2693 return false;
2694 }
2695 if (n->is_Store() && needs_releasing_store(n)) {
2696 return false;
2697 }
2698 }
2699
2700 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2701 return true;
2702 }
2703
2704 Node *off = m->in(AddPNode::Offset);
2705 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2706 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2707 // Are there other uses besides address expressions?
2708 !is_visited(off)) {
2709 address_visited.set(off->_idx); // Flag as address_visited
2710 mstack.push(off->in(2), Visit);
2711 Node *conv = off->in(1);
2712 if (conv->Opcode() == Op_ConvI2L &&
2713 // Are there other uses besides address expressions?
2714 !is_visited(conv)) {
2715 address_visited.set(conv->_idx); // Flag as address_visited
2716 mstack.push(conv->in(1), Pre_Visit);
2717 } else {
2718 mstack.push(conv, Pre_Visit);
2719 }
2720 address_visited.test_set(m->_idx); // Flag as address_visited
2721 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2722 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2723 return true;
2724 } else if (off->Opcode() == Op_ConvI2L &&
2725 // Are there other uses besides address expressions?
2726 !is_visited(off)) {
2727 address_visited.test_set(m->_idx); // Flag as address_visited
2728 address_visited.set(off->_idx); // Flag as address_visited
2729 mstack.push(off->in(1), Pre_Visit);
2730 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2731 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2732 return true;
2733 }
2734 return false;
2735 }
2736
2737 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2738 { \
2739 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2740 guarantee(DISP == 0, "mode not permitted for volatile"); \
2741 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2742 __ INSN(REG, as_Register(BASE)); \
2743 }
2744
2745
2746 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2747 {
2748 Address::extend scale;
2749
2750 // Hooboy, this is fugly. We need a way to communicate to the
2751 // encoder that the index needs to be sign extended, so we have to
2752 // enumerate all the cases.
2753 switch (opcode) {
2754 case INDINDEXSCALEDI2L:
2755 case INDINDEXSCALEDI2LN:
2756 case INDINDEXI2L:
2757 case INDINDEXI2LN:
2758 scale = Address::sxtw(size);
2759 break;
2760 default:
2761 scale = Address::lsl(size);
2762 }
2763
2764 if (index == -1) {
2765 return Address(base, disp);
2766 } else {
2767 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2768 return Address(base, as_Register(index), scale);
2769 }
2770 }
2771
2772
2773 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2774 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2775 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2776 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2777 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2778
2779 // Used for all non-volatile memory accesses. The use of
2780 // $mem->opcode() to discover whether this pattern uses sign-extended
2781 // offsets is something of a kludge.
2782 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2783 Register reg, int opcode,
2784 Register base, int index, int scale, int disp,
2785 int size_in_memory)
2786 {
2787 Address addr = mem2address(opcode, base, index, scale, disp);
2788 if (addr.getMode() == Address::base_plus_offset) {
2789 /* Fix up any out-of-range offsets. */
2790 assert_different_registers(rscratch1, base);
2791 assert_different_registers(rscratch1, reg);
2792 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2793 }
2794 (masm->*insn)(reg, addr);
2795 }
2796
2797 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2798 FloatRegister reg, int opcode,
2799 Register base, int index, int size, int disp,
2800 int size_in_memory)
2801 {
2802 Address::extend scale;
2803
2804 switch (opcode) {
2805 case INDINDEXSCALEDI2L:
2806 case INDINDEXSCALEDI2LN:
2807 scale = Address::sxtw(size);
2808 break;
2809 default:
2810 scale = Address::lsl(size);
2811 }
2812
2813 if (index == -1) {
2814 // Fix up any out-of-range offsets.
2815 assert_different_registers(rscratch1, base);
2816 Address addr = Address(base, disp);
2817 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2818 (masm->*insn)(reg, addr);
2819 } else {
2820 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2821 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2822 }
2823 }
2824
2825 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2826 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2827 int opcode, Register base, int index, int size, int disp)
2828 {
2829 if (index == -1) {
2830 (masm->*insn)(reg, T, Address(base, disp));
2831 } else {
2832 assert(disp == 0, "unsupported address mode");
2833 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2834 }
2835 }
2836
2837 %}
2838
2839
2840
2841 //----------ENCODING BLOCK-----------------------------------------------------
2842 // This block specifies the encoding classes used by the compiler to
2843 // output byte streams. Encoding classes are parameterized macros
2844 // used by Machine Instruction Nodes in order to generate the bit
2845 // encoding of the instruction. Operands specify their base encoding
2846 // interface with the interface keyword. There are currently
2847 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2848 // COND_INTER. REG_INTER causes an operand to generate a function
2849 // which returns its register number when queried. CONST_INTER causes
2850 // an operand to generate a function which returns the value of the
2851 // constant when queried. MEMORY_INTER causes an operand to generate
2852 // four functions which return the Base Register, the Index Register,
2853 // the Scale Value, and the Offset Value of the operand when queried.
2854 // COND_INTER causes an operand to generate six functions which return
2855 // the encoding code (ie - encoding bits for the instruction)
2856 // associated with each basic boolean condition for a conditional
2857 // instruction.
2858 //
2859 // Instructions specify two basic values for encoding. Again, a
2860 // function is available to check if the constant displacement is an
2861 // oop. They use the ins_encode keyword to specify their encoding
2862 // classes (which must be a sequence of enc_class names, and their
2863 // parameters, specified in the encoding block), and they use the
2864 // opcode keyword to specify, in order, their primary, secondary, and
2865 // tertiary opcode. Only the opcode sections which a particular
2866 // instruction needs for encoding need to be specified.
2867 encode %{
2868 // Build emit functions for each basic byte or larger field in the
2869 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2870 // from C++ code in the enc_class source block. Emit functions will
2871 // live in the main source block for now. In future, we can
2872 // generalize this by adding a syntax that specifies the sizes of
2873 // fields in an order, so that the adlc can build the emit functions
2874 // automagically
2875
2876 // catch all for unimplemented encodings
2877 enc_class enc_unimplemented %{
2878 __ unimplemented("C2 catch all");
2879 %}
2880
2881 // BEGIN Non-volatile memory access
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2934 Register dst_reg = as_Register($dst$$reg);
2935 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2942 Register dst_reg = as_Register($dst$$reg);
2943 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2950 Register dst_reg = as_Register($dst$$reg);
2951 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2958 Register dst_reg = as_Register($dst$$reg);
2959 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2960 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2961 %}
2962
2963 // This encoding class is generated automatically from ad_encode.m4.
2964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2965 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2966 Register dst_reg = as_Register($dst$$reg);
2967 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2968 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2969 %}
2970
2971 // This encoding class is generated automatically from ad_encode.m4.
2972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2973 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2974 Register dst_reg = as_Register($dst$$reg);
2975 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2976 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2977 %}
2978
2979 // This encoding class is generated automatically from ad_encode.m4.
2980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2981 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2982 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2983 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2984 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2985 %}
2986
2987 // This encoding class is generated automatically from ad_encode.m4.
2988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2989 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2990 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2991 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2998 Register src_reg = as_Register($src$$reg);
2999 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3000 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3001 %}
3002
3003 // This encoding class is generated automatically from ad_encode.m4.
3004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3005 enc_class aarch64_enc_strb0(memory1 mem) %{
3006 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3013 Register src_reg = as_Register($src$$reg);
3014 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3015 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3016 %}
3017
3018 // This encoding class is generated automatically from ad_encode.m4.
3019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3020 enc_class aarch64_enc_strh0(memory2 mem) %{
3021 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3022 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3023 %}
3024
3025 // This encoding class is generated automatically from ad_encode.m4.
3026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3027 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3028 Register src_reg = as_Register($src$$reg);
3029 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3030 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3031 %}
3032
3033 // This encoding class is generated automatically from ad_encode.m4.
3034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3035 enc_class aarch64_enc_strw0(memory4 mem) %{
3036 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3037 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3038 %}
3039
3040 // This encoding class is generated automatically from ad_encode.m4.
3041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3042 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3043 Register src_reg = as_Register($src$$reg);
3044 // we sometimes get asked to store the stack pointer into the
3045 // current thread -- we cannot do that directly on AArch64
3046 if (src_reg == r31_sp) {
3047 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3048 __ mov(rscratch2, sp);
3049 src_reg = rscratch2;
3050 }
3051 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3052 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3053 %}
3054
3055 // This encoding class is generated automatically from ad_encode.m4.
3056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3057 enc_class aarch64_enc_str0(memory8 mem) %{
3058 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3059 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3060 %}
3061
3062 // This encoding class is generated automatically from ad_encode.m4.
3063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3064 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3065 FloatRegister src_reg = as_FloatRegister($src$$reg);
3066 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3067 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3068 %}
3069
3070 // This encoding class is generated automatically from ad_encode.m4.
3071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3072 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3073 FloatRegister src_reg = as_FloatRegister($src$$reg);
3074 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3075 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3076 %}
3077
3078 // This encoding class is generated automatically from ad_encode.m4.
3079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3080 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3081 __ membar(Assembler::StoreStore);
3082 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3083 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3084 %}
3085
3086 // END Non-volatile memory access
3087
3088 // Vector loads and stores
3089 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3090 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3091 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3092 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3093 %}
3094
3095 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3096 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3097 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3098 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3099 %}
3100
3101 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3102 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3103 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3104 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3105 %}
3106
3107 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3108 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3109 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3110 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3111 %}
3112
3113 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3114 FloatRegister src_reg = as_FloatRegister($src$$reg);
3115 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3116 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3117 %}
3118
3119 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3120 FloatRegister src_reg = as_FloatRegister($src$$reg);
3121 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3122 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3123 %}
3124
3125 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3126 FloatRegister src_reg = as_FloatRegister($src$$reg);
3127 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3128 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3129 %}
3130
3131 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3132 FloatRegister src_reg = as_FloatRegister($src$$reg);
3133 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3134 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3135 %}
3136
3137 // volatile loads and stores
3138
3139 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3140 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3141 rscratch1, stlrb);
3142 %}
3143
3144 enc_class aarch64_enc_stlrb0(memory mem) %{
3145 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3146 rscratch1, stlrb);
3147 %}
3148
3149 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3150 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3151 rscratch1, stlrh);
3152 %}
3153
3154 enc_class aarch64_enc_stlrh0(memory mem) %{
3155 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3156 rscratch1, stlrh);
3157 %}
3158
3159 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3160 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3161 rscratch1, stlrw);
3162 %}
3163
3164 enc_class aarch64_enc_stlrw0(memory mem) %{
3165 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, stlrw);
3167 %}
3168
3169 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3170 Register dst_reg = as_Register($dst$$reg);
3171 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3172 rscratch1, ldarb);
3173 __ sxtbw(dst_reg, dst_reg);
3174 %}
3175
3176 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3177 Register dst_reg = as_Register($dst$$reg);
3178 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3179 rscratch1, ldarb);
3180 __ sxtb(dst_reg, dst_reg);
3181 %}
3182
3183 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3184 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3185 rscratch1, ldarb);
3186 %}
3187
3188 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3189 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3190 rscratch1, ldarb);
3191 %}
3192
3193 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3194 Register dst_reg = as_Register($dst$$reg);
3195 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3196 rscratch1, ldarh);
3197 __ sxthw(dst_reg, dst_reg);
3198 %}
3199
3200 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3201 Register dst_reg = as_Register($dst$$reg);
3202 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3203 rscratch1, ldarh);
3204 __ sxth(dst_reg, dst_reg);
3205 %}
3206
3207 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3208 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3209 rscratch1, ldarh);
3210 %}
3211
3212 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3213 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3214 rscratch1, ldarh);
3215 %}
3216
3217 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3218 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3219 rscratch1, ldarw);
3220 %}
3221
3222 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3223 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3224 rscratch1, ldarw);
3225 %}
3226
3227 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3228 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3229 rscratch1, ldar);
3230 %}
3231
3232 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3233 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3234 rscratch1, ldarw);
3235 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3236 %}
3237
3238 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3239 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3240 rscratch1, ldar);
3241 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3242 %}
3243
3244 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3245 Register src_reg = as_Register($src$$reg);
3246 // we sometimes get asked to store the stack pointer into the
3247 // current thread -- we cannot do that directly on AArch64
3248 if (src_reg == r31_sp) {
3249 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3250 __ mov(rscratch2, sp);
3251 src_reg = rscratch2;
3252 }
3253 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3254 rscratch1, stlr);
3255 %}
3256
3257 enc_class aarch64_enc_stlr0(memory mem) %{
3258 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3259 rscratch1, stlr);
3260 %}
3261
3262 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3263 {
3264 FloatRegister src_reg = as_FloatRegister($src$$reg);
3265 __ fmovs(rscratch2, src_reg);
3266 }
3267 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3268 rscratch1, stlrw);
3269 %}
3270
3271 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3272 {
3273 FloatRegister src_reg = as_FloatRegister($src$$reg);
3274 __ fmovd(rscratch2, src_reg);
3275 }
3276 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3277 rscratch1, stlr);
3278 %}
3279
3280 // synchronized read/update encodings
3281
3282 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3283 Register dst_reg = as_Register($dst$$reg);
3284 Register base = as_Register($mem$$base);
3285 int index = $mem$$index;
3286 int scale = $mem$$scale;
3287 int disp = $mem$$disp;
3288 if (index == -1) {
3289 if (disp != 0) {
3290 __ lea(rscratch1, Address(base, disp));
3291 __ ldaxr(dst_reg, rscratch1);
3292 } else {
3293 // TODO
3294 // should we ever get anything other than this case?
3295 __ ldaxr(dst_reg, base);
3296 }
3297 } else {
3298 Register index_reg = as_Register(index);
3299 if (disp == 0) {
3300 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3301 __ ldaxr(dst_reg, rscratch1);
3302 } else {
3303 __ lea(rscratch1, Address(base, disp));
3304 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3305 __ ldaxr(dst_reg, rscratch1);
3306 }
3307 }
3308 %}
3309
3310 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3311 Register src_reg = as_Register($src$$reg);
3312 Register base = as_Register($mem$$base);
3313 int index = $mem$$index;
3314 int scale = $mem$$scale;
3315 int disp = $mem$$disp;
3316 if (index == -1) {
3317 if (disp != 0) {
3318 __ lea(rscratch2, Address(base, disp));
3319 __ stlxr(rscratch1, src_reg, rscratch2);
3320 } else {
3321 // TODO
3322 // should we ever get anything other than this case?
3323 __ stlxr(rscratch1, src_reg, base);
3324 }
3325 } else {
3326 Register index_reg = as_Register(index);
3327 if (disp == 0) {
3328 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3329 __ stlxr(rscratch1, src_reg, rscratch2);
3330 } else {
3331 __ lea(rscratch2, Address(base, disp));
3332 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3333 __ stlxr(rscratch1, src_reg, rscratch2);
3334 }
3335 }
3336 __ cmpw(rscratch1, zr);
3337 %}
3338
3339 // prefetch encodings
3340
3341 enc_class aarch64_enc_prefetchw(memory mem) %{
3342 Register base = as_Register($mem$$base);
3343 int index = $mem$$index;
3344 int scale = $mem$$scale;
3345 int disp = $mem$$disp;
3346 if (index == -1) {
3347 // Fix up any out-of-range offsets.
3348 assert_different_registers(rscratch1, base);
3349 Address addr = Address(base, disp);
3350 addr = __ legitimize_address(addr, 8, rscratch1);
3351 __ prfm(addr, PSTL1KEEP);
3352 } else {
3353 Register index_reg = as_Register(index);
3354 if (disp == 0) {
3355 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3356 } else {
3357 __ lea(rscratch1, Address(base, disp));
3358 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3359 }
3360 }
3361 %}
3362
3363 // mov encodings
3364
3365 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3366 uint32_t con = (uint32_t)$src$$constant;
3367 Register dst_reg = as_Register($dst$$reg);
3368 if (con == 0) {
3369 __ movw(dst_reg, zr);
3370 } else {
3371 __ movw(dst_reg, con);
3372 }
3373 %}
3374
3375 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3376 Register dst_reg = as_Register($dst$$reg);
3377 uint64_t con = (uint64_t)$src$$constant;
3378 if (con == 0) {
3379 __ mov(dst_reg, zr);
3380 } else {
3381 __ mov(dst_reg, con);
3382 }
3383 %}
3384
3385 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3386 Register dst_reg = as_Register($dst$$reg);
3387 address con = (address)$src$$constant;
3388 if (con == nullptr || con == (address)1) {
3389 ShouldNotReachHere();
3390 } else {
3391 relocInfo::relocType rtype = $src->constant_reloc();
3392 if (rtype == relocInfo::oop_type) {
3393 __ movoop(dst_reg, (jobject)con);
3394 } else if (rtype == relocInfo::metadata_type) {
3395 __ mov_metadata(dst_reg, (Metadata*)con);
3396 } else {
3397 assert(rtype == relocInfo::none, "unexpected reloc type");
3398 if (! __ is_valid_AArch64_address(con) ||
3399 con < (address)(uintptr_t)os::vm_page_size()) {
3400 __ mov(dst_reg, con);
3401 } else {
3402 uint64_t offset;
3403 __ adrp(dst_reg, con, offset);
3404 __ add(dst_reg, dst_reg, offset);
3405 }
3406 }
3407 }
3408 %}
3409
3410 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3411 Register dst_reg = as_Register($dst$$reg);
3412 __ mov(dst_reg, zr);
3413 %}
3414
3415 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3416 Register dst_reg = as_Register($dst$$reg);
3417 __ mov(dst_reg, (uint64_t)1);
3418 %}
3419
3420 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3421 Register dst_reg = as_Register($dst$$reg);
3422 address con = (address)$src$$constant;
3423 if (con == nullptr) {
3424 ShouldNotReachHere();
3425 } else {
3426 relocInfo::relocType rtype = $src->constant_reloc();
3427 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3428 __ set_narrow_oop(dst_reg, (jobject)con);
3429 }
3430 %}
3431
3432 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3433 Register dst_reg = as_Register($dst$$reg);
3434 __ mov(dst_reg, zr);
3435 %}
3436
3437 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3438 Register dst_reg = as_Register($dst$$reg);
3439 address con = (address)$src$$constant;
3440 if (con == nullptr) {
3441 ShouldNotReachHere();
3442 } else {
3443 relocInfo::relocType rtype = $src->constant_reloc();
3444 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3445 __ set_narrow_klass(dst_reg, (Klass *)con);
3446 }
3447 %}
3448
3449 // arithmetic encodings
3450
3451 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3452 Register dst_reg = as_Register($dst$$reg);
3453 Register src_reg = as_Register($src1$$reg);
3454 int32_t con = (int32_t)$src2$$constant;
3455 // add has primary == 0, subtract has primary == 1
3456 if ($primary) { con = -con; }
3457 if (con < 0) {
3458 __ subw(dst_reg, src_reg, -con);
3459 } else {
3460 __ addw(dst_reg, src_reg, con);
3461 }
3462 %}
3463
3464 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3465 Register dst_reg = as_Register($dst$$reg);
3466 Register src_reg = as_Register($src1$$reg);
3467 int32_t con = (int32_t)$src2$$constant;
3468 // add has primary == 0, subtract has primary == 1
3469 if ($primary) { con = -con; }
3470 if (con < 0) {
3471 __ sub(dst_reg, src_reg, -con);
3472 } else {
3473 __ add(dst_reg, src_reg, con);
3474 }
3475 %}
3476
3477 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3478 Register dst_reg = as_Register($dst$$reg);
3479 Register src1_reg = as_Register($src1$$reg);
3480 Register src2_reg = as_Register($src2$$reg);
3481 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3482 %}
3483
3484 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3485 Register dst_reg = as_Register($dst$$reg);
3486 Register src1_reg = as_Register($src1$$reg);
3487 Register src2_reg = as_Register($src2$$reg);
3488 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3489 %}
3490
3491 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3492 Register dst_reg = as_Register($dst$$reg);
3493 Register src1_reg = as_Register($src1$$reg);
3494 Register src2_reg = as_Register($src2$$reg);
3495 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3496 %}
3497
3498 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3499 Register dst_reg = as_Register($dst$$reg);
3500 Register src1_reg = as_Register($src1$$reg);
3501 Register src2_reg = as_Register($src2$$reg);
3502 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3503 %}
3504
3505 // compare instruction encodings
3506
3507 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3508 Register reg1 = as_Register($src1$$reg);
3509 Register reg2 = as_Register($src2$$reg);
3510 __ cmpw(reg1, reg2);
3511 %}
3512
3513 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3514 Register reg = as_Register($src1$$reg);
3515 int32_t val = $src2$$constant;
3516 if (val >= 0) {
3517 __ subsw(zr, reg, val);
3518 } else {
3519 __ addsw(zr, reg, -val);
3520 }
3521 %}
3522
3523 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3524 Register reg1 = as_Register($src1$$reg);
3525 uint32_t val = (uint32_t)$src2$$constant;
3526 __ movw(rscratch1, val);
3527 __ cmpw(reg1, rscratch1);
3528 %}
3529
3530 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3531 Register reg1 = as_Register($src1$$reg);
3532 Register reg2 = as_Register($src2$$reg);
3533 __ cmp(reg1, reg2);
3534 %}
3535
3536 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3537 Register reg = as_Register($src1$$reg);
3538 int64_t val = $src2$$constant;
3539 if (val >= 0) {
3540 __ subs(zr, reg, val);
3541 } else if (val != -val) {
3542 __ adds(zr, reg, -val);
3543 } else {
3544 // aargh, Long.MIN_VALUE is a special case
3545 __ orr(rscratch1, zr, (uint64_t)val);
3546 __ subs(zr, reg, rscratch1);
3547 }
3548 %}
3549
3550 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3551 Register reg1 = as_Register($src1$$reg);
3552 uint64_t val = (uint64_t)$src2$$constant;
3553 __ mov(rscratch1, val);
3554 __ cmp(reg1, rscratch1);
3555 %}
3556
3557 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3558 Register reg1 = as_Register($src1$$reg);
3559 Register reg2 = as_Register($src2$$reg);
3560 __ cmp(reg1, reg2);
3561 %}
3562
3563 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3564 Register reg1 = as_Register($src1$$reg);
3565 Register reg2 = as_Register($src2$$reg);
3566 __ cmpw(reg1, reg2);
3567 %}
3568
3569 enc_class aarch64_enc_testp(iRegP src) %{
3570 Register reg = as_Register($src$$reg);
3571 __ cmp(reg, zr);
3572 %}
3573
3574 enc_class aarch64_enc_testn(iRegN src) %{
3575 Register reg = as_Register($src$$reg);
3576 __ cmpw(reg, zr);
3577 %}
3578
3579 enc_class aarch64_enc_b(label lbl) %{
3580 Label *L = $lbl$$label;
3581 __ b(*L);
3582 %}
3583
3584 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3585 Label *L = $lbl$$label;
3586 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3587 %}
3588
3589 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3590 Label *L = $lbl$$label;
3591 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3592 %}
3593
3594 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3595 %{
3596 Register sub_reg = as_Register($sub$$reg);
3597 Register super_reg = as_Register($super$$reg);
3598 Register temp_reg = as_Register($temp$$reg);
3599 Register result_reg = as_Register($result$$reg);
3600
3601 Label miss;
3602 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3603 nullptr, &miss,
3604 /*set_cond_codes:*/ true);
3605 if ($primary) {
3606 __ mov(result_reg, zr);
3607 }
3608 __ bind(miss);
3609 %}
3610
3611 enc_class aarch64_enc_java_static_call(method meth) %{
3612 address addr = (address)$meth$$method;
3613 address call;
3614 if (!_method) {
3615 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3616 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3617 if (call == nullptr) {
3618 ciEnv::current()->record_failure("CodeCache is full");
3619 return;
3620 }
3621 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3622 // The NOP here is purely to ensure that eliding a call to
3623 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3624 __ nop();
3625 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3626 } else {
3627 int method_index = resolved_method_index(masm);
3628 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3629 : static_call_Relocation::spec(method_index);
3630 call = __ trampoline_call(Address(addr, rspec));
3631 if (call == nullptr) {
3632 ciEnv::current()->record_failure("CodeCache is full");
3633 return;
3634 }
3635 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3636 // Calls of the same statically bound method can share
3637 // a stub to the interpreter.
3638 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3639 } else {
3640 // Emit stub for static call
3641 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3642 if (stub == nullptr) {
3643 ciEnv::current()->record_failure("CodeCache is full");
3644 return;
3645 }
3646 }
3647 }
3648
3649 __ post_call_nop();
3650
3651 // Only non uncommon_trap calls need to reinitialize ptrue.
3652 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3653 __ reinitialize_ptrue();
3654 }
3655 %}
3656
3657 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3658 int method_index = resolved_method_index(masm);
3659 address call = __ ic_call((address)$meth$$method, method_index);
3660 if (call == nullptr) {
3661 ciEnv::current()->record_failure("CodeCache is full");
3662 return;
3663 }
3664 __ post_call_nop();
3665 if (Compile::current()->max_vector_size() > 0) {
3666 __ reinitialize_ptrue();
3667 }
3668 %}
3669
3670 enc_class aarch64_enc_call_epilog() %{
3671 if (VerifyStackAtCalls) {
3672 // Check that stack depth is unchanged: find majik cookie on stack
3673 __ call_Unimplemented();
3674 }
3675 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
3676 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3677 // Search for the corresponding projection, get the register and emit code that initialized it.
3678 uint con = (tf()->range_cc()->cnt() - 1);
3679 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3680 ProjNode* proj = fast_out(i)->as_Proj();
3681 if (proj->_con == con) {
3682 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3683 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3684 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3685 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3686 __ cmp(r0, zr);
3687 __ cset(toReg, Assembler::NE);
3688 if (reg->is_stack()) {
3689 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3690 __ str(toReg, Address(sp, st_off));
3691 }
3692 break;
3693 }
3694 }
3695 if (return_value_is_used()) {
3696 // An inline type is returned as fields in multiple registers.
3697 // R0 either contains an oop if the inline type is buffered or a pointer
3698 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3699 // if the lowest bit is set to allow C2 to use the oop after null checking.
3700 // r0 &= (r0 & 1) - 1
3701 __ andr(rscratch1, r0, 0x1);
3702 __ sub(rscratch1, rscratch1, 0x1);
3703 __ andr(r0, r0, rscratch1);
3704 }
3705 }
3706 %}
3707
3708 enc_class aarch64_enc_java_to_runtime(method meth) %{
3709 // some calls to generated routines (arraycopy code) are scheduled
3710 // by C2 as runtime calls. if so we can call them using a br (they
3711 // will be in a reachable segment) otherwise we have to use a blr
3712 // which loads the absolute address into a register.
3713 address entry = (address)$meth$$method;
3714 CodeBlob *cb = CodeCache::find_blob(entry);
3715 if (cb) {
3716 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3717 if (call == nullptr) {
3718 ciEnv::current()->record_failure("CodeCache is full");
3719 return;
3720 }
3721 __ post_call_nop();
3722 } else {
3723 Label retaddr;
3724 // Make the anchor frame walkable
3725 __ adr(rscratch2, retaddr);
3726 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3727 __ lea(rscratch1, RuntimeAddress(entry));
3728 __ blr(rscratch1);
3729 __ bind(retaddr);
3730 __ post_call_nop();
3731 }
3732 if (Compile::current()->max_vector_size() > 0) {
3733 __ reinitialize_ptrue();
3734 }
3735 %}
3736
3737 enc_class aarch64_enc_rethrow() %{
3738 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3739 %}
3740
3741 enc_class aarch64_enc_ret() %{
3742 #ifdef ASSERT
3743 if (Compile::current()->max_vector_size() > 0) {
3744 __ verify_ptrue();
3745 }
3746 #endif
3747 __ ret(lr);
3748 %}
3749
3750 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3751 Register target_reg = as_Register($jump_target$$reg);
3752 __ br(target_reg);
3753 %}
3754
3755 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3756 Register target_reg = as_Register($jump_target$$reg);
3757 // exception oop should be in r0
3758 // ret addr has been popped into lr
3759 // callee expects it in r3
3760 __ mov(r3, lr);
3761 __ br(target_reg);
3762 %}
3763
3764 %}
3765
3766 //----------FRAME--------------------------------------------------------------
3767 // Definition of frame structure and management information.
3768 //
3769 // S T A C K L A Y O U T Allocators stack-slot number
3770 // | (to get allocators register number
3771 // G Owned by | | v add OptoReg::stack0())
3772 // r CALLER | |
3773 // o | +--------+ pad to even-align allocators stack-slot
3774 // w V | pad0 | numbers; owned by CALLER
3775 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3776 // h ^ | in | 5
3777 // | | args | 4 Holes in incoming args owned by SELF
3778 // | | | | 3
3779 // | | +--------+
3780 // V | | old out| Empty on Intel, window on Sparc
3781 // | old |preserve| Must be even aligned.
3782 // | SP-+--------+----> Matcher::_old_SP, even aligned
3783 // | | in | 3 area for Intel ret address
3784 // Owned by |preserve| Empty on Sparc.
3785 // SELF +--------+
3786 // | | pad2 | 2 pad to align old SP
3787 // | +--------+ 1
3788 // | | locks | 0
3789 // | +--------+----> OptoReg::stack0(), even aligned
3790 // | | pad1 | 11 pad to align new SP
3791 // | +--------+
3792 // | | | 10
3793 // | | spills | 9 spills
3794 // V | | 8 (pad0 slot for callee)
3795 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3796 // ^ | out | 7
3797 // | | args | 6 Holes in outgoing args owned by CALLEE
3798 // Owned by +--------+
3799 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3800 // | new |preserve| Must be even-aligned.
3801 // | SP-+--------+----> Matcher::_new_SP, even aligned
3802 // | | |
3803 //
3804 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3805 // known from SELF's arguments and the Java calling convention.
3806 // Region 6-7 is determined per call site.
3807 // Note 2: If the calling convention leaves holes in the incoming argument
3808 // area, those holes are owned by SELF. Holes in the outgoing area
3809 // are owned by the CALLEE. Holes should not be necessary in the
3810 // incoming area, as the Java calling convention is completely under
3811 // the control of the AD file. Doubles can be sorted and packed to
3812 // avoid holes. Holes in the outgoing arguments may be necessary for
3813 // varargs C calling conventions.
3814 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3815 // even aligned with pad0 as needed.
3816 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3817 // (the latter is true on Intel but is it false on AArch64?)
3818 // region 6-11 is even aligned; it may be padded out more so that
3819 // the region from SP to FP meets the minimum stack alignment.
3820 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3821 // alignment. Region 11, pad1, may be dynamically extended so that
3822 // SP meets the minimum alignment.
3823
3824 frame %{
3825 // These three registers define part of the calling convention
3826 // between compiled code and the interpreter.
3827
3828 // Inline Cache Register or Method for I2C.
3829 inline_cache_reg(R12);
3830
3831 // Number of stack slots consumed by locking an object
3832 sync_stack_slots(2);
3833
3834 // Compiled code's Frame Pointer
3835 frame_pointer(R31);
3836
3837 // Interpreter stores its frame pointer in a register which is
3838 // stored to the stack by I2CAdaptors.
3839 // I2CAdaptors convert from interpreted java to compiled java.
3840 interpreter_frame_pointer(R29);
3841
3842 // Stack alignment requirement
3843 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3844
3845 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3846 // for calls to C. Supports the var-args backing area for register parms.
3847 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3848
3849 // The after-PROLOG location of the return address. Location of
3850 // return address specifies a type (REG or STACK) and a number
3851 // representing the register number (i.e. - use a register name) or
3852 // stack slot.
3853 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3854 // Otherwise, it is above the locks and verification slot and alignment word
3855 // TODO this may well be correct but need to check why that - 2 is there
3856 // ppc port uses 0 but we definitely need to allow for fixed_slots
3857 // which folds in the space used for monitors
3858 return_addr(STACK - 2 +
3859 align_up((Compile::current()->in_preserve_stack_slots() +
3860 Compile::current()->fixed_slots()),
3861 stack_alignment_in_slots()));
3862
3863 // Location of compiled Java return values. Same as C for now.
3864 return_value
3865 %{
3866 // TODO do we allow ideal_reg == Op_RegN???
3867 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3868 "only return normal values");
3869
3870 static const int lo[Op_RegL + 1] = { // enum name
3871 0, // Op_Node
3872 0, // Op_Set
3873 R0_num, // Op_RegN
3874 R0_num, // Op_RegI
3875 R0_num, // Op_RegP
3876 V0_num, // Op_RegF
3877 V0_num, // Op_RegD
3878 R0_num // Op_RegL
3879 };
3880
3881 static const int hi[Op_RegL + 1] = { // enum name
3882 0, // Op_Node
3883 0, // Op_Set
3884 OptoReg::Bad, // Op_RegN
3885 OptoReg::Bad, // Op_RegI
3886 R0_H_num, // Op_RegP
3887 OptoReg::Bad, // Op_RegF
3888 V0_H_num, // Op_RegD
3889 R0_H_num // Op_RegL
3890 };
3891
3892 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3893 %}
3894 %}
3895
3896 //----------ATTRIBUTES---------------------------------------------------------
3897 //----------Operand Attributes-------------------------------------------------
3898 op_attrib op_cost(1); // Required cost attribute
3899
3900 //----------Instruction Attributes---------------------------------------------
3901 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3902 ins_attrib ins_size(32); // Required size attribute (in bits)
3903 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3904 // a non-matching short branch variant
3905 // of some long branch?
3906 ins_attrib ins_alignment(4); // Required alignment attribute (must
3907 // be a power of 2) specifies the
3908 // alignment that some part of the
3909 // instruction (not necessarily the
3910 // start) requires. If > 1, a
3911 // compute_padding() function must be
3912 // provided for the instruction
3913
3914 // Whether this node is expanded during code emission into a sequence of
3915 // instructions and the first instruction can perform an implicit null check.
3916 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3917
3918 //----------OPERANDS-----------------------------------------------------------
3919 // Operand definitions must precede instruction definitions for correct parsing
3920 // in the ADLC because operands constitute user defined types which are used in
3921 // instruction definitions.
3922
3923 //----------Simple Operands----------------------------------------------------
3924
3925 // Integer operands 32 bit
3926 // 32 bit immediate
3927 operand immI()
3928 %{
3929 match(ConI);
3930
3931 op_cost(0);
3932 format %{ %}
3933 interface(CONST_INTER);
3934 %}
3935
3936 // 32 bit zero
3937 operand immI0()
3938 %{
3939 predicate(n->get_int() == 0);
3940 match(ConI);
3941
3942 op_cost(0);
3943 format %{ %}
3944 interface(CONST_INTER);
3945 %}
3946
3947 // 32 bit unit increment
3948 operand immI_1()
3949 %{
3950 predicate(n->get_int() == 1);
3951 match(ConI);
3952
3953 op_cost(0);
3954 format %{ %}
3955 interface(CONST_INTER);
3956 %}
3957
3958 // 32 bit unit decrement
3959 operand immI_M1()
3960 %{
3961 predicate(n->get_int() == -1);
3962 match(ConI);
3963
3964 op_cost(0);
3965 format %{ %}
3966 interface(CONST_INTER);
3967 %}
3968
3969 // Shift values for add/sub extension shift
3970 operand immIExt()
3971 %{
3972 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3973 match(ConI);
3974
3975 op_cost(0);
3976 format %{ %}
3977 interface(CONST_INTER);
3978 %}
3979
3980 operand immI_gt_1()
3981 %{
3982 predicate(n->get_int() > 1);
3983 match(ConI);
3984
3985 op_cost(0);
3986 format %{ %}
3987 interface(CONST_INTER);
3988 %}
3989
3990 operand immI_le_4()
3991 %{
3992 predicate(n->get_int() <= 4);
3993 match(ConI);
3994
3995 op_cost(0);
3996 format %{ %}
3997 interface(CONST_INTER);
3998 %}
3999
4000 operand immI_16()
4001 %{
4002 predicate(n->get_int() == 16);
4003 match(ConI);
4004
4005 op_cost(0);
4006 format %{ %}
4007 interface(CONST_INTER);
4008 %}
4009
4010 operand immI_24()
4011 %{
4012 predicate(n->get_int() == 24);
4013 match(ConI);
4014
4015 op_cost(0);
4016 format %{ %}
4017 interface(CONST_INTER);
4018 %}
4019
4020 operand immI_32()
4021 %{
4022 predicate(n->get_int() == 32);
4023 match(ConI);
4024
4025 op_cost(0);
4026 format %{ %}
4027 interface(CONST_INTER);
4028 %}
4029
4030 operand immI_48()
4031 %{
4032 predicate(n->get_int() == 48);
4033 match(ConI);
4034
4035 op_cost(0);
4036 format %{ %}
4037 interface(CONST_INTER);
4038 %}
4039
4040 operand immI_56()
4041 %{
4042 predicate(n->get_int() == 56);
4043 match(ConI);
4044
4045 op_cost(0);
4046 format %{ %}
4047 interface(CONST_INTER);
4048 %}
4049
4050 operand immI_255()
4051 %{
4052 predicate(n->get_int() == 255);
4053 match(ConI);
4054
4055 op_cost(0);
4056 format %{ %}
4057 interface(CONST_INTER);
4058 %}
4059
4060 operand immI_65535()
4061 %{
4062 predicate(n->get_int() == 65535);
4063 match(ConI);
4064
4065 op_cost(0);
4066 format %{ %}
4067 interface(CONST_INTER);
4068 %}
4069
4070 operand immI_positive()
4071 %{
4072 predicate(n->get_int() > 0);
4073 match(ConI);
4074
4075 op_cost(0);
4076 format %{ %}
4077 interface(CONST_INTER);
4078 %}
4079
4080 // BoolTest condition for signed compare
4081 operand immI_cmp_cond()
4082 %{
4083 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4084 match(ConI);
4085
4086 op_cost(0);
4087 format %{ %}
4088 interface(CONST_INTER);
4089 %}
4090
4091 // BoolTest condition for unsigned compare
4092 operand immI_cmpU_cond()
4093 %{
4094 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4095 match(ConI);
4096
4097 op_cost(0);
4098 format %{ %}
4099 interface(CONST_INTER);
4100 %}
4101
4102 operand immL_255()
4103 %{
4104 predicate(n->get_long() == 255L);
4105 match(ConL);
4106
4107 op_cost(0);
4108 format %{ %}
4109 interface(CONST_INTER);
4110 %}
4111
4112 operand immL_65535()
4113 %{
4114 predicate(n->get_long() == 65535L);
4115 match(ConL);
4116
4117 op_cost(0);
4118 format %{ %}
4119 interface(CONST_INTER);
4120 %}
4121
4122 operand immL_4294967295()
4123 %{
4124 predicate(n->get_long() == 4294967295L);
4125 match(ConL);
4126
4127 op_cost(0);
4128 format %{ %}
4129 interface(CONST_INTER);
4130 %}
4131
4132 operand immL_bitmask()
4133 %{
4134 predicate((n->get_long() != 0)
4135 && ((n->get_long() & 0xc000000000000000l) == 0)
4136 && is_power_of_2(n->get_long() + 1));
4137 match(ConL);
4138
4139 op_cost(0);
4140 format %{ %}
4141 interface(CONST_INTER);
4142 %}
4143
4144 operand immI_bitmask()
4145 %{
4146 predicate((n->get_int() != 0)
4147 && ((n->get_int() & 0xc0000000) == 0)
4148 && is_power_of_2(n->get_int() + 1));
4149 match(ConI);
4150
4151 op_cost(0);
4152 format %{ %}
4153 interface(CONST_INTER);
4154 %}
4155
4156 operand immL_positive_bitmaskI()
4157 %{
4158 predicate((n->get_long() != 0)
4159 && ((julong)n->get_long() < 0x80000000ULL)
4160 && is_power_of_2(n->get_long() + 1));
4161 match(ConL);
4162
4163 op_cost(0);
4164 format %{ %}
4165 interface(CONST_INTER);
4166 %}
4167
4168 // Scale values for scaled offset addressing modes (up to long but not quad)
4169 operand immIScale()
4170 %{
4171 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4172 match(ConI);
4173
4174 op_cost(0);
4175 format %{ %}
4176 interface(CONST_INTER);
4177 %}
4178
4179 // 5 bit signed integer
4180 operand immI5()
4181 %{
4182 predicate(Assembler::is_simm(n->get_int(), 5));
4183 match(ConI);
4184
4185 op_cost(0);
4186 format %{ %}
4187 interface(CONST_INTER);
4188 %}
4189
4190 // 7 bit unsigned integer
4191 operand immIU7()
4192 %{
4193 predicate(Assembler::is_uimm(n->get_int(), 7));
4194 match(ConI);
4195
4196 op_cost(0);
4197 format %{ %}
4198 interface(CONST_INTER);
4199 %}
4200
4201 // Offset for scaled or unscaled immediate loads and stores
4202 operand immIOffset()
4203 %{
4204 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4205 match(ConI);
4206
4207 op_cost(0);
4208 format %{ %}
4209 interface(CONST_INTER);
4210 %}
4211
4212 operand immIOffset1()
4213 %{
4214 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4215 match(ConI);
4216
4217 op_cost(0);
4218 format %{ %}
4219 interface(CONST_INTER);
4220 %}
4221
4222 operand immIOffset2()
4223 %{
4224 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4225 match(ConI);
4226
4227 op_cost(0);
4228 format %{ %}
4229 interface(CONST_INTER);
4230 %}
4231
4232 operand immIOffset4()
4233 %{
4234 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4235 match(ConI);
4236
4237 op_cost(0);
4238 format %{ %}
4239 interface(CONST_INTER);
4240 %}
4241
4242 operand immIOffset8()
4243 %{
4244 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4245 match(ConI);
4246
4247 op_cost(0);
4248 format %{ %}
4249 interface(CONST_INTER);
4250 %}
4251
4252 operand immIOffset16()
4253 %{
4254 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4255 match(ConI);
4256
4257 op_cost(0);
4258 format %{ %}
4259 interface(CONST_INTER);
4260 %}
4261
4262 operand immLOffset()
4263 %{
4264 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4265 match(ConL);
4266
4267 op_cost(0);
4268 format %{ %}
4269 interface(CONST_INTER);
4270 %}
4271
4272 operand immLoffset1()
4273 %{
4274 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4275 match(ConL);
4276
4277 op_cost(0);
4278 format %{ %}
4279 interface(CONST_INTER);
4280 %}
4281
4282 operand immLoffset2()
4283 %{
4284 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4285 match(ConL);
4286
4287 op_cost(0);
4288 format %{ %}
4289 interface(CONST_INTER);
4290 %}
4291
4292 operand immLoffset4()
4293 %{
4294 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4295 match(ConL);
4296
4297 op_cost(0);
4298 format %{ %}
4299 interface(CONST_INTER);
4300 %}
4301
4302 operand immLoffset8()
4303 %{
4304 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4305 match(ConL);
4306
4307 op_cost(0);
4308 format %{ %}
4309 interface(CONST_INTER);
4310 %}
4311
4312 operand immLoffset16()
4313 %{
4314 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4315 match(ConL);
4316
4317 op_cost(0);
4318 format %{ %}
4319 interface(CONST_INTER);
4320 %}
4321
4322 // 5 bit signed long integer
4323 operand immL5()
4324 %{
4325 predicate(Assembler::is_simm(n->get_long(), 5));
4326 match(ConL);
4327
4328 op_cost(0);
4329 format %{ %}
4330 interface(CONST_INTER);
4331 %}
4332
4333 // 7 bit unsigned long integer
4334 operand immLU7()
4335 %{
4336 predicate(Assembler::is_uimm(n->get_long(), 7));
4337 match(ConL);
4338
4339 op_cost(0);
4340 format %{ %}
4341 interface(CONST_INTER);
4342 %}
4343
4344 // 8 bit signed value.
4345 operand immI8()
4346 %{
4347 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4348 match(ConI);
4349
4350 op_cost(0);
4351 format %{ %}
4352 interface(CONST_INTER);
4353 %}
4354
4355 // 8 bit signed value (simm8), or #simm8 LSL 8.
4356 operand immIDupV()
4357 %{
4358 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4359 match(ConI);
4360
4361 op_cost(0);
4362 format %{ %}
4363 interface(CONST_INTER);
4364 %}
4365
4366 // 8 bit signed value (simm8), or #simm8 LSL 8.
4367 operand immLDupV()
4368 %{
4369 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4370 match(ConL);
4371
4372 op_cost(0);
4373 format %{ %}
4374 interface(CONST_INTER);
4375 %}
4376
4377 // 8 bit signed value (simm8), or #simm8 LSL 8.
4378 operand immHDupV()
4379 %{
4380 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4381 match(ConH);
4382
4383 op_cost(0);
4384 format %{ %}
4385 interface(CONST_INTER);
4386 %}
4387
4388 // 8 bit integer valid for vector add sub immediate
4389 operand immBAddSubV()
4390 %{
4391 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4392 match(ConI);
4393
4394 op_cost(0);
4395 format %{ %}
4396 interface(CONST_INTER);
4397 %}
4398
4399 // 32 bit integer valid for add sub immediate
4400 operand immIAddSub()
4401 %{
4402 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4403 match(ConI);
4404 op_cost(0);
4405 format %{ %}
4406 interface(CONST_INTER);
4407 %}
4408
4409 // 32 bit integer valid for vector add sub immediate
4410 operand immIAddSubV()
4411 %{
4412 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4413 match(ConI);
4414
4415 op_cost(0);
4416 format %{ %}
4417 interface(CONST_INTER);
4418 %}
4419
4420 // 32 bit unsigned integer valid for logical immediate
4421
4422 operand immBLog()
4423 %{
4424 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4425 match(ConI);
4426
4427 op_cost(0);
4428 format %{ %}
4429 interface(CONST_INTER);
4430 %}
4431
4432 operand immSLog()
4433 %{
4434 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4435 match(ConI);
4436
4437 op_cost(0);
4438 format %{ %}
4439 interface(CONST_INTER);
4440 %}
4441
4442 operand immILog()
4443 %{
4444 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4445 match(ConI);
4446
4447 op_cost(0);
4448 format %{ %}
4449 interface(CONST_INTER);
4450 %}
4451
4452 // Integer operands 64 bit
4453 // 64 bit immediate
4454 operand immL()
4455 %{
4456 match(ConL);
4457
4458 op_cost(0);
4459 format %{ %}
4460 interface(CONST_INTER);
4461 %}
4462
4463 // 64 bit zero
4464 operand immL0()
4465 %{
4466 predicate(n->get_long() == 0);
4467 match(ConL);
4468
4469 op_cost(0);
4470 format %{ %}
4471 interface(CONST_INTER);
4472 %}
4473
4474 // 64 bit unit decrement
4475 operand immL_M1()
4476 %{
4477 predicate(n->get_long() == -1);
4478 match(ConL);
4479
4480 op_cost(0);
4481 format %{ %}
4482 interface(CONST_INTER);
4483 %}
4484
4485 // 64 bit integer valid for add sub immediate
4486 operand immLAddSub()
4487 %{
4488 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4489 match(ConL);
4490 op_cost(0);
4491 format %{ %}
4492 interface(CONST_INTER);
4493 %}
4494
4495 // 64 bit integer valid for addv subv immediate
4496 operand immLAddSubV()
4497 %{
4498 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4499 match(ConL);
4500
4501 op_cost(0);
4502 format %{ %}
4503 interface(CONST_INTER);
4504 %}
4505
4506 // 64 bit integer valid for logical immediate
4507 operand immLLog()
4508 %{
4509 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4510 match(ConL);
4511 op_cost(0);
4512 format %{ %}
4513 interface(CONST_INTER);
4514 %}
4515
4516 // Long Immediate: low 32-bit mask
4517 operand immL_32bits()
4518 %{
4519 predicate(n->get_long() == 0xFFFFFFFFL);
4520 match(ConL);
4521 op_cost(0);
4522 format %{ %}
4523 interface(CONST_INTER);
4524 %}
4525
4526 // Pointer operands
4527 // Pointer Immediate
4528 operand immP()
4529 %{
4530 match(ConP);
4531
4532 op_cost(0);
4533 format %{ %}
4534 interface(CONST_INTER);
4535 %}
4536
4537 // nullptr Pointer Immediate
4538 operand immP0()
4539 %{
4540 predicate(n->get_ptr() == 0);
4541 match(ConP);
4542
4543 op_cost(0);
4544 format %{ %}
4545 interface(CONST_INTER);
4546 %}
4547
4548 // Pointer Immediate One
4549 // this is used in object initialization (initial object header)
4550 operand immP_1()
4551 %{
4552 predicate(n->get_ptr() == 1);
4553 match(ConP);
4554
4555 op_cost(0);
4556 format %{ %}
4557 interface(CONST_INTER);
4558 %}
4559
4560 // Float and Double operands
4561 // Double Immediate
4562 operand immD()
4563 %{
4564 match(ConD);
4565 op_cost(0);
4566 format %{ %}
4567 interface(CONST_INTER);
4568 %}
4569
4570 // Double Immediate: +0.0d
4571 operand immD0()
4572 %{
4573 predicate(jlong_cast(n->getd()) == 0);
4574 match(ConD);
4575
4576 op_cost(0);
4577 format %{ %}
4578 interface(CONST_INTER);
4579 %}
4580
4581 // constant 'double +0.0'.
4582 operand immDPacked()
4583 %{
4584 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4585 match(ConD);
4586 op_cost(0);
4587 format %{ %}
4588 interface(CONST_INTER);
4589 %}
4590
4591 // Float Immediate
4592 operand immF()
4593 %{
4594 match(ConF);
4595 op_cost(0);
4596 format %{ %}
4597 interface(CONST_INTER);
4598 %}
4599
4600 // Float Immediate: +0.0f.
4601 operand immF0()
4602 %{
4603 predicate(jint_cast(n->getf()) == 0);
4604 match(ConF);
4605
4606 op_cost(0);
4607 format %{ %}
4608 interface(CONST_INTER);
4609 %}
4610
4611 // Half Float (FP16) Immediate
4612 operand immH()
4613 %{
4614 match(ConH);
4615 op_cost(0);
4616 format %{ %}
4617 interface(CONST_INTER);
4618 %}
4619
4620 //
4621 operand immFPacked()
4622 %{
4623 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4624 match(ConF);
4625 op_cost(0);
4626 format %{ %}
4627 interface(CONST_INTER);
4628 %}
4629
4630 // Narrow pointer operands
4631 // Narrow Pointer Immediate
4632 operand immN()
4633 %{
4634 match(ConN);
4635
4636 op_cost(0);
4637 format %{ %}
4638 interface(CONST_INTER);
4639 %}
4640
4641 // Narrow nullptr Pointer Immediate
4642 operand immN0()
4643 %{
4644 predicate(n->get_narrowcon() == 0);
4645 match(ConN);
4646
4647 op_cost(0);
4648 format %{ %}
4649 interface(CONST_INTER);
4650 %}
4651
4652 operand immNKlass()
4653 %{
4654 match(ConNKlass);
4655
4656 op_cost(0);
4657 format %{ %}
4658 interface(CONST_INTER);
4659 %}
4660
4661 // Integer 32 bit Register Operands
4662 // Integer 32 bitRegister (excludes SP)
4663 operand iRegI()
4664 %{
4665 constraint(ALLOC_IN_RC(any_reg32));
4666 match(RegI);
4667 match(iRegINoSp);
4668 op_cost(0);
4669 format %{ %}
4670 interface(REG_INTER);
4671 %}
4672
4673 // Integer 32 bit Register not Special
4674 operand iRegINoSp()
4675 %{
4676 constraint(ALLOC_IN_RC(no_special_reg32));
4677 match(RegI);
4678 op_cost(0);
4679 format %{ %}
4680 interface(REG_INTER);
4681 %}
4682
4683 // Integer 64 bit Register Operands
4684 // Integer 64 bit Register (includes SP)
4685 operand iRegL()
4686 %{
4687 constraint(ALLOC_IN_RC(any_reg));
4688 match(RegL);
4689 match(iRegLNoSp);
4690 op_cost(0);
4691 format %{ %}
4692 interface(REG_INTER);
4693 %}
4694
4695 // Integer 64 bit Register not Special
4696 operand iRegLNoSp()
4697 %{
4698 constraint(ALLOC_IN_RC(no_special_reg));
4699 match(RegL);
4700 match(iRegL_R0);
4701 format %{ %}
4702 interface(REG_INTER);
4703 %}
4704
4705 // Pointer Register Operands
4706 // Pointer Register
4707 operand iRegP()
4708 %{
4709 constraint(ALLOC_IN_RC(ptr_reg));
4710 match(RegP);
4711 match(iRegPNoSp);
4712 match(iRegP_R0);
4713 //match(iRegP_R2);
4714 //match(iRegP_R4);
4715 match(iRegP_R5);
4716 match(thread_RegP);
4717 op_cost(0);
4718 format %{ %}
4719 interface(REG_INTER);
4720 %}
4721
4722 // Pointer 64 bit Register not Special
4723 operand iRegPNoSp()
4724 %{
4725 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4726 match(RegP);
4727 // match(iRegP);
4728 // match(iRegP_R0);
4729 // match(iRegP_R2);
4730 // match(iRegP_R4);
4731 // match(iRegP_R5);
4732 // match(thread_RegP);
4733 op_cost(0);
4734 format %{ %}
4735 interface(REG_INTER);
4736 %}
4737
4738 // This operand is not allowed to use rfp even if
4739 // rfp is not used to hold the frame pointer.
4740 operand iRegPNoSpNoRfp()
4741 %{
4742 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4743 match(RegP);
4744 match(iRegPNoSp);
4745 op_cost(0);
4746 format %{ %}
4747 interface(REG_INTER);
4748 %}
4749
4750 // Pointer 64 bit Register R0 only
4751 operand iRegP_R0()
4752 %{
4753 constraint(ALLOC_IN_RC(r0_reg));
4754 match(RegP);
4755 // match(iRegP);
4756 match(iRegPNoSp);
4757 op_cost(0);
4758 format %{ %}
4759 interface(REG_INTER);
4760 %}
4761
4762 // Pointer 64 bit Register R1 only
4763 operand iRegP_R1()
4764 %{
4765 constraint(ALLOC_IN_RC(r1_reg));
4766 match(RegP);
4767 // match(iRegP);
4768 match(iRegPNoSp);
4769 op_cost(0);
4770 format %{ %}
4771 interface(REG_INTER);
4772 %}
4773
4774 // Pointer 64 bit Register R2 only
4775 operand iRegP_R2()
4776 %{
4777 constraint(ALLOC_IN_RC(r2_reg));
4778 match(RegP);
4779 // match(iRegP);
4780 match(iRegPNoSp);
4781 op_cost(0);
4782 format %{ %}
4783 interface(REG_INTER);
4784 %}
4785
4786 // Pointer 64 bit Register R3 only
4787 operand iRegP_R3()
4788 %{
4789 constraint(ALLOC_IN_RC(r3_reg));
4790 match(RegP);
4791 // match(iRegP);
4792 match(iRegPNoSp);
4793 op_cost(0);
4794 format %{ %}
4795 interface(REG_INTER);
4796 %}
4797
4798 // Pointer 64 bit Register R4 only
4799 operand iRegP_R4()
4800 %{
4801 constraint(ALLOC_IN_RC(r4_reg));
4802 match(RegP);
4803 // match(iRegP);
4804 match(iRegPNoSp);
4805 op_cost(0);
4806 format %{ %}
4807 interface(REG_INTER);
4808 %}
4809
4810 // Pointer 64 bit Register R5 only
4811 operand iRegP_R5()
4812 %{
4813 constraint(ALLOC_IN_RC(r5_reg));
4814 match(RegP);
4815 // match(iRegP);
4816 match(iRegPNoSp);
4817 op_cost(0);
4818 format %{ %}
4819 interface(REG_INTER);
4820 %}
4821
4822 // Pointer 64 bit Register R10 only
4823 operand iRegP_R10()
4824 %{
4825 constraint(ALLOC_IN_RC(r10_reg));
4826 match(RegP);
4827 // match(iRegP);
4828 match(iRegPNoSp);
4829 op_cost(0);
4830 format %{ %}
4831 interface(REG_INTER);
4832 %}
4833
4834 // Long 64 bit Register R0 only
4835 operand iRegL_R0()
4836 %{
4837 constraint(ALLOC_IN_RC(r0_reg));
4838 match(RegL);
4839 match(iRegLNoSp);
4840 op_cost(0);
4841 format %{ %}
4842 interface(REG_INTER);
4843 %}
4844
4845 // Long 64 bit Register R11 only
4846 operand iRegL_R11()
4847 %{
4848 constraint(ALLOC_IN_RC(r11_reg));
4849 match(RegL);
4850 match(iRegLNoSp);
4851 op_cost(0);
4852 format %{ %}
4853 interface(REG_INTER);
4854 %}
4855
4856 // Register R0 only
4857 operand iRegI_R0()
4858 %{
4859 constraint(ALLOC_IN_RC(int_r0_reg));
4860 match(RegI);
4861 match(iRegINoSp);
4862 op_cost(0);
4863 format %{ %}
4864 interface(REG_INTER);
4865 %}
4866
4867 // Register R2 only
4868 operand iRegI_R2()
4869 %{
4870 constraint(ALLOC_IN_RC(int_r2_reg));
4871 match(RegI);
4872 match(iRegINoSp);
4873 op_cost(0);
4874 format %{ %}
4875 interface(REG_INTER);
4876 %}
4877
4878 // Register R3 only
4879 operand iRegI_R3()
4880 %{
4881 constraint(ALLOC_IN_RC(int_r3_reg));
4882 match(RegI);
4883 match(iRegINoSp);
4884 op_cost(0);
4885 format %{ %}
4886 interface(REG_INTER);
4887 %}
4888
4889
4890 // Register R4 only
4891 operand iRegI_R4()
4892 %{
4893 constraint(ALLOC_IN_RC(int_r4_reg));
4894 match(RegI);
4895 match(iRegINoSp);
4896 op_cost(0);
4897 format %{ %}
4898 interface(REG_INTER);
4899 %}
4900
4901
4902 // Pointer Register Operands
4903 // Narrow Pointer Register
4904 operand iRegN()
4905 %{
4906 constraint(ALLOC_IN_RC(any_reg32));
4907 match(RegN);
4908 match(iRegNNoSp);
4909 op_cost(0);
4910 format %{ %}
4911 interface(REG_INTER);
4912 %}
4913
4914 // Integer 64 bit Register not Special
4915 operand iRegNNoSp()
4916 %{
4917 constraint(ALLOC_IN_RC(no_special_reg32));
4918 match(RegN);
4919 op_cost(0);
4920 format %{ %}
4921 interface(REG_INTER);
4922 %}
4923
4924 // Float Register
4925 // Float register operands
4926 operand vRegF()
4927 %{
4928 constraint(ALLOC_IN_RC(float_reg));
4929 match(RegF);
4930
4931 op_cost(0);
4932 format %{ %}
4933 interface(REG_INTER);
4934 %}
4935
4936 // Double Register
4937 // Double register operands
4938 operand vRegD()
4939 %{
4940 constraint(ALLOC_IN_RC(double_reg));
4941 match(RegD);
4942
4943 op_cost(0);
4944 format %{ %}
4945 interface(REG_INTER);
4946 %}
4947
4948 // Generic vector class. This will be used for
4949 // all vector operands, including NEON and SVE.
4950 operand vReg()
4951 %{
4952 constraint(ALLOC_IN_RC(dynamic));
4953 match(VecA);
4954 match(VecD);
4955 match(VecX);
4956
4957 op_cost(0);
4958 format %{ %}
4959 interface(REG_INTER);
4960 %}
4961
4962 operand vReg_V10()
4963 %{
4964 constraint(ALLOC_IN_RC(v10_veca_reg));
4965 match(vReg);
4966
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 operand vReg_V11()
4973 %{
4974 constraint(ALLOC_IN_RC(v11_veca_reg));
4975 match(vReg);
4976
4977 op_cost(0);
4978 format %{ %}
4979 interface(REG_INTER);
4980 %}
4981
4982 operand vReg_V12()
4983 %{
4984 constraint(ALLOC_IN_RC(v12_veca_reg));
4985 match(vReg);
4986
4987 op_cost(0);
4988 format %{ %}
4989 interface(REG_INTER);
4990 %}
4991
4992 operand vReg_V13()
4993 %{
4994 constraint(ALLOC_IN_RC(v13_veca_reg));
4995 match(vReg);
4996
4997 op_cost(0);
4998 format %{ %}
4999 interface(REG_INTER);
5000 %}
5001
5002 operand vReg_V17()
5003 %{
5004 constraint(ALLOC_IN_RC(v17_veca_reg));
5005 match(vReg);
5006
5007 op_cost(0);
5008 format %{ %}
5009 interface(REG_INTER);
5010 %}
5011
5012 operand vReg_V18()
5013 %{
5014 constraint(ALLOC_IN_RC(v18_veca_reg));
5015 match(vReg);
5016
5017 op_cost(0);
5018 format %{ %}
5019 interface(REG_INTER);
5020 %}
5021
5022 operand vReg_V23()
5023 %{
5024 constraint(ALLOC_IN_RC(v23_veca_reg));
5025 match(vReg);
5026
5027 op_cost(0);
5028 format %{ %}
5029 interface(REG_INTER);
5030 %}
5031
5032 operand vReg_V24()
5033 %{
5034 constraint(ALLOC_IN_RC(v24_veca_reg));
5035 match(vReg);
5036
5037 op_cost(0);
5038 format %{ %}
5039 interface(REG_INTER);
5040 %}
5041
5042 operand vecA()
5043 %{
5044 constraint(ALLOC_IN_RC(vectora_reg));
5045 match(VecA);
5046
5047 op_cost(0);
5048 format %{ %}
5049 interface(REG_INTER);
5050 %}
5051
5052 operand vecD()
5053 %{
5054 constraint(ALLOC_IN_RC(vectord_reg));
5055 match(VecD);
5056
5057 op_cost(0);
5058 format %{ %}
5059 interface(REG_INTER);
5060 %}
5061
5062 operand vecX()
5063 %{
5064 constraint(ALLOC_IN_RC(vectorx_reg));
5065 match(VecX);
5066
5067 op_cost(0);
5068 format %{ %}
5069 interface(REG_INTER);
5070 %}
5071
5072 operand vRegD_V0()
5073 %{
5074 constraint(ALLOC_IN_RC(v0_reg));
5075 match(RegD);
5076 op_cost(0);
5077 format %{ %}
5078 interface(REG_INTER);
5079 %}
5080
5081 operand vRegD_V1()
5082 %{
5083 constraint(ALLOC_IN_RC(v1_reg));
5084 match(RegD);
5085 op_cost(0);
5086 format %{ %}
5087 interface(REG_INTER);
5088 %}
5089
5090 operand vRegD_V2()
5091 %{
5092 constraint(ALLOC_IN_RC(v2_reg));
5093 match(RegD);
5094 op_cost(0);
5095 format %{ %}
5096 interface(REG_INTER);
5097 %}
5098
5099 operand vRegD_V3()
5100 %{
5101 constraint(ALLOC_IN_RC(v3_reg));
5102 match(RegD);
5103 op_cost(0);
5104 format %{ %}
5105 interface(REG_INTER);
5106 %}
5107
5108 operand vRegD_V4()
5109 %{
5110 constraint(ALLOC_IN_RC(v4_reg));
5111 match(RegD);
5112 op_cost(0);
5113 format %{ %}
5114 interface(REG_INTER);
5115 %}
5116
5117 operand vRegD_V5()
5118 %{
5119 constraint(ALLOC_IN_RC(v5_reg));
5120 match(RegD);
5121 op_cost(0);
5122 format %{ %}
5123 interface(REG_INTER);
5124 %}
5125
5126 operand vRegD_V6()
5127 %{
5128 constraint(ALLOC_IN_RC(v6_reg));
5129 match(RegD);
5130 op_cost(0);
5131 format %{ %}
5132 interface(REG_INTER);
5133 %}
5134
5135 operand vRegD_V7()
5136 %{
5137 constraint(ALLOC_IN_RC(v7_reg));
5138 match(RegD);
5139 op_cost(0);
5140 format %{ %}
5141 interface(REG_INTER);
5142 %}
5143
5144 operand vRegD_V12()
5145 %{
5146 constraint(ALLOC_IN_RC(v12_reg));
5147 match(RegD);
5148 op_cost(0);
5149 format %{ %}
5150 interface(REG_INTER);
5151 %}
5152
5153 operand vRegD_V13()
5154 %{
5155 constraint(ALLOC_IN_RC(v13_reg));
5156 match(RegD);
5157 op_cost(0);
5158 format %{ %}
5159 interface(REG_INTER);
5160 %}
5161
5162 operand pReg()
5163 %{
5164 constraint(ALLOC_IN_RC(pr_reg));
5165 match(RegVectMask);
5166 match(pRegGov);
5167 op_cost(0);
5168 format %{ %}
5169 interface(REG_INTER);
5170 %}
5171
5172 operand pRegGov()
5173 %{
5174 constraint(ALLOC_IN_RC(gov_pr));
5175 match(RegVectMask);
5176 match(pReg);
5177 op_cost(0);
5178 format %{ %}
5179 interface(REG_INTER);
5180 %}
5181
5182 operand pRegGov_P0()
5183 %{
5184 constraint(ALLOC_IN_RC(p0_reg));
5185 match(RegVectMask);
5186 op_cost(0);
5187 format %{ %}
5188 interface(REG_INTER);
5189 %}
5190
5191 operand pRegGov_P1()
5192 %{
5193 constraint(ALLOC_IN_RC(p1_reg));
5194 match(RegVectMask);
5195 op_cost(0);
5196 format %{ %}
5197 interface(REG_INTER);
5198 %}
5199
5200 // Flags register, used as output of signed compare instructions
5201
5202 // note that on AArch64 we also use this register as the output for
5203 // for floating point compare instructions (CmpF CmpD). this ensures
5204 // that ordered inequality tests use GT, GE, LT or LE none of which
5205 // pass through cases where the result is unordered i.e. one or both
5206 // inputs to the compare is a NaN. this means that the ideal code can
5207 // replace e.g. a GT with an LE and not end up capturing the NaN case
5208 // (where the comparison should always fail). EQ and NE tests are
5209 // always generated in ideal code so that unordered folds into the NE
5210 // case, matching the behaviour of AArch64 NE.
5211 //
5212 // This differs from x86 where the outputs of FP compares use a
5213 // special FP flags registers and where compares based on this
5214 // register are distinguished into ordered inequalities (cmpOpUCF) and
5215 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5216 // to explicitly handle the unordered case in branches. x86 also has
5217 // to include extra CMoveX rules to accept a cmpOpUCF input.
5218
5219 operand rFlagsReg()
5220 %{
5221 constraint(ALLOC_IN_RC(int_flags));
5222 match(RegFlags);
5223
5224 op_cost(0);
5225 format %{ "RFLAGS" %}
5226 interface(REG_INTER);
5227 %}
5228
5229 // Flags register, used as output of unsigned compare instructions
5230 operand rFlagsRegU()
5231 %{
5232 constraint(ALLOC_IN_RC(int_flags));
5233 match(RegFlags);
5234
5235 op_cost(0);
5236 format %{ "RFLAGSU" %}
5237 interface(REG_INTER);
5238 %}
5239
5240 // Special Registers
5241
5242 // Method Register
5243 operand inline_cache_RegP(iRegP reg)
5244 %{
5245 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5246 match(reg);
5247 match(iRegPNoSp);
5248 op_cost(0);
5249 format %{ %}
5250 interface(REG_INTER);
5251 %}
5252
5253 // Thread Register
5254 operand thread_RegP(iRegP reg)
5255 %{
5256 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5257 match(reg);
5258 op_cost(0);
5259 format %{ %}
5260 interface(REG_INTER);
5261 %}
5262
5263 //----------Memory Operands----------------------------------------------------
5264
5265 operand indirect(iRegP reg)
5266 %{
5267 constraint(ALLOC_IN_RC(ptr_reg));
5268 match(reg);
5269 op_cost(0);
5270 format %{ "[$reg]" %}
5271 interface(MEMORY_INTER) %{
5272 base($reg);
5273 index(0xffffffff);
5274 scale(0x0);
5275 disp(0x0);
5276 %}
5277 %}
5278
5279 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5280 %{
5281 constraint(ALLOC_IN_RC(ptr_reg));
5282 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5283 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5284 op_cost(0);
5285 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5286 interface(MEMORY_INTER) %{
5287 base($reg);
5288 index($ireg);
5289 scale($scale);
5290 disp(0x0);
5291 %}
5292 %}
5293
5294 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5295 %{
5296 constraint(ALLOC_IN_RC(ptr_reg));
5297 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5298 match(AddP reg (LShiftL lreg scale));
5299 op_cost(0);
5300 format %{ "$reg, $lreg lsl($scale)" %}
5301 interface(MEMORY_INTER) %{
5302 base($reg);
5303 index($lreg);
5304 scale($scale);
5305 disp(0x0);
5306 %}
5307 %}
5308
5309 operand indIndexI2L(iRegP reg, iRegI ireg)
5310 %{
5311 constraint(ALLOC_IN_RC(ptr_reg));
5312 match(AddP reg (ConvI2L ireg));
5313 op_cost(0);
5314 format %{ "$reg, $ireg, 0, I2L" %}
5315 interface(MEMORY_INTER) %{
5316 base($reg);
5317 index($ireg);
5318 scale(0x0);
5319 disp(0x0);
5320 %}
5321 %}
5322
5323 operand indIndex(iRegP reg, iRegL lreg)
5324 %{
5325 constraint(ALLOC_IN_RC(ptr_reg));
5326 match(AddP reg lreg);
5327 op_cost(0);
5328 format %{ "$reg, $lreg" %}
5329 interface(MEMORY_INTER) %{
5330 base($reg);
5331 index($lreg);
5332 scale(0x0);
5333 disp(0x0);
5334 %}
5335 %}
5336
5337 operand indOffI1(iRegP reg, immIOffset1 off)
5338 %{
5339 constraint(ALLOC_IN_RC(ptr_reg));
5340 match(AddP reg off);
5341 op_cost(0);
5342 format %{ "[$reg, $off]" %}
5343 interface(MEMORY_INTER) %{
5344 base($reg);
5345 index(0xffffffff);
5346 scale(0x0);
5347 disp($off);
5348 %}
5349 %}
5350
5351 operand indOffI2(iRegP reg, immIOffset2 off)
5352 %{
5353 constraint(ALLOC_IN_RC(ptr_reg));
5354 match(AddP reg off);
5355 op_cost(0);
5356 format %{ "[$reg, $off]" %}
5357 interface(MEMORY_INTER) %{
5358 base($reg);
5359 index(0xffffffff);
5360 scale(0x0);
5361 disp($off);
5362 %}
5363 %}
5364
5365 operand indOffI4(iRegP reg, immIOffset4 off)
5366 %{
5367 constraint(ALLOC_IN_RC(ptr_reg));
5368 match(AddP reg off);
5369 op_cost(0);
5370 format %{ "[$reg, $off]" %}
5371 interface(MEMORY_INTER) %{
5372 base($reg);
5373 index(0xffffffff);
5374 scale(0x0);
5375 disp($off);
5376 %}
5377 %}
5378
5379 operand indOffI8(iRegP reg, immIOffset8 off)
5380 %{
5381 constraint(ALLOC_IN_RC(ptr_reg));
5382 match(AddP reg off);
5383 op_cost(0);
5384 format %{ "[$reg, $off]" %}
5385 interface(MEMORY_INTER) %{
5386 base($reg);
5387 index(0xffffffff);
5388 scale(0x0);
5389 disp($off);
5390 %}
5391 %}
5392
5393 operand indOffI16(iRegP reg, immIOffset16 off)
5394 %{
5395 constraint(ALLOC_IN_RC(ptr_reg));
5396 match(AddP reg off);
5397 op_cost(0);
5398 format %{ "[$reg, $off]" %}
5399 interface(MEMORY_INTER) %{
5400 base($reg);
5401 index(0xffffffff);
5402 scale(0x0);
5403 disp($off);
5404 %}
5405 %}
5406
5407 operand indOffL1(iRegP reg, immLoffset1 off)
5408 %{
5409 constraint(ALLOC_IN_RC(ptr_reg));
5410 match(AddP reg off);
5411 op_cost(0);
5412 format %{ "[$reg, $off]" %}
5413 interface(MEMORY_INTER) %{
5414 base($reg);
5415 index(0xffffffff);
5416 scale(0x0);
5417 disp($off);
5418 %}
5419 %}
5420
5421 operand indOffL2(iRegP reg, immLoffset2 off)
5422 %{
5423 constraint(ALLOC_IN_RC(ptr_reg));
5424 match(AddP reg off);
5425 op_cost(0);
5426 format %{ "[$reg, $off]" %}
5427 interface(MEMORY_INTER) %{
5428 base($reg);
5429 index(0xffffffff);
5430 scale(0x0);
5431 disp($off);
5432 %}
5433 %}
5434
5435 operand indOffL4(iRegP reg, immLoffset4 off)
5436 %{
5437 constraint(ALLOC_IN_RC(ptr_reg));
5438 match(AddP reg off);
5439 op_cost(0);
5440 format %{ "[$reg, $off]" %}
5441 interface(MEMORY_INTER) %{
5442 base($reg);
5443 index(0xffffffff);
5444 scale(0x0);
5445 disp($off);
5446 %}
5447 %}
5448
5449 operand indOffL8(iRegP reg, immLoffset8 off)
5450 %{
5451 constraint(ALLOC_IN_RC(ptr_reg));
5452 match(AddP reg off);
5453 op_cost(0);
5454 format %{ "[$reg, $off]" %}
5455 interface(MEMORY_INTER) %{
5456 base($reg);
5457 index(0xffffffff);
5458 scale(0x0);
5459 disp($off);
5460 %}
5461 %}
5462
5463 operand indOffL16(iRegP reg, immLoffset16 off)
5464 %{
5465 constraint(ALLOC_IN_RC(ptr_reg));
5466 match(AddP reg off);
5467 op_cost(0);
5468 format %{ "[$reg, $off]" %}
5469 interface(MEMORY_INTER) %{
5470 base($reg);
5471 index(0xffffffff);
5472 scale(0x0);
5473 disp($off);
5474 %}
5475 %}
5476
5477 operand indirectX2P(iRegL reg)
5478 %{
5479 constraint(ALLOC_IN_RC(ptr_reg));
5480 match(CastX2P reg);
5481 op_cost(0);
5482 format %{ "[$reg]\t# long -> ptr" %}
5483 interface(MEMORY_INTER) %{
5484 base($reg);
5485 index(0xffffffff);
5486 scale(0x0);
5487 disp(0x0);
5488 %}
5489 %}
5490
5491 operand indOffX2P(iRegL reg, immLOffset off)
5492 %{
5493 constraint(ALLOC_IN_RC(ptr_reg));
5494 match(AddP (CastX2P reg) off);
5495 op_cost(0);
5496 format %{ "[$reg, $off]\t# long -> ptr" %}
5497 interface(MEMORY_INTER) %{
5498 base($reg);
5499 index(0xffffffff);
5500 scale(0x0);
5501 disp($off);
5502 %}
5503 %}
5504
5505 operand indirectN(iRegN reg)
5506 %{
5507 predicate(CompressedOops::shift() == 0);
5508 constraint(ALLOC_IN_RC(ptr_reg));
5509 match(DecodeN reg);
5510 op_cost(0);
5511 format %{ "[$reg]\t# narrow" %}
5512 interface(MEMORY_INTER) %{
5513 base($reg);
5514 index(0xffffffff);
5515 scale(0x0);
5516 disp(0x0);
5517 %}
5518 %}
5519
5520 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5521 %{
5522 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5523 constraint(ALLOC_IN_RC(ptr_reg));
5524 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5525 op_cost(0);
5526 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5527 interface(MEMORY_INTER) %{
5528 base($reg);
5529 index($ireg);
5530 scale($scale);
5531 disp(0x0);
5532 %}
5533 %}
5534
5535 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5536 %{
5537 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5538 constraint(ALLOC_IN_RC(ptr_reg));
5539 match(AddP (DecodeN reg) (LShiftL lreg scale));
5540 op_cost(0);
5541 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5542 interface(MEMORY_INTER) %{
5543 base($reg);
5544 index($lreg);
5545 scale($scale);
5546 disp(0x0);
5547 %}
5548 %}
5549
5550 operand indIndexI2LN(iRegN reg, iRegI ireg)
5551 %{
5552 predicate(CompressedOops::shift() == 0);
5553 constraint(ALLOC_IN_RC(ptr_reg));
5554 match(AddP (DecodeN reg) (ConvI2L ireg));
5555 op_cost(0);
5556 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5557 interface(MEMORY_INTER) %{
5558 base($reg);
5559 index($ireg);
5560 scale(0x0);
5561 disp(0x0);
5562 %}
5563 %}
5564
5565 operand indIndexN(iRegN reg, iRegL lreg)
5566 %{
5567 predicate(CompressedOops::shift() == 0);
5568 constraint(ALLOC_IN_RC(ptr_reg));
5569 match(AddP (DecodeN reg) lreg);
5570 op_cost(0);
5571 format %{ "$reg, $lreg\t# narrow" %}
5572 interface(MEMORY_INTER) %{
5573 base($reg);
5574 index($lreg);
5575 scale(0x0);
5576 disp(0x0);
5577 %}
5578 %}
5579
5580 operand indOffIN(iRegN reg, immIOffset off)
5581 %{
5582 predicate(CompressedOops::shift() == 0);
5583 constraint(ALLOC_IN_RC(ptr_reg));
5584 match(AddP (DecodeN reg) off);
5585 op_cost(0);
5586 format %{ "[$reg, $off]\t# narrow" %}
5587 interface(MEMORY_INTER) %{
5588 base($reg);
5589 index(0xffffffff);
5590 scale(0x0);
5591 disp($off);
5592 %}
5593 %}
5594
5595 operand indOffLN(iRegN reg, immLOffset off)
5596 %{
5597 predicate(CompressedOops::shift() == 0);
5598 constraint(ALLOC_IN_RC(ptr_reg));
5599 match(AddP (DecodeN reg) off);
5600 op_cost(0);
5601 format %{ "[$reg, $off]\t# narrow" %}
5602 interface(MEMORY_INTER) %{
5603 base($reg);
5604 index(0xffffffff);
5605 scale(0x0);
5606 disp($off);
5607 %}
5608 %}
5609
5610
5611 //----------Special Memory Operands--------------------------------------------
5612 // Stack Slot Operand - This operand is used for loading and storing temporary
5613 // values on the stack where a match requires a value to
5614 // flow through memory.
5615 operand stackSlotP(sRegP reg)
5616 %{
5617 constraint(ALLOC_IN_RC(stack_slots));
5618 op_cost(100);
5619 // No match rule because this operand is only generated in matching
5620 // match(RegP);
5621 format %{ "[$reg]" %}
5622 interface(MEMORY_INTER) %{
5623 base(0x1e); // RSP
5624 index(0x0); // No Index
5625 scale(0x0); // No Scale
5626 disp($reg); // Stack Offset
5627 %}
5628 %}
5629
5630 operand stackSlotI(sRegI reg)
5631 %{
5632 constraint(ALLOC_IN_RC(stack_slots));
5633 // No match rule because this operand is only generated in matching
5634 // match(RegI);
5635 format %{ "[$reg]" %}
5636 interface(MEMORY_INTER) %{
5637 base(0x1e); // RSP
5638 index(0x0); // No Index
5639 scale(0x0); // No Scale
5640 disp($reg); // Stack Offset
5641 %}
5642 %}
5643
5644 operand stackSlotF(sRegF reg)
5645 %{
5646 constraint(ALLOC_IN_RC(stack_slots));
5647 // No match rule because this operand is only generated in matching
5648 // match(RegF);
5649 format %{ "[$reg]" %}
5650 interface(MEMORY_INTER) %{
5651 base(0x1e); // RSP
5652 index(0x0); // No Index
5653 scale(0x0); // No Scale
5654 disp($reg); // Stack Offset
5655 %}
5656 %}
5657
5658 operand stackSlotD(sRegD reg)
5659 %{
5660 constraint(ALLOC_IN_RC(stack_slots));
5661 // No match rule because this operand is only generated in matching
5662 // match(RegD);
5663 format %{ "[$reg]" %}
5664 interface(MEMORY_INTER) %{
5665 base(0x1e); // RSP
5666 index(0x0); // No Index
5667 scale(0x0); // No Scale
5668 disp($reg); // Stack Offset
5669 %}
5670 %}
5671
5672 operand stackSlotL(sRegL reg)
5673 %{
5674 constraint(ALLOC_IN_RC(stack_slots));
5675 // No match rule because this operand is only generated in matching
5676 // match(RegL);
5677 format %{ "[$reg]" %}
5678 interface(MEMORY_INTER) %{
5679 base(0x1e); // RSP
5680 index(0x0); // No Index
5681 scale(0x0); // No Scale
5682 disp($reg); // Stack Offset
5683 %}
5684 %}
5685
5686 // Operands for expressing Control Flow
5687 // NOTE: Label is a predefined operand which should not be redefined in
5688 // the AD file. It is generically handled within the ADLC.
5689
5690 //----------Conditional Branch Operands----------------------------------------
5691 // Comparison Op - This is the operation of the comparison, and is limited to
5692 // the following set of codes:
5693 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5694 //
5695 // Other attributes of the comparison, such as unsignedness, are specified
5696 // by the comparison instruction that sets a condition code flags register.
5697 // That result is represented by a flags operand whose subtype is appropriate
5698 // to the unsignedness (etc.) of the comparison.
5699 //
5700 // Later, the instruction which matches both the Comparison Op (a Bool) and
5701 // the flags (produced by the Cmp) specifies the coding of the comparison op
5702 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5703
5704 // used for signed integral comparisons and fp comparisons
5705
5706 operand cmpOp()
5707 %{
5708 match(Bool);
5709
5710 format %{ "" %}
5711 interface(COND_INTER) %{
5712 equal(0x0, "eq");
5713 not_equal(0x1, "ne");
5714 less(0xb, "lt");
5715 greater_equal(0xa, "ge");
5716 less_equal(0xd, "le");
5717 greater(0xc, "gt");
5718 overflow(0x6, "vs");
5719 no_overflow(0x7, "vc");
5720 %}
5721 %}
5722
5723 // used for unsigned integral comparisons
5724
5725 operand cmpOpU()
5726 %{
5727 match(Bool);
5728
5729 format %{ "" %}
5730 interface(COND_INTER) %{
5731 equal(0x0, "eq");
5732 not_equal(0x1, "ne");
5733 less(0x3, "lo");
5734 greater_equal(0x2, "hs");
5735 less_equal(0x9, "ls");
5736 greater(0x8, "hi");
5737 overflow(0x6, "vs");
5738 no_overflow(0x7, "vc");
5739 %}
5740 %}
5741
5742 // used for certain integral comparisons which can be
5743 // converted to cbxx or tbxx instructions
5744
5745 operand cmpOpEqNe()
5746 %{
5747 match(Bool);
5748 op_cost(0);
5749 predicate(n->as_Bool()->_test._test == BoolTest::ne
5750 || n->as_Bool()->_test._test == BoolTest::eq);
5751
5752 format %{ "" %}
5753 interface(COND_INTER) %{
5754 equal(0x0, "eq");
5755 not_equal(0x1, "ne");
5756 less(0xb, "lt");
5757 greater_equal(0xa, "ge");
5758 less_equal(0xd, "le");
5759 greater(0xc, "gt");
5760 overflow(0x6, "vs");
5761 no_overflow(0x7, "vc");
5762 %}
5763 %}
5764
5765 // used for certain integral comparisons which can be
5766 // converted to cbxx or tbxx instructions
5767
5768 operand cmpOpLtGe()
5769 %{
5770 match(Bool);
5771 op_cost(0);
5772
5773 predicate(n->as_Bool()->_test._test == BoolTest::lt
5774 || n->as_Bool()->_test._test == BoolTest::ge);
5775
5776 format %{ "" %}
5777 interface(COND_INTER) %{
5778 equal(0x0, "eq");
5779 not_equal(0x1, "ne");
5780 less(0xb, "lt");
5781 greater_equal(0xa, "ge");
5782 less_equal(0xd, "le");
5783 greater(0xc, "gt");
5784 overflow(0x6, "vs");
5785 no_overflow(0x7, "vc");
5786 %}
5787 %}
5788
5789 // used for certain unsigned integral comparisons which can be
5790 // converted to cbxx or tbxx instructions
5791
5792 operand cmpOpUEqNeLeGt()
5793 %{
5794 match(Bool);
5795 op_cost(0);
5796
5797 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5798 n->as_Bool()->_test._test == BoolTest::ne ||
5799 n->as_Bool()->_test._test == BoolTest::le ||
5800 n->as_Bool()->_test._test == BoolTest::gt);
5801
5802 format %{ "" %}
5803 interface(COND_INTER) %{
5804 equal(0x0, "eq");
5805 not_equal(0x1, "ne");
5806 less(0x3, "lo");
5807 greater_equal(0x2, "hs");
5808 less_equal(0x9, "ls");
5809 greater(0x8, "hi");
5810 overflow(0x6, "vs");
5811 no_overflow(0x7, "vc");
5812 %}
5813 %}
5814
5815 // Special operand allowing long args to int ops to be truncated for free
5816
5817 operand iRegL2I(iRegL reg) %{
5818
5819 op_cost(0);
5820
5821 match(ConvL2I reg);
5822
5823 format %{ "l2i($reg)" %}
5824
5825 interface(REG_INTER)
5826 %}
5827
5828 operand iRegL2P(iRegL reg) %{
5829
5830 op_cost(0);
5831
5832 match(CastX2P reg);
5833
5834 format %{ "l2p($reg)" %}
5835
5836 interface(REG_INTER)
5837 %}
5838
5839 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5840 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5841 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5842 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5843
5844 //----------OPERAND CLASSES----------------------------------------------------
5845 // Operand Classes are groups of operands that are used as to simplify
5846 // instruction definitions by not requiring the AD writer to specify
5847 // separate instructions for every form of operand when the
5848 // instruction accepts multiple operand types with the same basic
5849 // encoding and format. The classic case of this is memory operands.
5850
5851 // memory is used to define read/write location for load/store
5852 // instruction defs. we can turn a memory op into an Address
5853
5854 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5855 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5856
5857 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5858 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5859
5860 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5861 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5862
5863 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5864 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5865
5866 // All of the memory operands. For the pipeline description.
5867 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5868 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5869 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5870
5871
5872 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5873 // operations. it allows the src to be either an iRegI or a (ConvL2I
5874 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5875 // can be elided because the 32-bit instruction will just employ the
5876 // lower 32 bits anyway.
5877 //
5878 // n.b. this does not elide all L2I conversions. if the truncated
5879 // value is consumed by more than one operation then the ConvL2I
5880 // cannot be bundled into the consuming nodes so an l2i gets planted
5881 // (actually a movw $dst $src) and the downstream instructions consume
5882 // the result of the l2i as an iRegI input. That's a shame since the
5883 // movw is actually redundant but its not too costly.
5884
5885 opclass iRegIorL2I(iRegI, iRegL2I);
5886 opclass iRegPorL2P(iRegP, iRegL2P);
5887
5888 //----------PIPELINE-----------------------------------------------------------
5889 // Rules which define the behavior of the target architectures pipeline.
5890
5891 // For specific pipelines, eg A53, define the stages of that pipeline
5892 //pipe_desc(ISS, EX1, EX2, WR);
5893 #define ISS S0
5894 #define EX1 S1
5895 #define EX2 S2
5896 #define WR S3
5897
5898 // Integer ALU reg operation
5899 pipeline %{
5900
5901 attributes %{
5902 // ARM instructions are of fixed length
5903 fixed_size_instructions; // Fixed size instructions TODO does
5904 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5905 // ARM instructions come in 32-bit word units
5906 instruction_unit_size = 4; // An instruction is 4 bytes long
5907 instruction_fetch_unit_size = 64; // The processor fetches one line
5908 instruction_fetch_units = 1; // of 64 bytes
5909 %}
5910
5911 // We don't use an actual pipeline model so don't care about resources
5912 // or description. we do use pipeline classes to introduce fixed
5913 // latencies
5914
5915 //----------RESOURCES----------------------------------------------------------
5916 // Resources are the functional units available to the machine
5917
5918 resources( INS0, INS1, INS01 = INS0 | INS1,
5919 ALU0, ALU1, ALU = ALU0 | ALU1,
5920 MAC,
5921 DIV,
5922 BRANCH,
5923 LDST,
5924 NEON_FP);
5925
5926 //----------PIPELINE DESCRIPTION-----------------------------------------------
5927 // Pipeline Description specifies the stages in the machine's pipeline
5928
5929 // Define the pipeline as a generic 6 stage pipeline
5930 pipe_desc(S0, S1, S2, S3, S4, S5);
5931
5932 //----------PIPELINE CLASSES---------------------------------------------------
5933 // Pipeline Classes describe the stages in which input and output are
5934 // referenced by the hardware pipeline.
5935
5936 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5937 %{
5938 single_instruction;
5939 src1 : S1(read);
5940 src2 : S2(read);
5941 dst : S5(write);
5942 INS01 : ISS;
5943 NEON_FP : S5;
5944 %}
5945
5946 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5947 %{
5948 single_instruction;
5949 src1 : S1(read);
5950 src2 : S2(read);
5951 dst : S5(write);
5952 INS01 : ISS;
5953 NEON_FP : S5;
5954 %}
5955
5956 pipe_class fp_uop_s(vRegF dst, vRegF src)
5957 %{
5958 single_instruction;
5959 src : S1(read);
5960 dst : S5(write);
5961 INS01 : ISS;
5962 NEON_FP : S5;
5963 %}
5964
5965 pipe_class fp_uop_d(vRegD dst, vRegD src)
5966 %{
5967 single_instruction;
5968 src : S1(read);
5969 dst : S5(write);
5970 INS01 : ISS;
5971 NEON_FP : S5;
5972 %}
5973
5974 pipe_class fp_d2f(vRegF dst, vRegD src)
5975 %{
5976 single_instruction;
5977 src : S1(read);
5978 dst : S5(write);
5979 INS01 : ISS;
5980 NEON_FP : S5;
5981 %}
5982
5983 pipe_class fp_f2d(vRegD dst, vRegF src)
5984 %{
5985 single_instruction;
5986 src : S1(read);
5987 dst : S5(write);
5988 INS01 : ISS;
5989 NEON_FP : S5;
5990 %}
5991
5992 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5993 %{
5994 single_instruction;
5995 src : S1(read);
5996 dst : S5(write);
5997 INS01 : ISS;
5998 NEON_FP : S5;
5999 %}
6000
6001 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6002 %{
6003 single_instruction;
6004 src : S1(read);
6005 dst : S5(write);
6006 INS01 : ISS;
6007 NEON_FP : S5;
6008 %}
6009
6010 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6011 %{
6012 single_instruction;
6013 src : S1(read);
6014 dst : S5(write);
6015 INS01 : ISS;
6016 NEON_FP : S5;
6017 %}
6018
6019 pipe_class fp_l2f(vRegF dst, iRegL src)
6020 %{
6021 single_instruction;
6022 src : S1(read);
6023 dst : S5(write);
6024 INS01 : ISS;
6025 NEON_FP : S5;
6026 %}
6027
6028 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6029 %{
6030 single_instruction;
6031 src : S1(read);
6032 dst : S5(write);
6033 INS01 : ISS;
6034 NEON_FP : S5;
6035 %}
6036
6037 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6038 %{
6039 single_instruction;
6040 src : S1(read);
6041 dst : S5(write);
6042 INS01 : ISS;
6043 NEON_FP : S5;
6044 %}
6045
6046 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6047 %{
6048 single_instruction;
6049 src : S1(read);
6050 dst : S5(write);
6051 INS01 : ISS;
6052 NEON_FP : S5;
6053 %}
6054
6055 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6056 %{
6057 single_instruction;
6058 src : S1(read);
6059 dst : S5(write);
6060 INS01 : ISS;
6061 NEON_FP : S5;
6062 %}
6063
6064 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6065 %{
6066 single_instruction;
6067 src1 : S1(read);
6068 src2 : S2(read);
6069 dst : S5(write);
6070 INS0 : ISS;
6071 NEON_FP : S5;
6072 %}
6073
6074 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6075 %{
6076 single_instruction;
6077 src1 : S1(read);
6078 src2 : S2(read);
6079 dst : S5(write);
6080 INS0 : ISS;
6081 NEON_FP : S5;
6082 %}
6083
6084 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6085 %{
6086 single_instruction;
6087 cr : S1(read);
6088 src1 : S1(read);
6089 src2 : S1(read);
6090 dst : S3(write);
6091 INS01 : ISS;
6092 NEON_FP : S3;
6093 %}
6094
6095 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6096 %{
6097 single_instruction;
6098 cr : S1(read);
6099 src1 : S1(read);
6100 src2 : S1(read);
6101 dst : S3(write);
6102 INS01 : ISS;
6103 NEON_FP : S3;
6104 %}
6105
6106 pipe_class fp_imm_s(vRegF dst)
6107 %{
6108 single_instruction;
6109 dst : S3(write);
6110 INS01 : ISS;
6111 NEON_FP : S3;
6112 %}
6113
6114 pipe_class fp_imm_d(vRegD dst)
6115 %{
6116 single_instruction;
6117 dst : S3(write);
6118 INS01 : ISS;
6119 NEON_FP : S3;
6120 %}
6121
6122 pipe_class fp_load_constant_s(vRegF dst)
6123 %{
6124 single_instruction;
6125 dst : S4(write);
6126 INS01 : ISS;
6127 NEON_FP : S4;
6128 %}
6129
6130 pipe_class fp_load_constant_d(vRegD dst)
6131 %{
6132 single_instruction;
6133 dst : S4(write);
6134 INS01 : ISS;
6135 NEON_FP : S4;
6136 %}
6137
6138 //------- Integer ALU operations --------------------------
6139
6140 // Integer ALU reg-reg operation
6141 // Operands needed in EX1, result generated in EX2
6142 // Eg. ADD x0, x1, x2
6143 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6144 %{
6145 single_instruction;
6146 dst : EX2(write);
6147 src1 : EX1(read);
6148 src2 : EX1(read);
6149 INS01 : ISS; // Dual issue as instruction 0 or 1
6150 ALU : EX2;
6151 %}
6152
6153 // Integer ALU reg-reg operation with constant shift
6154 // Shifted register must be available in LATE_ISS instead of EX1
6155 // Eg. ADD x0, x1, x2, LSL #2
6156 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6157 %{
6158 single_instruction;
6159 dst : EX2(write);
6160 src1 : EX1(read);
6161 src2 : ISS(read);
6162 INS01 : ISS;
6163 ALU : EX2;
6164 %}
6165
6166 // Integer ALU reg operation with constant shift
6167 // Eg. LSL x0, x1, #shift
6168 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6169 %{
6170 single_instruction;
6171 dst : EX2(write);
6172 src1 : ISS(read);
6173 INS01 : ISS;
6174 ALU : EX2;
6175 %}
6176
6177 // Integer ALU reg-reg operation with variable shift
6178 // Both operands must be available in LATE_ISS instead of EX1
6179 // Result is available in EX1 instead of EX2
6180 // Eg. LSLV x0, x1, x2
6181 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6182 %{
6183 single_instruction;
6184 dst : EX1(write);
6185 src1 : ISS(read);
6186 src2 : ISS(read);
6187 INS01 : ISS;
6188 ALU : EX1;
6189 %}
6190
6191 // Integer ALU reg-reg operation with extract
6192 // As for _vshift above, but result generated in EX2
6193 // Eg. EXTR x0, x1, x2, #N
6194 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6195 %{
6196 single_instruction;
6197 dst : EX2(write);
6198 src1 : ISS(read);
6199 src2 : ISS(read);
6200 INS1 : ISS; // Can only dual issue as Instruction 1
6201 ALU : EX1;
6202 %}
6203
6204 // Integer ALU reg operation
6205 // Eg. NEG x0, x1
6206 pipe_class ialu_reg(iRegI dst, iRegI src)
6207 %{
6208 single_instruction;
6209 dst : EX2(write);
6210 src : EX1(read);
6211 INS01 : ISS;
6212 ALU : EX2;
6213 %}
6214
6215 // Integer ALU reg mmediate operation
6216 // Eg. ADD x0, x1, #N
6217 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6218 %{
6219 single_instruction;
6220 dst : EX2(write);
6221 src1 : EX1(read);
6222 INS01 : ISS;
6223 ALU : EX2;
6224 %}
6225
6226 // Integer ALU immediate operation (no source operands)
6227 // Eg. MOV x0, #N
6228 pipe_class ialu_imm(iRegI dst)
6229 %{
6230 single_instruction;
6231 dst : EX1(write);
6232 INS01 : ISS;
6233 ALU : EX1;
6234 %}
6235
6236 //------- Compare operation -------------------------------
6237
6238 // Compare reg-reg
6239 // Eg. CMP x0, x1
6240 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6241 %{
6242 single_instruction;
6243 // fixed_latency(16);
6244 cr : EX2(write);
6245 op1 : EX1(read);
6246 op2 : EX1(read);
6247 INS01 : ISS;
6248 ALU : EX2;
6249 %}
6250
6251 // Compare reg-reg
6252 // Eg. CMP x0, #N
6253 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6254 %{
6255 single_instruction;
6256 // fixed_latency(16);
6257 cr : EX2(write);
6258 op1 : EX1(read);
6259 INS01 : ISS;
6260 ALU : EX2;
6261 %}
6262
6263 //------- Conditional instructions ------------------------
6264
6265 // Conditional no operands
6266 // Eg. CSINC x0, zr, zr, <cond>
6267 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6268 %{
6269 single_instruction;
6270 cr : EX1(read);
6271 dst : EX2(write);
6272 INS01 : ISS;
6273 ALU : EX2;
6274 %}
6275
6276 // Conditional 2 operand
6277 // EG. CSEL X0, X1, X2, <cond>
6278 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6279 %{
6280 single_instruction;
6281 cr : EX1(read);
6282 src1 : EX1(read);
6283 src2 : EX1(read);
6284 dst : EX2(write);
6285 INS01 : ISS;
6286 ALU : EX2;
6287 %}
6288
6289 // Conditional 2 operand
6290 // EG. CSEL X0, X1, X2, <cond>
6291 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6292 %{
6293 single_instruction;
6294 cr : EX1(read);
6295 src : EX1(read);
6296 dst : EX2(write);
6297 INS01 : ISS;
6298 ALU : EX2;
6299 %}
6300
6301 //------- Multiply pipeline operations --------------------
6302
6303 // Multiply reg-reg
6304 // Eg. MUL w0, w1, w2
6305 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6306 %{
6307 single_instruction;
6308 dst : WR(write);
6309 src1 : ISS(read);
6310 src2 : ISS(read);
6311 INS01 : ISS;
6312 MAC : WR;
6313 %}
6314
6315 // Multiply accumulate
6316 // Eg. MADD w0, w1, w2, w3
6317 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6318 %{
6319 single_instruction;
6320 dst : WR(write);
6321 src1 : ISS(read);
6322 src2 : ISS(read);
6323 src3 : ISS(read);
6324 INS01 : ISS;
6325 MAC : WR;
6326 %}
6327
6328 // Eg. MUL w0, w1, w2
6329 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6330 %{
6331 single_instruction;
6332 fixed_latency(3); // Maximum latency for 64 bit mul
6333 dst : WR(write);
6334 src1 : ISS(read);
6335 src2 : ISS(read);
6336 INS01 : ISS;
6337 MAC : WR;
6338 %}
6339
6340 // Multiply accumulate
6341 // Eg. MADD w0, w1, w2, w3
6342 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6343 %{
6344 single_instruction;
6345 fixed_latency(3); // Maximum latency for 64 bit mul
6346 dst : WR(write);
6347 src1 : ISS(read);
6348 src2 : ISS(read);
6349 src3 : ISS(read);
6350 INS01 : ISS;
6351 MAC : WR;
6352 %}
6353
6354 //------- Divide pipeline operations --------------------
6355
6356 // Eg. SDIV w0, w1, w2
6357 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6358 %{
6359 single_instruction;
6360 fixed_latency(8); // Maximum latency for 32 bit divide
6361 dst : WR(write);
6362 src1 : ISS(read);
6363 src2 : ISS(read);
6364 INS0 : ISS; // Can only dual issue as instruction 0
6365 DIV : WR;
6366 %}
6367
6368 // Eg. SDIV x0, x1, x2
6369 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6370 %{
6371 single_instruction;
6372 fixed_latency(16); // Maximum latency for 64 bit divide
6373 dst : WR(write);
6374 src1 : ISS(read);
6375 src2 : ISS(read);
6376 INS0 : ISS; // Can only dual issue as instruction 0
6377 DIV : WR;
6378 %}
6379
6380 //------- Load pipeline operations ------------------------
6381
6382 // Load - prefetch
6383 // Eg. PFRM <mem>
6384 pipe_class iload_prefetch(memory mem)
6385 %{
6386 single_instruction;
6387 mem : ISS(read);
6388 INS01 : ISS;
6389 LDST : WR;
6390 %}
6391
6392 // Load - reg, mem
6393 // Eg. LDR x0, <mem>
6394 pipe_class iload_reg_mem(iRegI dst, memory mem)
6395 %{
6396 single_instruction;
6397 dst : WR(write);
6398 mem : ISS(read);
6399 INS01 : ISS;
6400 LDST : WR;
6401 %}
6402
6403 // Load - reg, reg
6404 // Eg. LDR x0, [sp, x1]
6405 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6406 %{
6407 single_instruction;
6408 dst : WR(write);
6409 src : ISS(read);
6410 INS01 : ISS;
6411 LDST : WR;
6412 %}
6413
6414 //------- Store pipeline operations -----------------------
6415
6416 // Store - zr, mem
6417 // Eg. STR zr, <mem>
6418 pipe_class istore_mem(memory mem)
6419 %{
6420 single_instruction;
6421 mem : ISS(read);
6422 INS01 : ISS;
6423 LDST : WR;
6424 %}
6425
6426 // Store - reg, mem
6427 // Eg. STR x0, <mem>
6428 pipe_class istore_reg_mem(iRegI src, memory mem)
6429 %{
6430 single_instruction;
6431 mem : ISS(read);
6432 src : EX2(read);
6433 INS01 : ISS;
6434 LDST : WR;
6435 %}
6436
6437 // Store - reg, reg
6438 // Eg. STR x0, [sp, x1]
6439 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6440 %{
6441 single_instruction;
6442 dst : ISS(read);
6443 src : EX2(read);
6444 INS01 : ISS;
6445 LDST : WR;
6446 %}
6447
6448 //------- Store pipeline operations -----------------------
6449
6450 // Branch
6451 pipe_class pipe_branch()
6452 %{
6453 single_instruction;
6454 INS01 : ISS;
6455 BRANCH : EX1;
6456 %}
6457
6458 // Conditional branch
6459 pipe_class pipe_branch_cond(rFlagsReg cr)
6460 %{
6461 single_instruction;
6462 cr : EX1(read);
6463 INS01 : ISS;
6464 BRANCH : EX1;
6465 %}
6466
6467 // Compare & Branch
6468 // EG. CBZ/CBNZ
6469 pipe_class pipe_cmp_branch(iRegI op1)
6470 %{
6471 single_instruction;
6472 op1 : EX1(read);
6473 INS01 : ISS;
6474 BRANCH : EX1;
6475 %}
6476
6477 //------- Synchronisation operations ----------------------
6478
6479 // Any operation requiring serialization.
6480 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6481 pipe_class pipe_serial()
6482 %{
6483 single_instruction;
6484 force_serialization;
6485 fixed_latency(16);
6486 INS01 : ISS(2); // Cannot dual issue with any other instruction
6487 LDST : WR;
6488 %}
6489
6490 // Generic big/slow expanded idiom - also serialized
6491 pipe_class pipe_slow()
6492 %{
6493 instruction_count(10);
6494 multiple_bundles;
6495 force_serialization;
6496 fixed_latency(16);
6497 INS01 : ISS(2); // Cannot dual issue with any other instruction
6498 LDST : WR;
6499 %}
6500
6501 // Empty pipeline class
6502 pipe_class pipe_class_empty()
6503 %{
6504 single_instruction;
6505 fixed_latency(0);
6506 %}
6507
6508 // Default pipeline class.
6509 pipe_class pipe_class_default()
6510 %{
6511 single_instruction;
6512 fixed_latency(2);
6513 %}
6514
6515 // Pipeline class for compares.
6516 pipe_class pipe_class_compare()
6517 %{
6518 single_instruction;
6519 fixed_latency(16);
6520 %}
6521
6522 // Pipeline class for memory operations.
6523 pipe_class pipe_class_memory()
6524 %{
6525 single_instruction;
6526 fixed_latency(16);
6527 %}
6528
6529 // Pipeline class for call.
6530 pipe_class pipe_class_call()
6531 %{
6532 single_instruction;
6533 fixed_latency(100);
6534 %}
6535
6536 // Define the class for the Nop node.
6537 define %{
6538 MachNop = pipe_class_empty;
6539 %}
6540
6541 %}
6542 //----------INSTRUCTIONS-------------------------------------------------------
6543 //
6544 // match -- States which machine-independent subtree may be replaced
6545 // by this instruction.
6546 // ins_cost -- The estimated cost of this instruction is used by instruction
6547 // selection to identify a minimum cost tree of machine
6548 // instructions that matches a tree of machine-independent
6549 // instructions.
6550 // format -- A string providing the disassembly for this instruction.
6551 // The value of an instruction's operand may be inserted
6552 // by referring to it with a '$' prefix.
6553 // opcode -- Three instruction opcodes may be provided. These are referred
6554 // to within an encode class as $primary, $secondary, and $tertiary
6555 // rrspectively. The primary opcode is commonly used to
6556 // indicate the type of machine instruction, while secondary
6557 // and tertiary are often used for prefix options or addressing
6558 // modes.
6559 // ins_encode -- A list of encode classes with parameters. The encode class
6560 // name must have been defined in an 'enc_class' specification
6561 // in the encode section of the architecture description.
6562
6563 // ============================================================================
6564 // Memory (Load/Store) Instructions
6565
6566 // Load Instructions
6567
6568 // Load Byte (8 bit signed)
6569 instruct loadB(iRegINoSp dst, memory1 mem)
6570 %{
6571 match(Set dst (LoadB mem));
6572 predicate(!needs_acquiring_load(n));
6573
6574 ins_cost(4 * INSN_COST);
6575 format %{ "ldrsbw $dst, $mem\t# byte" %}
6576
6577 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6578
6579 ins_pipe(iload_reg_mem);
6580 %}
6581
6582 // Load Byte (8 bit signed) into long
6583 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6584 %{
6585 match(Set dst (ConvI2L (LoadB mem)));
6586 predicate(!needs_acquiring_load(n->in(1)));
6587
6588 ins_cost(4 * INSN_COST);
6589 format %{ "ldrsb $dst, $mem\t# byte" %}
6590
6591 ins_encode(aarch64_enc_ldrsb(dst, mem));
6592
6593 ins_pipe(iload_reg_mem);
6594 %}
6595
6596 // Load Byte (8 bit unsigned)
6597 instruct loadUB(iRegINoSp dst, memory1 mem)
6598 %{
6599 match(Set dst (LoadUB mem));
6600 predicate(!needs_acquiring_load(n));
6601
6602 ins_cost(4 * INSN_COST);
6603 format %{ "ldrbw $dst, $mem\t# byte" %}
6604
6605 ins_encode(aarch64_enc_ldrb(dst, mem));
6606
6607 ins_pipe(iload_reg_mem);
6608 %}
6609
6610 // Load Byte (8 bit unsigned) into long
6611 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6612 %{
6613 match(Set dst (ConvI2L (LoadUB mem)));
6614 predicate(!needs_acquiring_load(n->in(1)));
6615
6616 ins_cost(4 * INSN_COST);
6617 format %{ "ldrb $dst, $mem\t# byte" %}
6618
6619 ins_encode(aarch64_enc_ldrb(dst, mem));
6620
6621 ins_pipe(iload_reg_mem);
6622 %}
6623
6624 // Load Short (16 bit signed)
6625 instruct loadS(iRegINoSp dst, memory2 mem)
6626 %{
6627 match(Set dst (LoadS mem));
6628 predicate(!needs_acquiring_load(n));
6629
6630 ins_cost(4 * INSN_COST);
6631 format %{ "ldrshw $dst, $mem\t# short" %}
6632
6633 ins_encode(aarch64_enc_ldrshw(dst, mem));
6634
6635 ins_pipe(iload_reg_mem);
6636 %}
6637
6638 // Load Short (16 bit signed) into long
6639 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6640 %{
6641 match(Set dst (ConvI2L (LoadS mem)));
6642 predicate(!needs_acquiring_load(n->in(1)));
6643
6644 ins_cost(4 * INSN_COST);
6645 format %{ "ldrsh $dst, $mem\t# short" %}
6646
6647 ins_encode(aarch64_enc_ldrsh(dst, mem));
6648
6649 ins_pipe(iload_reg_mem);
6650 %}
6651
6652 // Load Char (16 bit unsigned)
6653 instruct loadUS(iRegINoSp dst, memory2 mem)
6654 %{
6655 match(Set dst (LoadUS mem));
6656 predicate(!needs_acquiring_load(n));
6657
6658 ins_cost(4 * INSN_COST);
6659 format %{ "ldrh $dst, $mem\t# short" %}
6660
6661 ins_encode(aarch64_enc_ldrh(dst, mem));
6662
6663 ins_pipe(iload_reg_mem);
6664 %}
6665
6666 // Load Short/Char (16 bit unsigned) into long
6667 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6668 %{
6669 match(Set dst (ConvI2L (LoadUS mem)));
6670 predicate(!needs_acquiring_load(n->in(1)));
6671
6672 ins_cost(4 * INSN_COST);
6673 format %{ "ldrh $dst, $mem\t# short" %}
6674
6675 ins_encode(aarch64_enc_ldrh(dst, mem));
6676
6677 ins_pipe(iload_reg_mem);
6678 %}
6679
6680 // Load Integer (32 bit signed)
6681 instruct loadI(iRegINoSp dst, memory4 mem)
6682 %{
6683 match(Set dst (LoadI mem));
6684 predicate(!needs_acquiring_load(n));
6685
6686 ins_cost(4 * INSN_COST);
6687 format %{ "ldrw $dst, $mem\t# int" %}
6688
6689 ins_encode(aarch64_enc_ldrw(dst, mem));
6690
6691 ins_pipe(iload_reg_mem);
6692 %}
6693
6694 // Load Integer (32 bit signed) into long
6695 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6696 %{
6697 match(Set dst (ConvI2L (LoadI mem)));
6698 predicate(!needs_acquiring_load(n->in(1)));
6699
6700 ins_cost(4 * INSN_COST);
6701 format %{ "ldrsw $dst, $mem\t# int" %}
6702
6703 ins_encode(aarch64_enc_ldrsw(dst, mem));
6704
6705 ins_pipe(iload_reg_mem);
6706 %}
6707
6708 // Load Integer (32 bit unsigned) into long
6709 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6710 %{
6711 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6712 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6713
6714 ins_cost(4 * INSN_COST);
6715 format %{ "ldrw $dst, $mem\t# int" %}
6716
6717 ins_encode(aarch64_enc_ldrw(dst, mem));
6718
6719 ins_pipe(iload_reg_mem);
6720 %}
6721
6722 // Load Long (64 bit signed)
6723 instruct loadL(iRegLNoSp dst, memory8 mem)
6724 %{
6725 match(Set dst (LoadL mem));
6726 predicate(!needs_acquiring_load(n));
6727
6728 ins_cost(4 * INSN_COST);
6729 format %{ "ldr $dst, $mem\t# int" %}
6730
6731 ins_encode(aarch64_enc_ldr(dst, mem));
6732
6733 ins_pipe(iload_reg_mem);
6734 %}
6735
6736 // Load Range
6737 instruct loadRange(iRegINoSp dst, memory4 mem)
6738 %{
6739 match(Set dst (LoadRange mem));
6740
6741 ins_cost(4 * INSN_COST);
6742 format %{ "ldrw $dst, $mem\t# range" %}
6743
6744 ins_encode(aarch64_enc_ldrw(dst, mem));
6745
6746 ins_pipe(iload_reg_mem);
6747 %}
6748
6749 // Load Pointer
6750 instruct loadP(iRegPNoSp dst, memory8 mem)
6751 %{
6752 match(Set dst (LoadP mem));
6753 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6754
6755 ins_cost(4 * INSN_COST);
6756 format %{ "ldr $dst, $mem\t# ptr" %}
6757
6758 ins_encode(aarch64_enc_ldr(dst, mem));
6759
6760 ins_pipe(iload_reg_mem);
6761 %}
6762
6763 // Load Compressed Pointer
6764 instruct loadN(iRegNNoSp dst, memory4 mem)
6765 %{
6766 match(Set dst (LoadN mem));
6767 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6768
6769 ins_cost(4 * INSN_COST);
6770 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6771
6772 ins_encode(aarch64_enc_ldrw(dst, mem));
6773
6774 ins_pipe(iload_reg_mem);
6775 %}
6776
6777 // Load Klass Pointer
6778 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6779 %{
6780 match(Set dst (LoadKlass mem));
6781 predicate(!needs_acquiring_load(n));
6782
6783 ins_cost(4 * INSN_COST);
6784 format %{ "ldr $dst, $mem\t# class" %}
6785
6786 ins_encode(aarch64_enc_ldr(dst, mem));
6787
6788 ins_pipe(iload_reg_mem);
6789 %}
6790
6791 // Load Narrow Klass Pointer
6792 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6793 %{
6794 match(Set dst (LoadNKlass mem));
6795 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6796
6797 ins_cost(4 * INSN_COST);
6798 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6799
6800 ins_encode(aarch64_enc_ldrw(dst, mem));
6801
6802 ins_pipe(iload_reg_mem);
6803 %}
6804
6805 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6806 %{
6807 match(Set dst (LoadNKlass mem));
6808 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6809
6810 ins_cost(4 * INSN_COST);
6811 format %{
6812 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6813 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6814 %}
6815 ins_encode %{
6816 // inlined aarch64_enc_ldrw
6817 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6818 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6819 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6820 %}
6821 ins_pipe(iload_reg_mem);
6822 %}
6823
6824 // Load Float
6825 instruct loadF(vRegF dst, memory4 mem)
6826 %{
6827 match(Set dst (LoadF mem));
6828 predicate(!needs_acquiring_load(n));
6829
6830 ins_cost(4 * INSN_COST);
6831 format %{ "ldrs $dst, $mem\t# float" %}
6832
6833 ins_encode( aarch64_enc_ldrs(dst, mem) );
6834
6835 ins_pipe(pipe_class_memory);
6836 %}
6837
6838 // Load Double
6839 instruct loadD(vRegD dst, memory8 mem)
6840 %{
6841 match(Set dst (LoadD mem));
6842 predicate(!needs_acquiring_load(n));
6843
6844 ins_cost(4 * INSN_COST);
6845 format %{ "ldrd $dst, $mem\t# double" %}
6846
6847 ins_encode( aarch64_enc_ldrd(dst, mem) );
6848
6849 ins_pipe(pipe_class_memory);
6850 %}
6851
6852
6853 // Load Int Constant
6854 instruct loadConI(iRegINoSp dst, immI src)
6855 %{
6856 match(Set dst src);
6857
6858 ins_cost(INSN_COST);
6859 format %{ "mov $dst, $src\t# int" %}
6860
6861 ins_encode( aarch64_enc_movw_imm(dst, src) );
6862
6863 ins_pipe(ialu_imm);
6864 %}
6865
6866 // Load Long Constant
6867 instruct loadConL(iRegLNoSp dst, immL src)
6868 %{
6869 match(Set dst src);
6870
6871 ins_cost(INSN_COST);
6872 format %{ "mov $dst, $src\t# long" %}
6873
6874 ins_encode( aarch64_enc_mov_imm(dst, src) );
6875
6876 ins_pipe(ialu_imm);
6877 %}
6878
6879 // Load Pointer Constant
6880
6881 instruct loadConP(iRegPNoSp dst, immP con)
6882 %{
6883 match(Set dst con);
6884
6885 ins_cost(INSN_COST * 4);
6886 format %{
6887 "mov $dst, $con\t# ptr"
6888 %}
6889
6890 ins_encode(aarch64_enc_mov_p(dst, con));
6891
6892 ins_pipe(ialu_imm);
6893 %}
6894
6895 // Load Null Pointer Constant
6896
6897 instruct loadConP0(iRegPNoSp dst, immP0 con)
6898 %{
6899 match(Set dst con);
6900
6901 ins_cost(INSN_COST);
6902 format %{ "mov $dst, $con\t# nullptr ptr" %}
6903
6904 ins_encode(aarch64_enc_mov_p0(dst, con));
6905
6906 ins_pipe(ialu_imm);
6907 %}
6908
6909 // Load Pointer Constant One
6910
6911 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6912 %{
6913 match(Set dst con);
6914
6915 ins_cost(INSN_COST);
6916 format %{ "mov $dst, $con\t# nullptr ptr" %}
6917
6918 ins_encode(aarch64_enc_mov_p1(dst, con));
6919
6920 ins_pipe(ialu_imm);
6921 %}
6922
6923 // Load Narrow Pointer Constant
6924
6925 instruct loadConN(iRegNNoSp dst, immN con)
6926 %{
6927 match(Set dst con);
6928
6929 ins_cost(INSN_COST * 4);
6930 format %{ "mov $dst, $con\t# compressed ptr" %}
6931
6932 ins_encode(aarch64_enc_mov_n(dst, con));
6933
6934 ins_pipe(ialu_imm);
6935 %}
6936
6937 // Load Narrow Null Pointer Constant
6938
6939 instruct loadConN0(iRegNNoSp dst, immN0 con)
6940 %{
6941 match(Set dst con);
6942
6943 ins_cost(INSN_COST);
6944 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6945
6946 ins_encode(aarch64_enc_mov_n0(dst, con));
6947
6948 ins_pipe(ialu_imm);
6949 %}
6950
6951 // Load Narrow Klass Constant
6952
6953 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6954 %{
6955 match(Set dst con);
6956
6957 ins_cost(INSN_COST);
6958 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6959
6960 ins_encode(aarch64_enc_mov_nk(dst, con));
6961
6962 ins_pipe(ialu_imm);
6963 %}
6964
6965 // Load Packed Float Constant
6966
6967 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6968 match(Set dst con);
6969 ins_cost(INSN_COST * 4);
6970 format %{ "fmovs $dst, $con"%}
6971 ins_encode %{
6972 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6973 %}
6974
6975 ins_pipe(fp_imm_s);
6976 %}
6977
6978 // Load Float Constant
6979
6980 instruct loadConF(vRegF dst, immF con) %{
6981 match(Set dst con);
6982
6983 ins_cost(INSN_COST * 4);
6984
6985 format %{
6986 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6987 %}
6988
6989 ins_encode %{
6990 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6991 %}
6992
6993 ins_pipe(fp_load_constant_s);
6994 %}
6995
6996 // Load Packed Double Constant
6997
6998 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6999 match(Set dst con);
7000 ins_cost(INSN_COST);
7001 format %{ "fmovd $dst, $con"%}
7002 ins_encode %{
7003 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7004 %}
7005
7006 ins_pipe(fp_imm_d);
7007 %}
7008
7009 // Load Double Constant
7010
7011 instruct loadConD(vRegD dst, immD con) %{
7012 match(Set dst con);
7013
7014 ins_cost(INSN_COST * 5);
7015 format %{
7016 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7017 %}
7018
7019 ins_encode %{
7020 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7021 %}
7022
7023 ins_pipe(fp_load_constant_d);
7024 %}
7025
7026 // Load Half Float Constant
7027 instruct loadConH(vRegF dst, immH con) %{
7028 match(Set dst con);
7029 format %{ "mov rscratch1, $con\n\t"
7030 "fmov $dst, rscratch1"
7031 %}
7032 ins_encode %{
7033 __ movw(rscratch1, (uint32_t)$con$$constant);
7034 __ fmovs($dst$$FloatRegister, rscratch1);
7035 %}
7036 ins_pipe(pipe_class_default);
7037 %}
7038
7039 // Store Instructions
7040
7041 // Store Byte
7042 instruct storeB(iRegIorL2I src, memory1 mem)
7043 %{
7044 match(Set mem (StoreB mem src));
7045 predicate(!needs_releasing_store(n));
7046
7047 ins_cost(INSN_COST);
7048 format %{ "strb $src, $mem\t# byte" %}
7049
7050 ins_encode(aarch64_enc_strb(src, mem));
7051
7052 ins_pipe(istore_reg_mem);
7053 %}
7054
7055
7056 instruct storeimmB0(immI0 zero, memory1 mem)
7057 %{
7058 match(Set mem (StoreB mem zero));
7059 predicate(!needs_releasing_store(n));
7060
7061 ins_cost(INSN_COST);
7062 format %{ "strb rscractch2, $mem\t# byte" %}
7063
7064 ins_encode(aarch64_enc_strb0(mem));
7065
7066 ins_pipe(istore_mem);
7067 %}
7068
7069 // Store Char/Short
7070 instruct storeC(iRegIorL2I src, memory2 mem)
7071 %{
7072 match(Set mem (StoreC mem src));
7073 predicate(!needs_releasing_store(n));
7074
7075 ins_cost(INSN_COST);
7076 format %{ "strh $src, $mem\t# short" %}
7077
7078 ins_encode(aarch64_enc_strh(src, mem));
7079
7080 ins_pipe(istore_reg_mem);
7081 %}
7082
7083 instruct storeimmC0(immI0 zero, memory2 mem)
7084 %{
7085 match(Set mem (StoreC mem zero));
7086 predicate(!needs_releasing_store(n));
7087
7088 ins_cost(INSN_COST);
7089 format %{ "strh zr, $mem\t# short" %}
7090
7091 ins_encode(aarch64_enc_strh0(mem));
7092
7093 ins_pipe(istore_mem);
7094 %}
7095
7096 // Store Integer
7097
7098 instruct storeI(iRegIorL2I src, memory4 mem)
7099 %{
7100 match(Set mem(StoreI mem src));
7101 predicate(!needs_releasing_store(n));
7102
7103 ins_cost(INSN_COST);
7104 format %{ "strw $src, $mem\t# int" %}
7105
7106 ins_encode(aarch64_enc_strw(src, mem));
7107
7108 ins_pipe(istore_reg_mem);
7109 %}
7110
7111 instruct storeimmI0(immI0 zero, memory4 mem)
7112 %{
7113 match(Set mem(StoreI mem zero));
7114 predicate(!needs_releasing_store(n));
7115
7116 ins_cost(INSN_COST);
7117 format %{ "strw zr, $mem\t# int" %}
7118
7119 ins_encode(aarch64_enc_strw0(mem));
7120
7121 ins_pipe(istore_mem);
7122 %}
7123
7124 // Store Long (64 bit signed)
7125 instruct storeL(iRegL src, memory8 mem)
7126 %{
7127 match(Set mem (StoreL mem src));
7128 predicate(!needs_releasing_store(n));
7129
7130 ins_cost(INSN_COST);
7131 format %{ "str $src, $mem\t# int" %}
7132
7133 ins_encode(aarch64_enc_str(src, mem));
7134
7135 ins_pipe(istore_reg_mem);
7136 %}
7137
7138 // Store Long (64 bit signed)
7139 instruct storeimmL0(immL0 zero, memory8 mem)
7140 %{
7141 match(Set mem (StoreL mem zero));
7142 predicate(!needs_releasing_store(n));
7143
7144 ins_cost(INSN_COST);
7145 format %{ "str zr, $mem\t# int" %}
7146
7147 ins_encode(aarch64_enc_str0(mem));
7148
7149 ins_pipe(istore_mem);
7150 %}
7151
7152 // Store Pointer
7153 instruct storeP(iRegP src, memory8 mem)
7154 %{
7155 match(Set mem (StoreP mem src));
7156 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7157
7158 ins_cost(INSN_COST);
7159 format %{ "str $src, $mem\t# ptr" %}
7160
7161 ins_encode(aarch64_enc_str(src, mem));
7162
7163 ins_pipe(istore_reg_mem);
7164 %}
7165
7166 // Store Pointer
7167 instruct storeimmP0(immP0 zero, memory8 mem)
7168 %{
7169 match(Set mem (StoreP mem zero));
7170 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7171
7172 ins_cost(INSN_COST);
7173 format %{ "str zr, $mem\t# ptr" %}
7174
7175 ins_encode(aarch64_enc_str0(mem));
7176
7177 ins_pipe(istore_mem);
7178 %}
7179
7180 // Store Compressed Pointer
7181 instruct storeN(iRegN src, memory4 mem)
7182 %{
7183 match(Set mem (StoreN mem src));
7184 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7185
7186 ins_cost(INSN_COST);
7187 format %{ "strw $src, $mem\t# compressed ptr" %}
7188
7189 ins_encode(aarch64_enc_strw(src, mem));
7190
7191 ins_pipe(istore_reg_mem);
7192 %}
7193
7194 instruct storeImmN0(immN0 zero, memory4 mem)
7195 %{
7196 match(Set mem (StoreN mem zero));
7197 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7198
7199 ins_cost(INSN_COST);
7200 format %{ "strw zr, $mem\t# compressed ptr" %}
7201
7202 ins_encode(aarch64_enc_strw0(mem));
7203
7204 ins_pipe(istore_mem);
7205 %}
7206
7207 // Store Float
7208 instruct storeF(vRegF src, memory4 mem)
7209 %{
7210 match(Set mem (StoreF mem src));
7211 predicate(!needs_releasing_store(n));
7212
7213 ins_cost(INSN_COST);
7214 format %{ "strs $src, $mem\t# float" %}
7215
7216 ins_encode( aarch64_enc_strs(src, mem) );
7217
7218 ins_pipe(pipe_class_memory);
7219 %}
7220
7221 // TODO
7222 // implement storeImmF0 and storeFImmPacked
7223
7224 // Store Double
7225 instruct storeD(vRegD src, memory8 mem)
7226 %{
7227 match(Set mem (StoreD mem src));
7228 predicate(!needs_releasing_store(n));
7229
7230 ins_cost(INSN_COST);
7231 format %{ "strd $src, $mem\t# double" %}
7232
7233 ins_encode( aarch64_enc_strd(src, mem) );
7234
7235 ins_pipe(pipe_class_memory);
7236 %}
7237
7238 // Store Compressed Klass Pointer
7239 instruct storeNKlass(iRegN src, memory4 mem)
7240 %{
7241 predicate(!needs_releasing_store(n));
7242 match(Set mem (StoreNKlass mem src));
7243
7244 ins_cost(INSN_COST);
7245 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7246
7247 ins_encode(aarch64_enc_strw(src, mem));
7248
7249 ins_pipe(istore_reg_mem);
7250 %}
7251
7252 // TODO
7253 // implement storeImmD0 and storeDImmPacked
7254
7255 // prefetch instructions
7256 // Must be safe to execute with invalid address (cannot fault).
7257
7258 instruct prefetchalloc( memory8 mem ) %{
7259 match(PrefetchAllocation mem);
7260
7261 ins_cost(INSN_COST);
7262 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7263
7264 ins_encode( aarch64_enc_prefetchw(mem) );
7265
7266 ins_pipe(iload_prefetch);
7267 %}
7268
7269 // ---------------- volatile loads and stores ----------------
7270
7271 // Load Byte (8 bit signed)
7272 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7273 %{
7274 match(Set dst (LoadB mem));
7275
7276 ins_cost(VOLATILE_REF_COST);
7277 format %{ "ldarsb $dst, $mem\t# byte" %}
7278
7279 ins_encode(aarch64_enc_ldarsb(dst, mem));
7280
7281 ins_pipe(pipe_serial);
7282 %}
7283
7284 // Load Byte (8 bit signed) into long
7285 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7286 %{
7287 match(Set dst (ConvI2L (LoadB mem)));
7288
7289 ins_cost(VOLATILE_REF_COST);
7290 format %{ "ldarsb $dst, $mem\t# byte" %}
7291
7292 ins_encode(aarch64_enc_ldarsb(dst, mem));
7293
7294 ins_pipe(pipe_serial);
7295 %}
7296
7297 // Load Byte (8 bit unsigned)
7298 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7299 %{
7300 match(Set dst (LoadUB mem));
7301
7302 ins_cost(VOLATILE_REF_COST);
7303 format %{ "ldarb $dst, $mem\t# byte" %}
7304
7305 ins_encode(aarch64_enc_ldarb(dst, mem));
7306
7307 ins_pipe(pipe_serial);
7308 %}
7309
7310 // Load Byte (8 bit unsigned) into long
7311 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7312 %{
7313 match(Set dst (ConvI2L (LoadUB mem)));
7314
7315 ins_cost(VOLATILE_REF_COST);
7316 format %{ "ldarb $dst, $mem\t# byte" %}
7317
7318 ins_encode(aarch64_enc_ldarb(dst, mem));
7319
7320 ins_pipe(pipe_serial);
7321 %}
7322
7323 // Load Short (16 bit signed)
7324 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7325 %{
7326 match(Set dst (LoadS mem));
7327
7328 ins_cost(VOLATILE_REF_COST);
7329 format %{ "ldarshw $dst, $mem\t# short" %}
7330
7331 ins_encode(aarch64_enc_ldarshw(dst, mem));
7332
7333 ins_pipe(pipe_serial);
7334 %}
7335
7336 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7337 %{
7338 match(Set dst (LoadUS mem));
7339
7340 ins_cost(VOLATILE_REF_COST);
7341 format %{ "ldarhw $dst, $mem\t# short" %}
7342
7343 ins_encode(aarch64_enc_ldarhw(dst, mem));
7344
7345 ins_pipe(pipe_serial);
7346 %}
7347
7348 // Load Short/Char (16 bit unsigned) into long
7349 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7350 %{
7351 match(Set dst (ConvI2L (LoadUS mem)));
7352
7353 ins_cost(VOLATILE_REF_COST);
7354 format %{ "ldarh $dst, $mem\t# short" %}
7355
7356 ins_encode(aarch64_enc_ldarh(dst, mem));
7357
7358 ins_pipe(pipe_serial);
7359 %}
7360
7361 // Load Short/Char (16 bit signed) into long
7362 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7363 %{
7364 match(Set dst (ConvI2L (LoadS mem)));
7365
7366 ins_cost(VOLATILE_REF_COST);
7367 format %{ "ldarh $dst, $mem\t# short" %}
7368
7369 ins_encode(aarch64_enc_ldarsh(dst, mem));
7370
7371 ins_pipe(pipe_serial);
7372 %}
7373
7374 // Load Integer (32 bit signed)
7375 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7376 %{
7377 match(Set dst (LoadI mem));
7378
7379 ins_cost(VOLATILE_REF_COST);
7380 format %{ "ldarw $dst, $mem\t# int" %}
7381
7382 ins_encode(aarch64_enc_ldarw(dst, mem));
7383
7384 ins_pipe(pipe_serial);
7385 %}
7386
7387 // Load Integer (32 bit unsigned) into long
7388 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7389 %{
7390 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7391
7392 ins_cost(VOLATILE_REF_COST);
7393 format %{ "ldarw $dst, $mem\t# int" %}
7394
7395 ins_encode(aarch64_enc_ldarw(dst, mem));
7396
7397 ins_pipe(pipe_serial);
7398 %}
7399
7400 // Load Long (64 bit signed)
7401 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7402 %{
7403 match(Set dst (LoadL mem));
7404
7405 ins_cost(VOLATILE_REF_COST);
7406 format %{ "ldar $dst, $mem\t# int" %}
7407
7408 ins_encode(aarch64_enc_ldar(dst, mem));
7409
7410 ins_pipe(pipe_serial);
7411 %}
7412
7413 // Load Pointer
7414 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7415 %{
7416 match(Set dst (LoadP mem));
7417 predicate(n->as_Load()->barrier_data() == 0);
7418
7419 ins_cost(VOLATILE_REF_COST);
7420 format %{ "ldar $dst, $mem\t# ptr" %}
7421
7422 ins_encode(aarch64_enc_ldar(dst, mem));
7423
7424 ins_pipe(pipe_serial);
7425 %}
7426
7427 // Load Compressed Pointer
7428 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7429 %{
7430 match(Set dst (LoadN mem));
7431 predicate(n->as_Load()->barrier_data() == 0);
7432
7433 ins_cost(VOLATILE_REF_COST);
7434 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7435
7436 ins_encode(aarch64_enc_ldarw(dst, mem));
7437
7438 ins_pipe(pipe_serial);
7439 %}
7440
7441 // Load Float
7442 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7443 %{
7444 match(Set dst (LoadF mem));
7445
7446 ins_cost(VOLATILE_REF_COST);
7447 format %{ "ldars $dst, $mem\t# float" %}
7448
7449 ins_encode( aarch64_enc_fldars(dst, mem) );
7450
7451 ins_pipe(pipe_serial);
7452 %}
7453
7454 // Load Double
7455 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7456 %{
7457 match(Set dst (LoadD mem));
7458
7459 ins_cost(VOLATILE_REF_COST);
7460 format %{ "ldard $dst, $mem\t# double" %}
7461
7462 ins_encode( aarch64_enc_fldard(dst, mem) );
7463
7464 ins_pipe(pipe_serial);
7465 %}
7466
7467 // Store Byte
7468 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7469 %{
7470 match(Set mem (StoreB mem src));
7471
7472 ins_cost(VOLATILE_REF_COST);
7473 format %{ "stlrb $src, $mem\t# byte" %}
7474
7475 ins_encode(aarch64_enc_stlrb(src, mem));
7476
7477 ins_pipe(pipe_class_memory);
7478 %}
7479
7480 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7481 %{
7482 match(Set mem (StoreB mem zero));
7483
7484 ins_cost(VOLATILE_REF_COST);
7485 format %{ "stlrb zr, $mem\t# byte" %}
7486
7487 ins_encode(aarch64_enc_stlrb0(mem));
7488
7489 ins_pipe(pipe_class_memory);
7490 %}
7491
7492 // Store Char/Short
7493 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7494 %{
7495 match(Set mem (StoreC mem src));
7496
7497 ins_cost(VOLATILE_REF_COST);
7498 format %{ "stlrh $src, $mem\t# short" %}
7499
7500 ins_encode(aarch64_enc_stlrh(src, mem));
7501
7502 ins_pipe(pipe_class_memory);
7503 %}
7504
7505 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7506 %{
7507 match(Set mem (StoreC mem zero));
7508
7509 ins_cost(VOLATILE_REF_COST);
7510 format %{ "stlrh zr, $mem\t# short" %}
7511
7512 ins_encode(aarch64_enc_stlrh0(mem));
7513
7514 ins_pipe(pipe_class_memory);
7515 %}
7516
7517 // Store Integer
7518
7519 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7520 %{
7521 match(Set mem(StoreI mem src));
7522
7523 ins_cost(VOLATILE_REF_COST);
7524 format %{ "stlrw $src, $mem\t# int" %}
7525
7526 ins_encode(aarch64_enc_stlrw(src, mem));
7527
7528 ins_pipe(pipe_class_memory);
7529 %}
7530
7531 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7532 %{
7533 match(Set mem(StoreI mem zero));
7534
7535 ins_cost(VOLATILE_REF_COST);
7536 format %{ "stlrw zr, $mem\t# int" %}
7537
7538 ins_encode(aarch64_enc_stlrw0(mem));
7539
7540 ins_pipe(pipe_class_memory);
7541 %}
7542
7543 // Store Long (64 bit signed)
7544 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7545 %{
7546 match(Set mem (StoreL mem src));
7547
7548 ins_cost(VOLATILE_REF_COST);
7549 format %{ "stlr $src, $mem\t# int" %}
7550
7551 ins_encode(aarch64_enc_stlr(src, mem));
7552
7553 ins_pipe(pipe_class_memory);
7554 %}
7555
7556 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7557 %{
7558 match(Set mem (StoreL mem zero));
7559
7560 ins_cost(VOLATILE_REF_COST);
7561 format %{ "stlr zr, $mem\t# int" %}
7562
7563 ins_encode(aarch64_enc_stlr0(mem));
7564
7565 ins_pipe(pipe_class_memory);
7566 %}
7567
7568 // Store Pointer
7569 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7570 %{
7571 match(Set mem (StoreP mem src));
7572 predicate(n->as_Store()->barrier_data() == 0);
7573
7574 ins_cost(VOLATILE_REF_COST);
7575 format %{ "stlr $src, $mem\t# ptr" %}
7576
7577 ins_encode(aarch64_enc_stlr(src, mem));
7578
7579 ins_pipe(pipe_class_memory);
7580 %}
7581
7582 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7583 %{
7584 match(Set mem (StoreP mem zero));
7585 predicate(n->as_Store()->barrier_data() == 0);
7586
7587 ins_cost(VOLATILE_REF_COST);
7588 format %{ "stlr zr, $mem\t# ptr" %}
7589
7590 ins_encode(aarch64_enc_stlr0(mem));
7591
7592 ins_pipe(pipe_class_memory);
7593 %}
7594
7595 // Store Compressed Pointer
7596 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7597 %{
7598 match(Set mem (StoreN mem src));
7599 predicate(n->as_Store()->barrier_data() == 0);
7600
7601 ins_cost(VOLATILE_REF_COST);
7602 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7603
7604 ins_encode(aarch64_enc_stlrw(src, mem));
7605
7606 ins_pipe(pipe_class_memory);
7607 %}
7608
7609 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7610 %{
7611 match(Set mem (StoreN mem zero));
7612 predicate(n->as_Store()->barrier_data() == 0);
7613
7614 ins_cost(VOLATILE_REF_COST);
7615 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7616
7617 ins_encode(aarch64_enc_stlrw0(mem));
7618
7619 ins_pipe(pipe_class_memory);
7620 %}
7621
7622 // Store Float
7623 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7624 %{
7625 match(Set mem (StoreF mem src));
7626
7627 ins_cost(VOLATILE_REF_COST);
7628 format %{ "stlrs $src, $mem\t# float" %}
7629
7630 ins_encode( aarch64_enc_fstlrs(src, mem) );
7631
7632 ins_pipe(pipe_class_memory);
7633 %}
7634
7635 // TODO
7636 // implement storeImmF0 and storeFImmPacked
7637
7638 // Store Double
7639 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7640 %{
7641 match(Set mem (StoreD mem src));
7642
7643 ins_cost(VOLATILE_REF_COST);
7644 format %{ "stlrd $src, $mem\t# double" %}
7645
7646 ins_encode( aarch64_enc_fstlrd(src, mem) );
7647
7648 ins_pipe(pipe_class_memory);
7649 %}
7650
7651 // ---------------- end of volatile loads and stores ----------------
7652
7653 instruct cacheWB(indirect addr)
7654 %{
7655 predicate(VM_Version::supports_data_cache_line_flush());
7656 match(CacheWB addr);
7657
7658 ins_cost(100);
7659 format %{"cache wb $addr" %}
7660 ins_encode %{
7661 assert($addr->index_position() < 0, "should be");
7662 assert($addr$$disp == 0, "should be");
7663 __ cache_wb(Address($addr$$base$$Register, 0));
7664 %}
7665 ins_pipe(pipe_slow); // XXX
7666 %}
7667
7668 instruct cacheWBPreSync()
7669 %{
7670 predicate(VM_Version::supports_data_cache_line_flush());
7671 match(CacheWBPreSync);
7672
7673 ins_cost(100);
7674 format %{"cache wb presync" %}
7675 ins_encode %{
7676 __ cache_wbsync(true);
7677 %}
7678 ins_pipe(pipe_slow); // XXX
7679 %}
7680
7681 instruct cacheWBPostSync()
7682 %{
7683 predicate(VM_Version::supports_data_cache_line_flush());
7684 match(CacheWBPostSync);
7685
7686 ins_cost(100);
7687 format %{"cache wb postsync" %}
7688 ins_encode %{
7689 __ cache_wbsync(false);
7690 %}
7691 ins_pipe(pipe_slow); // XXX
7692 %}
7693
7694 // ============================================================================
7695 // BSWAP Instructions
7696
7697 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7698 match(Set dst (ReverseBytesI src));
7699
7700 ins_cost(INSN_COST);
7701 format %{ "revw $dst, $src" %}
7702
7703 ins_encode %{
7704 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7705 %}
7706
7707 ins_pipe(ialu_reg);
7708 %}
7709
7710 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7711 match(Set dst (ReverseBytesL src));
7712
7713 ins_cost(INSN_COST);
7714 format %{ "rev $dst, $src" %}
7715
7716 ins_encode %{
7717 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7718 %}
7719
7720 ins_pipe(ialu_reg);
7721 %}
7722
7723 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7724 match(Set dst (ReverseBytesUS src));
7725
7726 ins_cost(INSN_COST);
7727 format %{ "rev16w $dst, $src" %}
7728
7729 ins_encode %{
7730 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7731 %}
7732
7733 ins_pipe(ialu_reg);
7734 %}
7735
7736 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7737 match(Set dst (ReverseBytesS src));
7738
7739 ins_cost(INSN_COST);
7740 format %{ "rev16w $dst, $src\n\t"
7741 "sbfmw $dst, $dst, #0, #15" %}
7742
7743 ins_encode %{
7744 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7745 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7746 %}
7747
7748 ins_pipe(ialu_reg);
7749 %}
7750
7751 // ============================================================================
7752 // Zero Count Instructions
7753
7754 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7755 match(Set dst (CountLeadingZerosI src));
7756
7757 ins_cost(INSN_COST);
7758 format %{ "clzw $dst, $src" %}
7759 ins_encode %{
7760 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7761 %}
7762
7763 ins_pipe(ialu_reg);
7764 %}
7765
7766 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7767 match(Set dst (CountLeadingZerosL src));
7768
7769 ins_cost(INSN_COST);
7770 format %{ "clz $dst, $src" %}
7771 ins_encode %{
7772 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7773 %}
7774
7775 ins_pipe(ialu_reg);
7776 %}
7777
7778 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7779 match(Set dst (CountTrailingZerosI src));
7780
7781 ins_cost(INSN_COST * 2);
7782 format %{ "rbitw $dst, $src\n\t"
7783 "clzw $dst, $dst" %}
7784 ins_encode %{
7785 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7786 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7787 %}
7788
7789 ins_pipe(ialu_reg);
7790 %}
7791
7792 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7793 match(Set dst (CountTrailingZerosL src));
7794
7795 ins_cost(INSN_COST * 2);
7796 format %{ "rbit $dst, $src\n\t"
7797 "clz $dst, $dst" %}
7798 ins_encode %{
7799 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7800 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7801 %}
7802
7803 ins_pipe(ialu_reg);
7804 %}
7805
7806 //---------- Population Count Instructions -------------------------------------
7807 //
7808
7809 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7810 match(Set dst (PopCountI src));
7811 effect(TEMP tmp);
7812 ins_cost(INSN_COST * 13);
7813
7814 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7815 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7816 "addv $tmp, $tmp\t# vector (8B)\n\t"
7817 "mov $dst, $tmp\t# vector (1D)" %}
7818 ins_encode %{
7819 __ fmovs($tmp$$FloatRegister, $src$$Register);
7820 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7821 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7822 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7823 %}
7824
7825 ins_pipe(pipe_class_default);
7826 %}
7827
7828 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7829 match(Set dst (PopCountI (LoadI mem)));
7830 effect(TEMP tmp);
7831 ins_cost(INSN_COST * 13);
7832
7833 format %{ "ldrs $tmp, $mem\n\t"
7834 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7835 "addv $tmp, $tmp\t# vector (8B)\n\t"
7836 "mov $dst, $tmp\t# vector (1D)" %}
7837 ins_encode %{
7838 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7839 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7840 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7841 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7842 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7843 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7844 %}
7845
7846 ins_pipe(pipe_class_default);
7847 %}
7848
7849 // Note: Long.bitCount(long) returns an int.
7850 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7851 match(Set dst (PopCountL src));
7852 effect(TEMP tmp);
7853 ins_cost(INSN_COST * 13);
7854
7855 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7856 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7857 "addv $tmp, $tmp\t# vector (8B)\n\t"
7858 "mov $dst, $tmp\t# vector (1D)" %}
7859 ins_encode %{
7860 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7861 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7862 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7863 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7864 %}
7865
7866 ins_pipe(pipe_class_default);
7867 %}
7868
7869 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7870 match(Set dst (PopCountL (LoadL mem)));
7871 effect(TEMP tmp);
7872 ins_cost(INSN_COST * 13);
7873
7874 format %{ "ldrd $tmp, $mem\n\t"
7875 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7876 "addv $tmp, $tmp\t# vector (8B)\n\t"
7877 "mov $dst, $tmp\t# vector (1D)" %}
7878 ins_encode %{
7879 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7880 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7881 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7882 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7883 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7884 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7885 %}
7886
7887 ins_pipe(pipe_class_default);
7888 %}
7889
7890 // ============================================================================
7891 // VerifyVectorAlignment Instruction
7892
7893 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7894 match(Set addr (VerifyVectorAlignment addr mask));
7895 effect(KILL cr);
7896 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7897 ins_encode %{
7898 Label Lskip;
7899 // check if masked bits of addr are zero
7900 __ tst($addr$$Register, $mask$$constant);
7901 __ br(Assembler::EQ, Lskip);
7902 __ stop("verify_vector_alignment found a misaligned vector memory access");
7903 __ bind(Lskip);
7904 %}
7905 ins_pipe(pipe_slow);
7906 %}
7907
7908 // ============================================================================
7909 // MemBar Instruction
7910
7911 instruct load_fence() %{
7912 match(LoadFence);
7913 ins_cost(VOLATILE_REF_COST);
7914
7915 format %{ "load_fence" %}
7916
7917 ins_encode %{
7918 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7919 %}
7920 ins_pipe(pipe_serial);
7921 %}
7922
7923 instruct unnecessary_membar_acquire() %{
7924 predicate(unnecessary_acquire(n));
7925 match(MemBarAcquire);
7926 ins_cost(0);
7927
7928 format %{ "membar_acquire (elided)" %}
7929
7930 ins_encode %{
7931 __ block_comment("membar_acquire (elided)");
7932 %}
7933
7934 ins_pipe(pipe_class_empty);
7935 %}
7936
7937 instruct membar_acquire() %{
7938 match(MemBarAcquire);
7939 ins_cost(VOLATILE_REF_COST);
7940
7941 format %{ "membar_acquire\n\t"
7942 "dmb ishld" %}
7943
7944 ins_encode %{
7945 __ block_comment("membar_acquire");
7946 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7947 %}
7948
7949 ins_pipe(pipe_serial);
7950 %}
7951
7952
7953 instruct membar_acquire_lock() %{
7954 match(MemBarAcquireLock);
7955 ins_cost(VOLATILE_REF_COST);
7956
7957 format %{ "membar_acquire_lock (elided)" %}
7958
7959 ins_encode %{
7960 __ block_comment("membar_acquire_lock (elided)");
7961 %}
7962
7963 ins_pipe(pipe_serial);
7964 %}
7965
7966 instruct store_fence() %{
7967 match(StoreFence);
7968 ins_cost(VOLATILE_REF_COST);
7969
7970 format %{ "store_fence" %}
7971
7972 ins_encode %{
7973 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7974 %}
7975 ins_pipe(pipe_serial);
7976 %}
7977
7978 instruct unnecessary_membar_release() %{
7979 predicate(unnecessary_release(n));
7980 match(MemBarRelease);
7981 ins_cost(0);
7982
7983 format %{ "membar_release (elided)" %}
7984
7985 ins_encode %{
7986 __ block_comment("membar_release (elided)");
7987 %}
7988 ins_pipe(pipe_serial);
7989 %}
7990
7991 instruct membar_release() %{
7992 match(MemBarRelease);
7993 ins_cost(VOLATILE_REF_COST);
7994
7995 format %{ "membar_release\n\t"
7996 "dmb ishst\n\tdmb ishld" %}
7997
7998 ins_encode %{
7999 __ block_comment("membar_release");
8000 // These will be merged if AlwaysMergeDMB is enabled.
8001 __ membar(Assembler::StoreStore);
8002 __ membar(Assembler::LoadStore);
8003 %}
8004 ins_pipe(pipe_serial);
8005 %}
8006
8007 instruct membar_storestore() %{
8008 match(MemBarStoreStore);
8009 match(StoreStoreFence);
8010 ins_cost(VOLATILE_REF_COST);
8011
8012 format %{ "MEMBAR-store-store" %}
8013
8014 ins_encode %{
8015 __ membar(Assembler::StoreStore);
8016 %}
8017 ins_pipe(pipe_serial);
8018 %}
8019
8020 instruct membar_release_lock() %{
8021 match(MemBarReleaseLock);
8022 ins_cost(VOLATILE_REF_COST);
8023
8024 format %{ "membar_release_lock (elided)" %}
8025
8026 ins_encode %{
8027 __ block_comment("membar_release_lock (elided)");
8028 %}
8029
8030 ins_pipe(pipe_serial);
8031 %}
8032
8033 instruct unnecessary_membar_volatile() %{
8034 predicate(unnecessary_volatile(n));
8035 match(MemBarVolatile);
8036 ins_cost(0);
8037
8038 format %{ "membar_volatile (elided)" %}
8039
8040 ins_encode %{
8041 __ block_comment("membar_volatile (elided)");
8042 %}
8043
8044 ins_pipe(pipe_serial);
8045 %}
8046
8047 instruct membar_volatile() %{
8048 match(MemBarVolatile);
8049 ins_cost(VOLATILE_REF_COST*100);
8050
8051 format %{ "membar_volatile\n\t"
8052 "dmb ish"%}
8053
8054 ins_encode %{
8055 __ block_comment("membar_volatile");
8056 __ membar(Assembler::StoreLoad);
8057 %}
8058
8059 ins_pipe(pipe_serial);
8060 %}
8061
8062 // ============================================================================
8063 // Cast/Convert Instructions
8064
8065 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8066 match(Set dst (CastX2P src));
8067
8068 ins_cost(INSN_COST);
8069 format %{ "mov $dst, $src\t# long -> ptr" %}
8070
8071 ins_encode %{
8072 if ($dst$$reg != $src$$reg) {
8073 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8074 }
8075 %}
8076
8077 ins_pipe(ialu_reg);
8078 %}
8079
8080 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8081 match(Set dst (CastI2N src));
8082
8083 ins_cost(INSN_COST);
8084 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8085
8086 ins_encode %{
8087 if ($dst$$reg != $src$$reg) {
8088 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8089 }
8090 %}
8091
8092 ins_pipe(ialu_reg);
8093 %}
8094
8095 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8096 match(Set dst (CastP2X src));
8097
8098 ins_cost(INSN_COST);
8099 format %{ "mov $dst, $src\t# ptr -> long" %}
8100
8101 ins_encode %{
8102 if ($dst$$reg != $src$$reg) {
8103 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8104 }
8105 %}
8106
8107 ins_pipe(ialu_reg);
8108 %}
8109
8110 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8111 match(Set dst (CastP2X src));
8112
8113 ins_cost(INSN_COST);
8114 format %{ "mov $dst, $src\t# ptr -> long" %}
8115
8116 ins_encode %{
8117 if ($dst$$reg != $src$$reg) {
8118 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8119 }
8120 %}
8121
8122 ins_pipe(ialu_reg);
8123 %}
8124
8125 // Convert oop into int for vectors alignment masking
8126 instruct convP2I(iRegINoSp dst, iRegP src) %{
8127 match(Set dst (ConvL2I (CastP2X src)));
8128
8129 ins_cost(INSN_COST);
8130 format %{ "movw $dst, $src\t# ptr -> int" %}
8131 ins_encode %{
8132 __ movw($dst$$Register, $src$$Register);
8133 %}
8134
8135 ins_pipe(ialu_reg);
8136 %}
8137
8138 // Convert compressed oop into int for vectors alignment masking
8139 // in case of 32bit oops (heap < 4Gb).
8140 instruct convN2I(iRegINoSp dst, iRegN src)
8141 %{
8142 predicate(CompressedOops::shift() == 0);
8143 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8144
8145 ins_cost(INSN_COST);
8146 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8147 ins_encode %{
8148 __ movw($dst$$Register, $src$$Register);
8149 %}
8150
8151 ins_pipe(ialu_reg);
8152 %}
8153
8154
8155 // Convert oop pointer into compressed form
8156 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8157 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8158 match(Set dst (EncodeP src));
8159 effect(KILL cr);
8160 ins_cost(INSN_COST * 3);
8161 format %{ "encode_heap_oop $dst, $src" %}
8162 ins_encode %{
8163 Register s = $src$$Register;
8164 Register d = $dst$$Register;
8165 __ encode_heap_oop(d, s);
8166 %}
8167 ins_pipe(ialu_reg);
8168 %}
8169
8170 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8171 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8172 match(Set dst (EncodeP src));
8173 ins_cost(INSN_COST * 3);
8174 format %{ "encode_heap_oop_not_null $dst, $src" %}
8175 ins_encode %{
8176 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8177 %}
8178 ins_pipe(ialu_reg);
8179 %}
8180
8181 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8182 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8183 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8184 match(Set dst (DecodeN src));
8185 ins_cost(INSN_COST * 3);
8186 format %{ "decode_heap_oop $dst, $src" %}
8187 ins_encode %{
8188 Register s = $src$$Register;
8189 Register d = $dst$$Register;
8190 __ decode_heap_oop(d, s);
8191 %}
8192 ins_pipe(ialu_reg);
8193 %}
8194
8195 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8196 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8197 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8198 match(Set dst (DecodeN src));
8199 ins_cost(INSN_COST * 3);
8200 format %{ "decode_heap_oop_not_null $dst, $src" %}
8201 ins_encode %{
8202 Register s = $src$$Register;
8203 Register d = $dst$$Register;
8204 __ decode_heap_oop_not_null(d, s);
8205 %}
8206 ins_pipe(ialu_reg);
8207 %}
8208
8209 // n.b. AArch64 implementations of encode_klass_not_null and
8210 // decode_klass_not_null do not modify the flags register so, unlike
8211 // Intel, we don't kill CR as a side effect here
8212
8213 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8214 match(Set dst (EncodePKlass src));
8215
8216 ins_cost(INSN_COST * 3);
8217 format %{ "encode_klass_not_null $dst,$src" %}
8218
8219 ins_encode %{
8220 Register src_reg = as_Register($src$$reg);
8221 Register dst_reg = as_Register($dst$$reg);
8222 __ encode_klass_not_null(dst_reg, src_reg);
8223 %}
8224
8225 ins_pipe(ialu_reg);
8226 %}
8227
8228 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8229 match(Set dst (DecodeNKlass src));
8230
8231 ins_cost(INSN_COST * 3);
8232 format %{ "decode_klass_not_null $dst,$src" %}
8233
8234 ins_encode %{
8235 Register src_reg = as_Register($src$$reg);
8236 Register dst_reg = as_Register($dst$$reg);
8237 if (dst_reg != src_reg) {
8238 __ decode_klass_not_null(dst_reg, src_reg);
8239 } else {
8240 __ decode_klass_not_null(dst_reg);
8241 }
8242 %}
8243
8244 ins_pipe(ialu_reg);
8245 %}
8246
8247 instruct checkCastPP(iRegPNoSp dst)
8248 %{
8249 match(Set dst (CheckCastPP dst));
8250
8251 size(0);
8252 format %{ "# checkcastPP of $dst" %}
8253 ins_encode(/* empty encoding */);
8254 ins_pipe(pipe_class_empty);
8255 %}
8256
8257 instruct castPP(iRegPNoSp dst)
8258 %{
8259 match(Set dst (CastPP dst));
8260
8261 size(0);
8262 format %{ "# castPP of $dst" %}
8263 ins_encode(/* empty encoding */);
8264 ins_pipe(pipe_class_empty);
8265 %}
8266
8267 instruct castII(iRegI dst)
8268 %{
8269 predicate(VerifyConstraintCasts == 0);
8270 match(Set dst (CastII dst));
8271
8272 size(0);
8273 format %{ "# castII of $dst" %}
8274 ins_encode(/* empty encoding */);
8275 ins_cost(0);
8276 ins_pipe(pipe_class_empty);
8277 %}
8278
8279 instruct castII_checked(iRegI dst, rFlagsReg cr)
8280 %{
8281 predicate(VerifyConstraintCasts > 0);
8282 match(Set dst (CastII dst));
8283 effect(KILL cr);
8284
8285 format %{ "# castII_checked of $dst" %}
8286 ins_encode %{
8287 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8288 %}
8289 ins_pipe(pipe_slow);
8290 %}
8291
8292 instruct castLL(iRegL dst)
8293 %{
8294 predicate(VerifyConstraintCasts == 0);
8295 match(Set dst (CastLL dst));
8296
8297 size(0);
8298 format %{ "# castLL of $dst" %}
8299 ins_encode(/* empty encoding */);
8300 ins_cost(0);
8301 ins_pipe(pipe_class_empty);
8302 %}
8303
8304 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8305 %{
8306 predicate(VerifyConstraintCasts > 0);
8307 match(Set dst (CastLL dst));
8308 effect(KILL cr);
8309
8310 format %{ "# castLL_checked of $dst" %}
8311 ins_encode %{
8312 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8313 %}
8314 ins_pipe(pipe_slow);
8315 %}
8316
8317 instruct castHH(vRegF dst)
8318 %{
8319 match(Set dst (CastHH dst));
8320 size(0);
8321 format %{ "# castHH of $dst" %}
8322 ins_encode(/* empty encoding */);
8323 ins_cost(0);
8324 ins_pipe(pipe_class_empty);
8325 %}
8326
8327 instruct castFF(vRegF dst)
8328 %{
8329 match(Set dst (CastFF dst));
8330
8331 size(0);
8332 format %{ "# castFF of $dst" %}
8333 ins_encode(/* empty encoding */);
8334 ins_cost(0);
8335 ins_pipe(pipe_class_empty);
8336 %}
8337
8338 instruct castDD(vRegD dst)
8339 %{
8340 match(Set dst (CastDD dst));
8341
8342 size(0);
8343 format %{ "# castDD of $dst" %}
8344 ins_encode(/* empty encoding */);
8345 ins_cost(0);
8346 ins_pipe(pipe_class_empty);
8347 %}
8348
8349 instruct castVV(vReg dst)
8350 %{
8351 match(Set dst (CastVV dst));
8352
8353 size(0);
8354 format %{ "# castVV of $dst" %}
8355 ins_encode(/* empty encoding */);
8356 ins_cost(0);
8357 ins_pipe(pipe_class_empty);
8358 %}
8359
8360 instruct castVVMask(pRegGov dst)
8361 %{
8362 match(Set dst (CastVV dst));
8363
8364 size(0);
8365 format %{ "# castVV of $dst" %}
8366 ins_encode(/* empty encoding */);
8367 ins_cost(0);
8368 ins_pipe(pipe_class_empty);
8369 %}
8370
8371 // Manifest a CmpU result in an integer register.
8372 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8373 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8374 %{
8375 match(Set dst (CmpU3 src1 src2));
8376 effect(KILL flags);
8377
8378 ins_cost(INSN_COST * 3);
8379 format %{
8380 "cmpw $src1, $src2\n\t"
8381 "csetw $dst, ne\n\t"
8382 "cnegw $dst, lo\t# CmpU3(reg)"
8383 %}
8384 ins_encode %{
8385 __ cmpw($src1$$Register, $src2$$Register);
8386 __ csetw($dst$$Register, Assembler::NE);
8387 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8388 %}
8389
8390 ins_pipe(pipe_class_default);
8391 %}
8392
8393 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8394 %{
8395 match(Set dst (CmpU3 src1 src2));
8396 effect(KILL flags);
8397
8398 ins_cost(INSN_COST * 3);
8399 format %{
8400 "subsw zr, $src1, $src2\n\t"
8401 "csetw $dst, ne\n\t"
8402 "cnegw $dst, lo\t# CmpU3(imm)"
8403 %}
8404 ins_encode %{
8405 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8406 __ csetw($dst$$Register, Assembler::NE);
8407 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8408 %}
8409
8410 ins_pipe(pipe_class_default);
8411 %}
8412
8413 // Manifest a CmpUL result in an integer register.
8414 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8415 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8416 %{
8417 match(Set dst (CmpUL3 src1 src2));
8418 effect(KILL flags);
8419
8420 ins_cost(INSN_COST * 3);
8421 format %{
8422 "cmp $src1, $src2\n\t"
8423 "csetw $dst, ne\n\t"
8424 "cnegw $dst, lo\t# CmpUL3(reg)"
8425 %}
8426 ins_encode %{
8427 __ cmp($src1$$Register, $src2$$Register);
8428 __ csetw($dst$$Register, Assembler::NE);
8429 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8430 %}
8431
8432 ins_pipe(pipe_class_default);
8433 %}
8434
8435 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8436 %{
8437 match(Set dst (CmpUL3 src1 src2));
8438 effect(KILL flags);
8439
8440 ins_cost(INSN_COST * 3);
8441 format %{
8442 "subs zr, $src1, $src2\n\t"
8443 "csetw $dst, ne\n\t"
8444 "cnegw $dst, lo\t# CmpUL3(imm)"
8445 %}
8446 ins_encode %{
8447 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8448 __ csetw($dst$$Register, Assembler::NE);
8449 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8450 %}
8451
8452 ins_pipe(pipe_class_default);
8453 %}
8454
8455 // Manifest a CmpL result in an integer register.
8456 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8457 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8458 %{
8459 match(Set dst (CmpL3 src1 src2));
8460 effect(KILL flags);
8461
8462 ins_cost(INSN_COST * 3);
8463 format %{
8464 "cmp $src1, $src2\n\t"
8465 "csetw $dst, ne\n\t"
8466 "cnegw $dst, lt\t# CmpL3(reg)"
8467 %}
8468 ins_encode %{
8469 __ cmp($src1$$Register, $src2$$Register);
8470 __ csetw($dst$$Register, Assembler::NE);
8471 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8472 %}
8473
8474 ins_pipe(pipe_class_default);
8475 %}
8476
8477 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8478 %{
8479 match(Set dst (CmpL3 src1 src2));
8480 effect(KILL flags);
8481
8482 ins_cost(INSN_COST * 3);
8483 format %{
8484 "subs zr, $src1, $src2\n\t"
8485 "csetw $dst, ne\n\t"
8486 "cnegw $dst, lt\t# CmpL3(imm)"
8487 %}
8488 ins_encode %{
8489 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8490 __ csetw($dst$$Register, Assembler::NE);
8491 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8492 %}
8493
8494 ins_pipe(pipe_class_default);
8495 %}
8496
8497 // ============================================================================
8498 // Conditional Move Instructions
8499
8500 // n.b. we have identical rules for both a signed compare op (cmpOp)
8501 // and an unsigned compare op (cmpOpU). it would be nice if we could
8502 // define an op class which merged both inputs and use it to type the
8503 // argument to a single rule. unfortunatelyt his fails because the
8504 // opclass does not live up to the COND_INTER interface of its
8505 // component operands. When the generic code tries to negate the
8506 // operand it ends up running the generci Machoper::negate method
8507 // which throws a ShouldNotHappen. So, we have to provide two flavours
8508 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8509
8510 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8511 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8512
8513 ins_cost(INSN_COST * 2);
8514 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8515
8516 ins_encode %{
8517 __ cselw(as_Register($dst$$reg),
8518 as_Register($src2$$reg),
8519 as_Register($src1$$reg),
8520 (Assembler::Condition)$cmp$$cmpcode);
8521 %}
8522
8523 ins_pipe(icond_reg_reg);
8524 %}
8525
8526 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8527 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8528
8529 ins_cost(INSN_COST * 2);
8530 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8531
8532 ins_encode %{
8533 __ cselw(as_Register($dst$$reg),
8534 as_Register($src2$$reg),
8535 as_Register($src1$$reg),
8536 (Assembler::Condition)$cmp$$cmpcode);
8537 %}
8538
8539 ins_pipe(icond_reg_reg);
8540 %}
8541
8542 // special cases where one arg is zero
8543
8544 // n.b. this is selected in preference to the rule above because it
8545 // avoids loading constant 0 into a source register
8546
8547 // TODO
8548 // we ought only to be able to cull one of these variants as the ideal
8549 // transforms ought always to order the zero consistently (to left/right?)
8550
8551 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8552 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8553
8554 ins_cost(INSN_COST * 2);
8555 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8556
8557 ins_encode %{
8558 __ cselw(as_Register($dst$$reg),
8559 as_Register($src$$reg),
8560 zr,
8561 (Assembler::Condition)$cmp$$cmpcode);
8562 %}
8563
8564 ins_pipe(icond_reg);
8565 %}
8566
8567 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8568 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8569
8570 ins_cost(INSN_COST * 2);
8571 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8572
8573 ins_encode %{
8574 __ cselw(as_Register($dst$$reg),
8575 as_Register($src$$reg),
8576 zr,
8577 (Assembler::Condition)$cmp$$cmpcode);
8578 %}
8579
8580 ins_pipe(icond_reg);
8581 %}
8582
8583 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8584 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8585
8586 ins_cost(INSN_COST * 2);
8587 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8588
8589 ins_encode %{
8590 __ cselw(as_Register($dst$$reg),
8591 zr,
8592 as_Register($src$$reg),
8593 (Assembler::Condition)$cmp$$cmpcode);
8594 %}
8595
8596 ins_pipe(icond_reg);
8597 %}
8598
8599 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8600 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8601
8602 ins_cost(INSN_COST * 2);
8603 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8604
8605 ins_encode %{
8606 __ cselw(as_Register($dst$$reg),
8607 zr,
8608 as_Register($src$$reg),
8609 (Assembler::Condition)$cmp$$cmpcode);
8610 %}
8611
8612 ins_pipe(icond_reg);
8613 %}
8614
8615 // special case for creating a boolean 0 or 1
8616
8617 // n.b. this is selected in preference to the rule above because it
8618 // avoids loading constants 0 and 1 into a source register
8619
8620 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8621 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8622
8623 ins_cost(INSN_COST * 2);
8624 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8625
8626 ins_encode %{
8627 // equivalently
8628 // cset(as_Register($dst$$reg),
8629 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8630 __ csincw(as_Register($dst$$reg),
8631 zr,
8632 zr,
8633 (Assembler::Condition)$cmp$$cmpcode);
8634 %}
8635
8636 ins_pipe(icond_none);
8637 %}
8638
8639 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8640 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8641
8642 ins_cost(INSN_COST * 2);
8643 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8644
8645 ins_encode %{
8646 // equivalently
8647 // cset(as_Register($dst$$reg),
8648 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8649 __ csincw(as_Register($dst$$reg),
8650 zr,
8651 zr,
8652 (Assembler::Condition)$cmp$$cmpcode);
8653 %}
8654
8655 ins_pipe(icond_none);
8656 %}
8657
8658 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8659 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8660
8661 ins_cost(INSN_COST * 2);
8662 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8663
8664 ins_encode %{
8665 __ csel(as_Register($dst$$reg),
8666 as_Register($src2$$reg),
8667 as_Register($src1$$reg),
8668 (Assembler::Condition)$cmp$$cmpcode);
8669 %}
8670
8671 ins_pipe(icond_reg_reg);
8672 %}
8673
8674 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8675 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8676
8677 ins_cost(INSN_COST * 2);
8678 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8679
8680 ins_encode %{
8681 __ csel(as_Register($dst$$reg),
8682 as_Register($src2$$reg),
8683 as_Register($src1$$reg),
8684 (Assembler::Condition)$cmp$$cmpcode);
8685 %}
8686
8687 ins_pipe(icond_reg_reg);
8688 %}
8689
8690 // special cases where one arg is zero
8691
8692 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8693 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8694
8695 ins_cost(INSN_COST * 2);
8696 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8697
8698 ins_encode %{
8699 __ csel(as_Register($dst$$reg),
8700 zr,
8701 as_Register($src$$reg),
8702 (Assembler::Condition)$cmp$$cmpcode);
8703 %}
8704
8705 ins_pipe(icond_reg);
8706 %}
8707
8708 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8709 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8710
8711 ins_cost(INSN_COST * 2);
8712 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8713
8714 ins_encode %{
8715 __ csel(as_Register($dst$$reg),
8716 zr,
8717 as_Register($src$$reg),
8718 (Assembler::Condition)$cmp$$cmpcode);
8719 %}
8720
8721 ins_pipe(icond_reg);
8722 %}
8723
8724 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8725 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8726
8727 ins_cost(INSN_COST * 2);
8728 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8729
8730 ins_encode %{
8731 __ csel(as_Register($dst$$reg),
8732 as_Register($src$$reg),
8733 zr,
8734 (Assembler::Condition)$cmp$$cmpcode);
8735 %}
8736
8737 ins_pipe(icond_reg);
8738 %}
8739
8740 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8741 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8742
8743 ins_cost(INSN_COST * 2);
8744 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8745
8746 ins_encode %{
8747 __ csel(as_Register($dst$$reg),
8748 as_Register($src$$reg),
8749 zr,
8750 (Assembler::Condition)$cmp$$cmpcode);
8751 %}
8752
8753 ins_pipe(icond_reg);
8754 %}
8755
8756 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8757 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8758
8759 ins_cost(INSN_COST * 2);
8760 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8761
8762 ins_encode %{
8763 __ csel(as_Register($dst$$reg),
8764 as_Register($src2$$reg),
8765 as_Register($src1$$reg),
8766 (Assembler::Condition)$cmp$$cmpcode);
8767 %}
8768
8769 ins_pipe(icond_reg_reg);
8770 %}
8771
8772 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8773 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8774
8775 ins_cost(INSN_COST * 2);
8776 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8777
8778 ins_encode %{
8779 __ csel(as_Register($dst$$reg),
8780 as_Register($src2$$reg),
8781 as_Register($src1$$reg),
8782 (Assembler::Condition)$cmp$$cmpcode);
8783 %}
8784
8785 ins_pipe(icond_reg_reg);
8786 %}
8787
8788 // special cases where one arg is zero
8789
8790 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8791 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8792
8793 ins_cost(INSN_COST * 2);
8794 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8795
8796 ins_encode %{
8797 __ csel(as_Register($dst$$reg),
8798 zr,
8799 as_Register($src$$reg),
8800 (Assembler::Condition)$cmp$$cmpcode);
8801 %}
8802
8803 ins_pipe(icond_reg);
8804 %}
8805
8806 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8807 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8808
8809 ins_cost(INSN_COST * 2);
8810 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8811
8812 ins_encode %{
8813 __ csel(as_Register($dst$$reg),
8814 zr,
8815 as_Register($src$$reg),
8816 (Assembler::Condition)$cmp$$cmpcode);
8817 %}
8818
8819 ins_pipe(icond_reg);
8820 %}
8821
8822 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8823 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8824
8825 ins_cost(INSN_COST * 2);
8826 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8827
8828 ins_encode %{
8829 __ csel(as_Register($dst$$reg),
8830 as_Register($src$$reg),
8831 zr,
8832 (Assembler::Condition)$cmp$$cmpcode);
8833 %}
8834
8835 ins_pipe(icond_reg);
8836 %}
8837
8838 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8839 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8840
8841 ins_cost(INSN_COST * 2);
8842 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8843
8844 ins_encode %{
8845 __ csel(as_Register($dst$$reg),
8846 as_Register($src$$reg),
8847 zr,
8848 (Assembler::Condition)$cmp$$cmpcode);
8849 %}
8850
8851 ins_pipe(icond_reg);
8852 %}
8853
8854 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8855 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8856
8857 ins_cost(INSN_COST * 2);
8858 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8859
8860 ins_encode %{
8861 __ cselw(as_Register($dst$$reg),
8862 as_Register($src2$$reg),
8863 as_Register($src1$$reg),
8864 (Assembler::Condition)$cmp$$cmpcode);
8865 %}
8866
8867 ins_pipe(icond_reg_reg);
8868 %}
8869
8870 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8871 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8872
8873 ins_cost(INSN_COST * 2);
8874 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8875
8876 ins_encode %{
8877 __ cselw(as_Register($dst$$reg),
8878 as_Register($src2$$reg),
8879 as_Register($src1$$reg),
8880 (Assembler::Condition)$cmp$$cmpcode);
8881 %}
8882
8883 ins_pipe(icond_reg_reg);
8884 %}
8885
8886 // special cases where one arg is zero
8887
8888 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8889 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8890
8891 ins_cost(INSN_COST * 2);
8892 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8893
8894 ins_encode %{
8895 __ cselw(as_Register($dst$$reg),
8896 zr,
8897 as_Register($src$$reg),
8898 (Assembler::Condition)$cmp$$cmpcode);
8899 %}
8900
8901 ins_pipe(icond_reg);
8902 %}
8903
8904 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8905 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8906
8907 ins_cost(INSN_COST * 2);
8908 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8909
8910 ins_encode %{
8911 __ cselw(as_Register($dst$$reg),
8912 zr,
8913 as_Register($src$$reg),
8914 (Assembler::Condition)$cmp$$cmpcode);
8915 %}
8916
8917 ins_pipe(icond_reg);
8918 %}
8919
8920 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8921 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8922
8923 ins_cost(INSN_COST * 2);
8924 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8925
8926 ins_encode %{
8927 __ cselw(as_Register($dst$$reg),
8928 as_Register($src$$reg),
8929 zr,
8930 (Assembler::Condition)$cmp$$cmpcode);
8931 %}
8932
8933 ins_pipe(icond_reg);
8934 %}
8935
8936 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8937 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8938
8939 ins_cost(INSN_COST * 2);
8940 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
8941
8942 ins_encode %{
8943 __ cselw(as_Register($dst$$reg),
8944 as_Register($src$$reg),
8945 zr,
8946 (Assembler::Condition)$cmp$$cmpcode);
8947 %}
8948
8949 ins_pipe(icond_reg);
8950 %}
8951
8952 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
8953 %{
8954 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8955
8956 ins_cost(INSN_COST * 3);
8957
8958 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8959 ins_encode %{
8960 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8961 __ fcsels(as_FloatRegister($dst$$reg),
8962 as_FloatRegister($src2$$reg),
8963 as_FloatRegister($src1$$reg),
8964 cond);
8965 %}
8966
8967 ins_pipe(fp_cond_reg_reg_s);
8968 %}
8969
8970 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
8971 %{
8972 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
8973
8974 ins_cost(INSN_COST * 3);
8975
8976 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
8977 ins_encode %{
8978 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8979 __ fcsels(as_FloatRegister($dst$$reg),
8980 as_FloatRegister($src2$$reg),
8981 as_FloatRegister($src1$$reg),
8982 cond);
8983 %}
8984
8985 ins_pipe(fp_cond_reg_reg_s);
8986 %}
8987
8988 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
8989 %{
8990 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
8991
8992 ins_cost(INSN_COST * 3);
8993
8994 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
8995 ins_encode %{
8996 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
8997 __ fcseld(as_FloatRegister($dst$$reg),
8998 as_FloatRegister($src2$$reg),
8999 as_FloatRegister($src1$$reg),
9000 cond);
9001 %}
9002
9003 ins_pipe(fp_cond_reg_reg_d);
9004 %}
9005
9006 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9007 %{
9008 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9009
9010 ins_cost(INSN_COST * 3);
9011
9012 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9013 ins_encode %{
9014 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9015 __ fcseld(as_FloatRegister($dst$$reg),
9016 as_FloatRegister($src2$$reg),
9017 as_FloatRegister($src1$$reg),
9018 cond);
9019 %}
9020
9021 ins_pipe(fp_cond_reg_reg_d);
9022 %}
9023
9024 // ============================================================================
9025 // Arithmetic Instructions
9026 //
9027
9028 // Integer Addition
9029
9030 // TODO
9031 // these currently employ operations which do not set CR and hence are
9032 // not flagged as killing CR but we would like to isolate the cases
9033 // where we want to set flags from those where we don't. need to work
9034 // out how to do that.
9035
9036 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9037 match(Set dst (AddI src1 src2));
9038
9039 ins_cost(INSN_COST);
9040 format %{ "addw $dst, $src1, $src2" %}
9041
9042 ins_encode %{
9043 __ addw(as_Register($dst$$reg),
9044 as_Register($src1$$reg),
9045 as_Register($src2$$reg));
9046 %}
9047
9048 ins_pipe(ialu_reg_reg);
9049 %}
9050
9051 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9052 match(Set dst (AddI src1 src2));
9053
9054 ins_cost(INSN_COST);
9055 format %{ "addw $dst, $src1, $src2" %}
9056
9057 // use opcode to indicate that this is an add not a sub
9058 opcode(0x0);
9059
9060 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9061
9062 ins_pipe(ialu_reg_imm);
9063 %}
9064
9065 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9066 match(Set dst (AddI (ConvL2I src1) src2));
9067
9068 ins_cost(INSN_COST);
9069 format %{ "addw $dst, $src1, $src2" %}
9070
9071 // use opcode to indicate that this is an add not a sub
9072 opcode(0x0);
9073
9074 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9075
9076 ins_pipe(ialu_reg_imm);
9077 %}
9078
9079 // Pointer Addition
9080 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9081 match(Set dst (AddP src1 src2));
9082
9083 ins_cost(INSN_COST);
9084 format %{ "add $dst, $src1, $src2\t# ptr" %}
9085
9086 ins_encode %{
9087 __ add(as_Register($dst$$reg),
9088 as_Register($src1$$reg),
9089 as_Register($src2$$reg));
9090 %}
9091
9092 ins_pipe(ialu_reg_reg);
9093 %}
9094
9095 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9096 match(Set dst (AddP src1 (ConvI2L src2)));
9097
9098 ins_cost(1.9 * INSN_COST);
9099 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9100
9101 ins_encode %{
9102 __ add(as_Register($dst$$reg),
9103 as_Register($src1$$reg),
9104 as_Register($src2$$reg), ext::sxtw);
9105 %}
9106
9107 ins_pipe(ialu_reg_reg);
9108 %}
9109
9110 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9111 match(Set dst (AddP src1 (LShiftL src2 scale)));
9112
9113 ins_cost(1.9 * INSN_COST);
9114 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9115
9116 ins_encode %{
9117 __ lea(as_Register($dst$$reg),
9118 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9119 Address::lsl($scale$$constant)));
9120 %}
9121
9122 ins_pipe(ialu_reg_reg_shift);
9123 %}
9124
9125 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9126 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9127
9128 ins_cost(1.9 * INSN_COST);
9129 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9130
9131 ins_encode %{
9132 __ lea(as_Register($dst$$reg),
9133 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9134 Address::sxtw($scale$$constant)));
9135 %}
9136
9137 ins_pipe(ialu_reg_reg_shift);
9138 %}
9139
9140 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9141 match(Set dst (LShiftL (ConvI2L src) scale));
9142
9143 ins_cost(INSN_COST);
9144 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9145
9146 ins_encode %{
9147 __ sbfiz(as_Register($dst$$reg),
9148 as_Register($src$$reg),
9149 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9150 %}
9151
9152 ins_pipe(ialu_reg_shift);
9153 %}
9154
9155 // Pointer Immediate Addition
9156 // n.b. this needs to be more expensive than using an indirect memory
9157 // operand
9158 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9159 match(Set dst (AddP src1 src2));
9160
9161 ins_cost(INSN_COST);
9162 format %{ "add $dst, $src1, $src2\t# ptr" %}
9163
9164 // use opcode to indicate that this is an add not a sub
9165 opcode(0x0);
9166
9167 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9168
9169 ins_pipe(ialu_reg_imm);
9170 %}
9171
9172 // Long Addition
9173 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9174
9175 match(Set dst (AddL src1 src2));
9176
9177 ins_cost(INSN_COST);
9178 format %{ "add $dst, $src1, $src2" %}
9179
9180 ins_encode %{
9181 __ add(as_Register($dst$$reg),
9182 as_Register($src1$$reg),
9183 as_Register($src2$$reg));
9184 %}
9185
9186 ins_pipe(ialu_reg_reg);
9187 %}
9188
9189 // No constant pool entries requiredLong Immediate Addition.
9190 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9191 match(Set dst (AddL src1 src2));
9192
9193 ins_cost(INSN_COST);
9194 format %{ "add $dst, $src1, $src2" %}
9195
9196 // use opcode to indicate that this is an add not a sub
9197 opcode(0x0);
9198
9199 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9200
9201 ins_pipe(ialu_reg_imm);
9202 %}
9203
9204 // Integer Subtraction
9205 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9206 match(Set dst (SubI src1 src2));
9207
9208 ins_cost(INSN_COST);
9209 format %{ "subw $dst, $src1, $src2" %}
9210
9211 ins_encode %{
9212 __ subw(as_Register($dst$$reg),
9213 as_Register($src1$$reg),
9214 as_Register($src2$$reg));
9215 %}
9216
9217 ins_pipe(ialu_reg_reg);
9218 %}
9219
9220 // Immediate Subtraction
9221 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9222 match(Set dst (SubI src1 src2));
9223
9224 ins_cost(INSN_COST);
9225 format %{ "subw $dst, $src1, $src2" %}
9226
9227 // use opcode to indicate that this is a sub not an add
9228 opcode(0x1);
9229
9230 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9231
9232 ins_pipe(ialu_reg_imm);
9233 %}
9234
9235 // Long Subtraction
9236 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9237
9238 match(Set dst (SubL src1 src2));
9239
9240 ins_cost(INSN_COST);
9241 format %{ "sub $dst, $src1, $src2" %}
9242
9243 ins_encode %{
9244 __ sub(as_Register($dst$$reg),
9245 as_Register($src1$$reg),
9246 as_Register($src2$$reg));
9247 %}
9248
9249 ins_pipe(ialu_reg_reg);
9250 %}
9251
9252 // No constant pool entries requiredLong Immediate Subtraction.
9253 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9254 match(Set dst (SubL src1 src2));
9255
9256 ins_cost(INSN_COST);
9257 format %{ "sub$dst, $src1, $src2" %}
9258
9259 // use opcode to indicate that this is a sub not an add
9260 opcode(0x1);
9261
9262 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9263
9264 ins_pipe(ialu_reg_imm);
9265 %}
9266
9267 // Integer Negation (special case for sub)
9268
9269 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9270 match(Set dst (SubI zero src));
9271
9272 ins_cost(INSN_COST);
9273 format %{ "negw $dst, $src\t# int" %}
9274
9275 ins_encode %{
9276 __ negw(as_Register($dst$$reg),
9277 as_Register($src$$reg));
9278 %}
9279
9280 ins_pipe(ialu_reg);
9281 %}
9282
9283 // Long Negation
9284
9285 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9286 match(Set dst (SubL zero src));
9287
9288 ins_cost(INSN_COST);
9289 format %{ "neg $dst, $src\t# long" %}
9290
9291 ins_encode %{
9292 __ neg(as_Register($dst$$reg),
9293 as_Register($src$$reg));
9294 %}
9295
9296 ins_pipe(ialu_reg);
9297 %}
9298
9299 // Integer Multiply
9300
9301 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9302 match(Set dst (MulI src1 src2));
9303
9304 ins_cost(INSN_COST * 3);
9305 format %{ "mulw $dst, $src1, $src2" %}
9306
9307 ins_encode %{
9308 __ mulw(as_Register($dst$$reg),
9309 as_Register($src1$$reg),
9310 as_Register($src2$$reg));
9311 %}
9312
9313 ins_pipe(imul_reg_reg);
9314 %}
9315
9316 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9317 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9318
9319 ins_cost(INSN_COST * 3);
9320 format %{ "smull $dst, $src1, $src2" %}
9321
9322 ins_encode %{
9323 __ smull(as_Register($dst$$reg),
9324 as_Register($src1$$reg),
9325 as_Register($src2$$reg));
9326 %}
9327
9328 ins_pipe(imul_reg_reg);
9329 %}
9330
9331 // Long Multiply
9332
9333 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9334 match(Set dst (MulL src1 src2));
9335
9336 ins_cost(INSN_COST * 5);
9337 format %{ "mul $dst, $src1, $src2" %}
9338
9339 ins_encode %{
9340 __ mul(as_Register($dst$$reg),
9341 as_Register($src1$$reg),
9342 as_Register($src2$$reg));
9343 %}
9344
9345 ins_pipe(lmul_reg_reg);
9346 %}
9347
9348 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9349 %{
9350 match(Set dst (MulHiL src1 src2));
9351
9352 ins_cost(INSN_COST * 7);
9353 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9354
9355 ins_encode %{
9356 __ smulh(as_Register($dst$$reg),
9357 as_Register($src1$$reg),
9358 as_Register($src2$$reg));
9359 %}
9360
9361 ins_pipe(lmul_reg_reg);
9362 %}
9363
9364 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9365 %{
9366 match(Set dst (UMulHiL src1 src2));
9367
9368 ins_cost(INSN_COST * 7);
9369 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9370
9371 ins_encode %{
9372 __ umulh(as_Register($dst$$reg),
9373 as_Register($src1$$reg),
9374 as_Register($src2$$reg));
9375 %}
9376
9377 ins_pipe(lmul_reg_reg);
9378 %}
9379
9380 // Combined Integer Multiply & Add/Sub
9381
9382 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9383 match(Set dst (AddI src3 (MulI src1 src2)));
9384
9385 ins_cost(INSN_COST * 3);
9386 format %{ "madd $dst, $src1, $src2, $src3" %}
9387
9388 ins_encode %{
9389 __ maddw(as_Register($dst$$reg),
9390 as_Register($src1$$reg),
9391 as_Register($src2$$reg),
9392 as_Register($src3$$reg));
9393 %}
9394
9395 ins_pipe(imac_reg_reg);
9396 %}
9397
9398 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9399 match(Set dst (SubI src3 (MulI src1 src2)));
9400
9401 ins_cost(INSN_COST * 3);
9402 format %{ "msub $dst, $src1, $src2, $src3" %}
9403
9404 ins_encode %{
9405 __ msubw(as_Register($dst$$reg),
9406 as_Register($src1$$reg),
9407 as_Register($src2$$reg),
9408 as_Register($src3$$reg));
9409 %}
9410
9411 ins_pipe(imac_reg_reg);
9412 %}
9413
9414 // Combined Integer Multiply & Neg
9415
9416 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9417 match(Set dst (MulI (SubI zero src1) src2));
9418
9419 ins_cost(INSN_COST * 3);
9420 format %{ "mneg $dst, $src1, $src2" %}
9421
9422 ins_encode %{
9423 __ mnegw(as_Register($dst$$reg),
9424 as_Register($src1$$reg),
9425 as_Register($src2$$reg));
9426 %}
9427
9428 ins_pipe(imac_reg_reg);
9429 %}
9430
9431 // Combined Long Multiply & Add/Sub
9432
9433 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9434 match(Set dst (AddL src3 (MulL src1 src2)));
9435
9436 ins_cost(INSN_COST * 5);
9437 format %{ "madd $dst, $src1, $src2, $src3" %}
9438
9439 ins_encode %{
9440 __ madd(as_Register($dst$$reg),
9441 as_Register($src1$$reg),
9442 as_Register($src2$$reg),
9443 as_Register($src3$$reg));
9444 %}
9445
9446 ins_pipe(lmac_reg_reg);
9447 %}
9448
9449 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9450 match(Set dst (SubL src3 (MulL src1 src2)));
9451
9452 ins_cost(INSN_COST * 5);
9453 format %{ "msub $dst, $src1, $src2, $src3" %}
9454
9455 ins_encode %{
9456 __ msub(as_Register($dst$$reg),
9457 as_Register($src1$$reg),
9458 as_Register($src2$$reg),
9459 as_Register($src3$$reg));
9460 %}
9461
9462 ins_pipe(lmac_reg_reg);
9463 %}
9464
9465 // Combined Long Multiply & Neg
9466
9467 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9468 match(Set dst (MulL (SubL zero src1) src2));
9469
9470 ins_cost(INSN_COST * 5);
9471 format %{ "mneg $dst, $src1, $src2" %}
9472
9473 ins_encode %{
9474 __ mneg(as_Register($dst$$reg),
9475 as_Register($src1$$reg),
9476 as_Register($src2$$reg));
9477 %}
9478
9479 ins_pipe(lmac_reg_reg);
9480 %}
9481
9482 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9483
9484 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9485 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9486
9487 ins_cost(INSN_COST * 3);
9488 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9489
9490 ins_encode %{
9491 __ smaddl(as_Register($dst$$reg),
9492 as_Register($src1$$reg),
9493 as_Register($src2$$reg),
9494 as_Register($src3$$reg));
9495 %}
9496
9497 ins_pipe(imac_reg_reg);
9498 %}
9499
9500 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9501 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9502
9503 ins_cost(INSN_COST * 3);
9504 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9505
9506 ins_encode %{
9507 __ smsubl(as_Register($dst$$reg),
9508 as_Register($src1$$reg),
9509 as_Register($src2$$reg),
9510 as_Register($src3$$reg));
9511 %}
9512
9513 ins_pipe(imac_reg_reg);
9514 %}
9515
9516 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9517 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9518
9519 ins_cost(INSN_COST * 3);
9520 format %{ "smnegl $dst, $src1, $src2" %}
9521
9522 ins_encode %{
9523 __ smnegl(as_Register($dst$$reg),
9524 as_Register($src1$$reg),
9525 as_Register($src2$$reg));
9526 %}
9527
9528 ins_pipe(imac_reg_reg);
9529 %}
9530
9531 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9532
9533 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9534 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9535
9536 ins_cost(INSN_COST * 5);
9537 format %{ "mulw rscratch1, $src1, $src2\n\t"
9538 "maddw $dst, $src3, $src4, rscratch1" %}
9539
9540 ins_encode %{
9541 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9542 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9543
9544 ins_pipe(imac_reg_reg);
9545 %}
9546
9547 // Integer Divide
9548
9549 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9550 match(Set dst (DivI src1 src2));
9551
9552 ins_cost(INSN_COST * 19);
9553 format %{ "sdivw $dst, $src1, $src2" %}
9554
9555 ins_encode(aarch64_enc_divw(dst, src1, src2));
9556 ins_pipe(idiv_reg_reg);
9557 %}
9558
9559 // Long Divide
9560
9561 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9562 match(Set dst (DivL src1 src2));
9563
9564 ins_cost(INSN_COST * 35);
9565 format %{ "sdiv $dst, $src1, $src2" %}
9566
9567 ins_encode(aarch64_enc_div(dst, src1, src2));
9568 ins_pipe(ldiv_reg_reg);
9569 %}
9570
9571 // Integer Remainder
9572
9573 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9574 match(Set dst (ModI src1 src2));
9575
9576 ins_cost(INSN_COST * 22);
9577 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9578 "msubw $dst, rscratch1, $src2, $src1" %}
9579
9580 ins_encode(aarch64_enc_modw(dst, src1, src2));
9581 ins_pipe(idiv_reg_reg);
9582 %}
9583
9584 // Long Remainder
9585
9586 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9587 match(Set dst (ModL src1 src2));
9588
9589 ins_cost(INSN_COST * 38);
9590 format %{ "sdiv rscratch1, $src1, $src2\n"
9591 "msub $dst, rscratch1, $src2, $src1" %}
9592
9593 ins_encode(aarch64_enc_mod(dst, src1, src2));
9594 ins_pipe(ldiv_reg_reg);
9595 %}
9596
9597 // Unsigned Integer Divide
9598
9599 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9600 match(Set dst (UDivI src1 src2));
9601
9602 ins_cost(INSN_COST * 19);
9603 format %{ "udivw $dst, $src1, $src2" %}
9604
9605 ins_encode %{
9606 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9607 %}
9608
9609 ins_pipe(idiv_reg_reg);
9610 %}
9611
9612 // Unsigned Long Divide
9613
9614 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9615 match(Set dst (UDivL src1 src2));
9616
9617 ins_cost(INSN_COST * 35);
9618 format %{ "udiv $dst, $src1, $src2" %}
9619
9620 ins_encode %{
9621 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9622 %}
9623
9624 ins_pipe(ldiv_reg_reg);
9625 %}
9626
9627 // Unsigned Integer Remainder
9628
9629 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9630 match(Set dst (UModI src1 src2));
9631
9632 ins_cost(INSN_COST * 22);
9633 format %{ "udivw rscratch1, $src1, $src2\n\t"
9634 "msubw $dst, rscratch1, $src2, $src1" %}
9635
9636 ins_encode %{
9637 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9638 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9639 %}
9640
9641 ins_pipe(idiv_reg_reg);
9642 %}
9643
9644 // Unsigned Long Remainder
9645
9646 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9647 match(Set dst (UModL src1 src2));
9648
9649 ins_cost(INSN_COST * 38);
9650 format %{ "udiv rscratch1, $src1, $src2\n"
9651 "msub $dst, rscratch1, $src2, $src1" %}
9652
9653 ins_encode %{
9654 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9655 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9656 %}
9657
9658 ins_pipe(ldiv_reg_reg);
9659 %}
9660
9661 // Integer Shifts
9662
9663 // Shift Left Register
9664 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9665 match(Set dst (LShiftI src1 src2));
9666
9667 ins_cost(INSN_COST * 2);
9668 format %{ "lslvw $dst, $src1, $src2" %}
9669
9670 ins_encode %{
9671 __ lslvw(as_Register($dst$$reg),
9672 as_Register($src1$$reg),
9673 as_Register($src2$$reg));
9674 %}
9675
9676 ins_pipe(ialu_reg_reg_vshift);
9677 %}
9678
9679 // Shift Left Immediate
9680 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9681 match(Set dst (LShiftI src1 src2));
9682
9683 ins_cost(INSN_COST);
9684 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9685
9686 ins_encode %{
9687 __ lslw(as_Register($dst$$reg),
9688 as_Register($src1$$reg),
9689 $src2$$constant & 0x1f);
9690 %}
9691
9692 ins_pipe(ialu_reg_shift);
9693 %}
9694
9695 // Shift Right Logical Register
9696 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9697 match(Set dst (URShiftI src1 src2));
9698
9699 ins_cost(INSN_COST * 2);
9700 format %{ "lsrvw $dst, $src1, $src2" %}
9701
9702 ins_encode %{
9703 __ lsrvw(as_Register($dst$$reg),
9704 as_Register($src1$$reg),
9705 as_Register($src2$$reg));
9706 %}
9707
9708 ins_pipe(ialu_reg_reg_vshift);
9709 %}
9710
9711 // Shift Right Logical Immediate
9712 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9713 match(Set dst (URShiftI src1 src2));
9714
9715 ins_cost(INSN_COST);
9716 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9717
9718 ins_encode %{
9719 __ lsrw(as_Register($dst$$reg),
9720 as_Register($src1$$reg),
9721 $src2$$constant & 0x1f);
9722 %}
9723
9724 ins_pipe(ialu_reg_shift);
9725 %}
9726
9727 // Shift Right Arithmetic Register
9728 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9729 match(Set dst (RShiftI src1 src2));
9730
9731 ins_cost(INSN_COST * 2);
9732 format %{ "asrvw $dst, $src1, $src2" %}
9733
9734 ins_encode %{
9735 __ asrvw(as_Register($dst$$reg),
9736 as_Register($src1$$reg),
9737 as_Register($src2$$reg));
9738 %}
9739
9740 ins_pipe(ialu_reg_reg_vshift);
9741 %}
9742
9743 // Shift Right Arithmetic Immediate
9744 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9745 match(Set dst (RShiftI src1 src2));
9746
9747 ins_cost(INSN_COST);
9748 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9749
9750 ins_encode %{
9751 __ asrw(as_Register($dst$$reg),
9752 as_Register($src1$$reg),
9753 $src2$$constant & 0x1f);
9754 %}
9755
9756 ins_pipe(ialu_reg_shift);
9757 %}
9758
9759 // Combined Int Mask and Right Shift (using UBFM)
9760 // TODO
9761
9762 // Long Shifts
9763
9764 // Shift Left Register
9765 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9766 match(Set dst (LShiftL src1 src2));
9767
9768 ins_cost(INSN_COST * 2);
9769 format %{ "lslv $dst, $src1, $src2" %}
9770
9771 ins_encode %{
9772 __ lslv(as_Register($dst$$reg),
9773 as_Register($src1$$reg),
9774 as_Register($src2$$reg));
9775 %}
9776
9777 ins_pipe(ialu_reg_reg_vshift);
9778 %}
9779
9780 // Shift Left Immediate
9781 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9782 match(Set dst (LShiftL src1 src2));
9783
9784 ins_cost(INSN_COST);
9785 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9786
9787 ins_encode %{
9788 __ lsl(as_Register($dst$$reg),
9789 as_Register($src1$$reg),
9790 $src2$$constant & 0x3f);
9791 %}
9792
9793 ins_pipe(ialu_reg_shift);
9794 %}
9795
9796 // Shift Right Logical Register
9797 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9798 match(Set dst (URShiftL src1 src2));
9799
9800 ins_cost(INSN_COST * 2);
9801 format %{ "lsrv $dst, $src1, $src2" %}
9802
9803 ins_encode %{
9804 __ lsrv(as_Register($dst$$reg),
9805 as_Register($src1$$reg),
9806 as_Register($src2$$reg));
9807 %}
9808
9809 ins_pipe(ialu_reg_reg_vshift);
9810 %}
9811
9812 // Shift Right Logical Immediate
9813 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9814 match(Set dst (URShiftL src1 src2));
9815
9816 ins_cost(INSN_COST);
9817 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9818
9819 ins_encode %{
9820 __ lsr(as_Register($dst$$reg),
9821 as_Register($src1$$reg),
9822 $src2$$constant & 0x3f);
9823 %}
9824
9825 ins_pipe(ialu_reg_shift);
9826 %}
9827
9828 // A special-case pattern for card table stores.
9829 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9830 match(Set dst (URShiftL (CastP2X src1) src2));
9831
9832 ins_cost(INSN_COST);
9833 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9834
9835 ins_encode %{
9836 __ lsr(as_Register($dst$$reg),
9837 as_Register($src1$$reg),
9838 $src2$$constant & 0x3f);
9839 %}
9840
9841 ins_pipe(ialu_reg_shift);
9842 %}
9843
9844 // Shift Right Arithmetic Register
9845 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9846 match(Set dst (RShiftL src1 src2));
9847
9848 ins_cost(INSN_COST * 2);
9849 format %{ "asrv $dst, $src1, $src2" %}
9850
9851 ins_encode %{
9852 __ asrv(as_Register($dst$$reg),
9853 as_Register($src1$$reg),
9854 as_Register($src2$$reg));
9855 %}
9856
9857 ins_pipe(ialu_reg_reg_vshift);
9858 %}
9859
9860 // Shift Right Arithmetic Immediate
9861 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9862 match(Set dst (RShiftL src1 src2));
9863
9864 ins_cost(INSN_COST);
9865 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9866
9867 ins_encode %{
9868 __ asr(as_Register($dst$$reg),
9869 as_Register($src1$$reg),
9870 $src2$$constant & 0x3f);
9871 %}
9872
9873 ins_pipe(ialu_reg_shift);
9874 %}
9875
9876 // BEGIN This section of the file is automatically generated. Do not edit --------------
9877 // This section is generated from aarch64_ad.m4
9878
9879 // This pattern is automatically generated from aarch64_ad.m4
9880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9881 instruct regL_not_reg(iRegLNoSp dst,
9882 iRegL src1, immL_M1 m1,
9883 rFlagsReg cr) %{
9884 match(Set dst (XorL src1 m1));
9885 ins_cost(INSN_COST);
9886 format %{ "eon $dst, $src1, zr" %}
9887
9888 ins_encode %{
9889 __ eon(as_Register($dst$$reg),
9890 as_Register($src1$$reg),
9891 zr,
9892 Assembler::LSL, 0);
9893 %}
9894
9895 ins_pipe(ialu_reg);
9896 %}
9897
9898 // This pattern is automatically generated from aarch64_ad.m4
9899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9900 instruct regI_not_reg(iRegINoSp dst,
9901 iRegIorL2I src1, immI_M1 m1,
9902 rFlagsReg cr) %{
9903 match(Set dst (XorI src1 m1));
9904 ins_cost(INSN_COST);
9905 format %{ "eonw $dst, $src1, zr" %}
9906
9907 ins_encode %{
9908 __ eonw(as_Register($dst$$reg),
9909 as_Register($src1$$reg),
9910 zr,
9911 Assembler::LSL, 0);
9912 %}
9913
9914 ins_pipe(ialu_reg);
9915 %}
9916
9917 // This pattern is automatically generated from aarch64_ad.m4
9918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9919 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9920 immI0 zero, iRegIorL2I src1, immI src2) %{
9921 match(Set dst (SubI zero (URShiftI src1 src2)));
9922
9923 ins_cost(1.9 * INSN_COST);
9924 format %{ "negw $dst, $src1, LSR $src2" %}
9925
9926 ins_encode %{
9927 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9928 Assembler::LSR, $src2$$constant & 0x1f);
9929 %}
9930
9931 ins_pipe(ialu_reg_shift);
9932 %}
9933
9934 // This pattern is automatically generated from aarch64_ad.m4
9935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9936 instruct NegI_reg_RShift_reg(iRegINoSp dst,
9937 immI0 zero, iRegIorL2I src1, immI src2) %{
9938 match(Set dst (SubI zero (RShiftI src1 src2)));
9939
9940 ins_cost(1.9 * INSN_COST);
9941 format %{ "negw $dst, $src1, ASR $src2" %}
9942
9943 ins_encode %{
9944 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9945 Assembler::ASR, $src2$$constant & 0x1f);
9946 %}
9947
9948 ins_pipe(ialu_reg_shift);
9949 %}
9950
9951 // This pattern is automatically generated from aarch64_ad.m4
9952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9953 instruct NegI_reg_LShift_reg(iRegINoSp dst,
9954 immI0 zero, iRegIorL2I src1, immI src2) %{
9955 match(Set dst (SubI zero (LShiftI src1 src2)));
9956
9957 ins_cost(1.9 * INSN_COST);
9958 format %{ "negw $dst, $src1, LSL $src2" %}
9959
9960 ins_encode %{
9961 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
9962 Assembler::LSL, $src2$$constant & 0x1f);
9963 %}
9964
9965 ins_pipe(ialu_reg_shift);
9966 %}
9967
9968 // This pattern is automatically generated from aarch64_ad.m4
9969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9970 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
9971 immL0 zero, iRegL src1, immI src2) %{
9972 match(Set dst (SubL zero (URShiftL src1 src2)));
9973
9974 ins_cost(1.9 * INSN_COST);
9975 format %{ "neg $dst, $src1, LSR $src2" %}
9976
9977 ins_encode %{
9978 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9979 Assembler::LSR, $src2$$constant & 0x3f);
9980 %}
9981
9982 ins_pipe(ialu_reg_shift);
9983 %}
9984
9985 // This pattern is automatically generated from aarch64_ad.m4
9986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9987 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
9988 immL0 zero, iRegL src1, immI src2) %{
9989 match(Set dst (SubL zero (RShiftL src1 src2)));
9990
9991 ins_cost(1.9 * INSN_COST);
9992 format %{ "neg $dst, $src1, ASR $src2" %}
9993
9994 ins_encode %{
9995 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
9996 Assembler::ASR, $src2$$constant & 0x3f);
9997 %}
9998
9999 ins_pipe(ialu_reg_shift);
10000 %}
10001
10002 // This pattern is automatically generated from aarch64_ad.m4
10003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10004 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10005 immL0 zero, iRegL src1, immI src2) %{
10006 match(Set dst (SubL zero (LShiftL src1 src2)));
10007
10008 ins_cost(1.9 * INSN_COST);
10009 format %{ "neg $dst, $src1, LSL $src2" %}
10010
10011 ins_encode %{
10012 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10013 Assembler::LSL, $src2$$constant & 0x3f);
10014 %}
10015
10016 ins_pipe(ialu_reg_shift);
10017 %}
10018
10019 // This pattern is automatically generated from aarch64_ad.m4
10020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10021 instruct AndI_reg_not_reg(iRegINoSp dst,
10022 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10023 match(Set dst (AndI src1 (XorI src2 m1)));
10024 ins_cost(INSN_COST);
10025 format %{ "bicw $dst, $src1, $src2" %}
10026
10027 ins_encode %{
10028 __ bicw(as_Register($dst$$reg),
10029 as_Register($src1$$reg),
10030 as_Register($src2$$reg),
10031 Assembler::LSL, 0);
10032 %}
10033
10034 ins_pipe(ialu_reg_reg);
10035 %}
10036
10037 // This pattern is automatically generated from aarch64_ad.m4
10038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10039 instruct AndL_reg_not_reg(iRegLNoSp dst,
10040 iRegL src1, iRegL src2, immL_M1 m1) %{
10041 match(Set dst (AndL src1 (XorL src2 m1)));
10042 ins_cost(INSN_COST);
10043 format %{ "bic $dst, $src1, $src2" %}
10044
10045 ins_encode %{
10046 __ bic(as_Register($dst$$reg),
10047 as_Register($src1$$reg),
10048 as_Register($src2$$reg),
10049 Assembler::LSL, 0);
10050 %}
10051
10052 ins_pipe(ialu_reg_reg);
10053 %}
10054
10055 // This pattern is automatically generated from aarch64_ad.m4
10056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10057 instruct OrI_reg_not_reg(iRegINoSp dst,
10058 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10059 match(Set dst (OrI src1 (XorI src2 m1)));
10060 ins_cost(INSN_COST);
10061 format %{ "ornw $dst, $src1, $src2" %}
10062
10063 ins_encode %{
10064 __ ornw(as_Register($dst$$reg),
10065 as_Register($src1$$reg),
10066 as_Register($src2$$reg),
10067 Assembler::LSL, 0);
10068 %}
10069
10070 ins_pipe(ialu_reg_reg);
10071 %}
10072
10073 // This pattern is automatically generated from aarch64_ad.m4
10074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10075 instruct OrL_reg_not_reg(iRegLNoSp dst,
10076 iRegL src1, iRegL src2, immL_M1 m1) %{
10077 match(Set dst (OrL src1 (XorL src2 m1)));
10078 ins_cost(INSN_COST);
10079 format %{ "orn $dst, $src1, $src2" %}
10080
10081 ins_encode %{
10082 __ orn(as_Register($dst$$reg),
10083 as_Register($src1$$reg),
10084 as_Register($src2$$reg),
10085 Assembler::LSL, 0);
10086 %}
10087
10088 ins_pipe(ialu_reg_reg);
10089 %}
10090
10091 // This pattern is automatically generated from aarch64_ad.m4
10092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10093 instruct XorI_reg_not_reg(iRegINoSp dst,
10094 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10095 match(Set dst (XorI m1 (XorI src2 src1)));
10096 ins_cost(INSN_COST);
10097 format %{ "eonw $dst, $src1, $src2" %}
10098
10099 ins_encode %{
10100 __ eonw(as_Register($dst$$reg),
10101 as_Register($src1$$reg),
10102 as_Register($src2$$reg),
10103 Assembler::LSL, 0);
10104 %}
10105
10106 ins_pipe(ialu_reg_reg);
10107 %}
10108
10109 // This pattern is automatically generated from aarch64_ad.m4
10110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10111 instruct XorL_reg_not_reg(iRegLNoSp dst,
10112 iRegL src1, iRegL src2, immL_M1 m1) %{
10113 match(Set dst (XorL m1 (XorL src2 src1)));
10114 ins_cost(INSN_COST);
10115 format %{ "eon $dst, $src1, $src2" %}
10116
10117 ins_encode %{
10118 __ eon(as_Register($dst$$reg),
10119 as_Register($src1$$reg),
10120 as_Register($src2$$reg),
10121 Assembler::LSL, 0);
10122 %}
10123
10124 ins_pipe(ialu_reg_reg);
10125 %}
10126
10127 // This pattern is automatically generated from aarch64_ad.m4
10128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10129 // val & (-1 ^ (val >>> shift)) ==> bicw
10130 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10131 iRegIorL2I src1, iRegIorL2I src2,
10132 immI src3, immI_M1 src4) %{
10133 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10134 ins_cost(1.9 * INSN_COST);
10135 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10136
10137 ins_encode %{
10138 __ bicw(as_Register($dst$$reg),
10139 as_Register($src1$$reg),
10140 as_Register($src2$$reg),
10141 Assembler::LSR,
10142 $src3$$constant & 0x1f);
10143 %}
10144
10145 ins_pipe(ialu_reg_reg_shift);
10146 %}
10147
10148 // This pattern is automatically generated from aarch64_ad.m4
10149 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10150 // val & (-1 ^ (val >>> shift)) ==> bic
10151 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10152 iRegL src1, iRegL src2,
10153 immI src3, immL_M1 src4) %{
10154 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10155 ins_cost(1.9 * INSN_COST);
10156 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10157
10158 ins_encode %{
10159 __ bic(as_Register($dst$$reg),
10160 as_Register($src1$$reg),
10161 as_Register($src2$$reg),
10162 Assembler::LSR,
10163 $src3$$constant & 0x3f);
10164 %}
10165
10166 ins_pipe(ialu_reg_reg_shift);
10167 %}
10168
10169 // This pattern is automatically generated from aarch64_ad.m4
10170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10171 // val & (-1 ^ (val >> shift)) ==> bicw
10172 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10173 iRegIorL2I src1, iRegIorL2I src2,
10174 immI src3, immI_M1 src4) %{
10175 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10176 ins_cost(1.9 * INSN_COST);
10177 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10178
10179 ins_encode %{
10180 __ bicw(as_Register($dst$$reg),
10181 as_Register($src1$$reg),
10182 as_Register($src2$$reg),
10183 Assembler::ASR,
10184 $src3$$constant & 0x1f);
10185 %}
10186
10187 ins_pipe(ialu_reg_reg_shift);
10188 %}
10189
10190 // This pattern is automatically generated from aarch64_ad.m4
10191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10192 // val & (-1 ^ (val >> shift)) ==> bic
10193 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10194 iRegL src1, iRegL src2,
10195 immI src3, immL_M1 src4) %{
10196 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10197 ins_cost(1.9 * INSN_COST);
10198 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10199
10200 ins_encode %{
10201 __ bic(as_Register($dst$$reg),
10202 as_Register($src1$$reg),
10203 as_Register($src2$$reg),
10204 Assembler::ASR,
10205 $src3$$constant & 0x3f);
10206 %}
10207
10208 ins_pipe(ialu_reg_reg_shift);
10209 %}
10210
10211 // This pattern is automatically generated from aarch64_ad.m4
10212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10213 // val & (-1 ^ (val ror shift)) ==> bicw
10214 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10215 iRegIorL2I src1, iRegIorL2I src2,
10216 immI src3, immI_M1 src4) %{
10217 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10218 ins_cost(1.9 * INSN_COST);
10219 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10220
10221 ins_encode %{
10222 __ bicw(as_Register($dst$$reg),
10223 as_Register($src1$$reg),
10224 as_Register($src2$$reg),
10225 Assembler::ROR,
10226 $src3$$constant & 0x1f);
10227 %}
10228
10229 ins_pipe(ialu_reg_reg_shift);
10230 %}
10231
10232 // This pattern is automatically generated from aarch64_ad.m4
10233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10234 // val & (-1 ^ (val ror shift)) ==> bic
10235 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10236 iRegL src1, iRegL src2,
10237 immI src3, immL_M1 src4) %{
10238 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10239 ins_cost(1.9 * INSN_COST);
10240 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10241
10242 ins_encode %{
10243 __ bic(as_Register($dst$$reg),
10244 as_Register($src1$$reg),
10245 as_Register($src2$$reg),
10246 Assembler::ROR,
10247 $src3$$constant & 0x3f);
10248 %}
10249
10250 ins_pipe(ialu_reg_reg_shift);
10251 %}
10252
10253 // This pattern is automatically generated from aarch64_ad.m4
10254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10255 // val & (-1 ^ (val << shift)) ==> bicw
10256 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10257 iRegIorL2I src1, iRegIorL2I src2,
10258 immI src3, immI_M1 src4) %{
10259 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10260 ins_cost(1.9 * INSN_COST);
10261 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10262
10263 ins_encode %{
10264 __ bicw(as_Register($dst$$reg),
10265 as_Register($src1$$reg),
10266 as_Register($src2$$reg),
10267 Assembler::LSL,
10268 $src3$$constant & 0x1f);
10269 %}
10270
10271 ins_pipe(ialu_reg_reg_shift);
10272 %}
10273
10274 // This pattern is automatically generated from aarch64_ad.m4
10275 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10276 // val & (-1 ^ (val << shift)) ==> bic
10277 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10278 iRegL src1, iRegL src2,
10279 immI src3, immL_M1 src4) %{
10280 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10281 ins_cost(1.9 * INSN_COST);
10282 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10283
10284 ins_encode %{
10285 __ bic(as_Register($dst$$reg),
10286 as_Register($src1$$reg),
10287 as_Register($src2$$reg),
10288 Assembler::LSL,
10289 $src3$$constant & 0x3f);
10290 %}
10291
10292 ins_pipe(ialu_reg_reg_shift);
10293 %}
10294
10295 // This pattern is automatically generated from aarch64_ad.m4
10296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10297 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10298 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10299 iRegIorL2I src1, iRegIorL2I src2,
10300 immI src3, immI_M1 src4) %{
10301 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10302 ins_cost(1.9 * INSN_COST);
10303 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10304
10305 ins_encode %{
10306 __ eonw(as_Register($dst$$reg),
10307 as_Register($src1$$reg),
10308 as_Register($src2$$reg),
10309 Assembler::LSR,
10310 $src3$$constant & 0x1f);
10311 %}
10312
10313 ins_pipe(ialu_reg_reg_shift);
10314 %}
10315
10316 // This pattern is automatically generated from aarch64_ad.m4
10317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10318 // val ^ (-1 ^ (val >>> shift)) ==> eon
10319 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10320 iRegL src1, iRegL src2,
10321 immI src3, immL_M1 src4) %{
10322 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10323 ins_cost(1.9 * INSN_COST);
10324 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10325
10326 ins_encode %{
10327 __ eon(as_Register($dst$$reg),
10328 as_Register($src1$$reg),
10329 as_Register($src2$$reg),
10330 Assembler::LSR,
10331 $src3$$constant & 0x3f);
10332 %}
10333
10334 ins_pipe(ialu_reg_reg_shift);
10335 %}
10336
10337 // This pattern is automatically generated from aarch64_ad.m4
10338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10339 // val ^ (-1 ^ (val >> shift)) ==> eonw
10340 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10341 iRegIorL2I src1, iRegIorL2I src2,
10342 immI src3, immI_M1 src4) %{
10343 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10344 ins_cost(1.9 * INSN_COST);
10345 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10346
10347 ins_encode %{
10348 __ eonw(as_Register($dst$$reg),
10349 as_Register($src1$$reg),
10350 as_Register($src2$$reg),
10351 Assembler::ASR,
10352 $src3$$constant & 0x1f);
10353 %}
10354
10355 ins_pipe(ialu_reg_reg_shift);
10356 %}
10357
10358 // This pattern is automatically generated from aarch64_ad.m4
10359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10360 // val ^ (-1 ^ (val >> shift)) ==> eon
10361 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10362 iRegL src1, iRegL src2,
10363 immI src3, immL_M1 src4) %{
10364 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10365 ins_cost(1.9 * INSN_COST);
10366 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10367
10368 ins_encode %{
10369 __ eon(as_Register($dst$$reg),
10370 as_Register($src1$$reg),
10371 as_Register($src2$$reg),
10372 Assembler::ASR,
10373 $src3$$constant & 0x3f);
10374 %}
10375
10376 ins_pipe(ialu_reg_reg_shift);
10377 %}
10378
10379 // This pattern is automatically generated from aarch64_ad.m4
10380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10381 // val ^ (-1 ^ (val ror shift)) ==> eonw
10382 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10383 iRegIorL2I src1, iRegIorL2I src2,
10384 immI src3, immI_M1 src4) %{
10385 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10386 ins_cost(1.9 * INSN_COST);
10387 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10388
10389 ins_encode %{
10390 __ eonw(as_Register($dst$$reg),
10391 as_Register($src1$$reg),
10392 as_Register($src2$$reg),
10393 Assembler::ROR,
10394 $src3$$constant & 0x1f);
10395 %}
10396
10397 ins_pipe(ialu_reg_reg_shift);
10398 %}
10399
10400 // This pattern is automatically generated from aarch64_ad.m4
10401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10402 // val ^ (-1 ^ (val ror shift)) ==> eon
10403 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10404 iRegL src1, iRegL src2,
10405 immI src3, immL_M1 src4) %{
10406 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10407 ins_cost(1.9 * INSN_COST);
10408 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10409
10410 ins_encode %{
10411 __ eon(as_Register($dst$$reg),
10412 as_Register($src1$$reg),
10413 as_Register($src2$$reg),
10414 Assembler::ROR,
10415 $src3$$constant & 0x3f);
10416 %}
10417
10418 ins_pipe(ialu_reg_reg_shift);
10419 %}
10420
10421 // This pattern is automatically generated from aarch64_ad.m4
10422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10423 // val ^ (-1 ^ (val << shift)) ==> eonw
10424 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10425 iRegIorL2I src1, iRegIorL2I src2,
10426 immI src3, immI_M1 src4) %{
10427 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10428 ins_cost(1.9 * INSN_COST);
10429 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10430
10431 ins_encode %{
10432 __ eonw(as_Register($dst$$reg),
10433 as_Register($src1$$reg),
10434 as_Register($src2$$reg),
10435 Assembler::LSL,
10436 $src3$$constant & 0x1f);
10437 %}
10438
10439 ins_pipe(ialu_reg_reg_shift);
10440 %}
10441
10442 // This pattern is automatically generated from aarch64_ad.m4
10443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10444 // val ^ (-1 ^ (val << shift)) ==> eon
10445 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10446 iRegL src1, iRegL src2,
10447 immI src3, immL_M1 src4) %{
10448 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10449 ins_cost(1.9 * INSN_COST);
10450 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10451
10452 ins_encode %{
10453 __ eon(as_Register($dst$$reg),
10454 as_Register($src1$$reg),
10455 as_Register($src2$$reg),
10456 Assembler::LSL,
10457 $src3$$constant & 0x3f);
10458 %}
10459
10460 ins_pipe(ialu_reg_reg_shift);
10461 %}
10462
10463 // This pattern is automatically generated from aarch64_ad.m4
10464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10465 // val | (-1 ^ (val >>> shift)) ==> ornw
10466 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10467 iRegIorL2I src1, iRegIorL2I src2,
10468 immI src3, immI_M1 src4) %{
10469 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10470 ins_cost(1.9 * INSN_COST);
10471 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10472
10473 ins_encode %{
10474 __ ornw(as_Register($dst$$reg),
10475 as_Register($src1$$reg),
10476 as_Register($src2$$reg),
10477 Assembler::LSR,
10478 $src3$$constant & 0x1f);
10479 %}
10480
10481 ins_pipe(ialu_reg_reg_shift);
10482 %}
10483
10484 // This pattern is automatically generated from aarch64_ad.m4
10485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10486 // val | (-1 ^ (val >>> shift)) ==> orn
10487 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10488 iRegL src1, iRegL src2,
10489 immI src3, immL_M1 src4) %{
10490 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10491 ins_cost(1.9 * INSN_COST);
10492 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10493
10494 ins_encode %{
10495 __ orn(as_Register($dst$$reg),
10496 as_Register($src1$$reg),
10497 as_Register($src2$$reg),
10498 Assembler::LSR,
10499 $src3$$constant & 0x3f);
10500 %}
10501
10502 ins_pipe(ialu_reg_reg_shift);
10503 %}
10504
10505 // This pattern is automatically generated from aarch64_ad.m4
10506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10507 // val | (-1 ^ (val >> shift)) ==> ornw
10508 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10509 iRegIorL2I src1, iRegIorL2I src2,
10510 immI src3, immI_M1 src4) %{
10511 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10512 ins_cost(1.9 * INSN_COST);
10513 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10514
10515 ins_encode %{
10516 __ ornw(as_Register($dst$$reg),
10517 as_Register($src1$$reg),
10518 as_Register($src2$$reg),
10519 Assembler::ASR,
10520 $src3$$constant & 0x1f);
10521 %}
10522
10523 ins_pipe(ialu_reg_reg_shift);
10524 %}
10525
10526 // This pattern is automatically generated from aarch64_ad.m4
10527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10528 // val | (-1 ^ (val >> shift)) ==> orn
10529 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10530 iRegL src1, iRegL src2,
10531 immI src3, immL_M1 src4) %{
10532 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10533 ins_cost(1.9 * INSN_COST);
10534 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10535
10536 ins_encode %{
10537 __ orn(as_Register($dst$$reg),
10538 as_Register($src1$$reg),
10539 as_Register($src2$$reg),
10540 Assembler::ASR,
10541 $src3$$constant & 0x3f);
10542 %}
10543
10544 ins_pipe(ialu_reg_reg_shift);
10545 %}
10546
10547 // This pattern is automatically generated from aarch64_ad.m4
10548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10549 // val | (-1 ^ (val ror shift)) ==> ornw
10550 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10551 iRegIorL2I src1, iRegIorL2I src2,
10552 immI src3, immI_M1 src4) %{
10553 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10554 ins_cost(1.9 * INSN_COST);
10555 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10556
10557 ins_encode %{
10558 __ ornw(as_Register($dst$$reg),
10559 as_Register($src1$$reg),
10560 as_Register($src2$$reg),
10561 Assembler::ROR,
10562 $src3$$constant & 0x1f);
10563 %}
10564
10565 ins_pipe(ialu_reg_reg_shift);
10566 %}
10567
10568 // This pattern is automatically generated from aarch64_ad.m4
10569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10570 // val | (-1 ^ (val ror shift)) ==> orn
10571 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10572 iRegL src1, iRegL src2,
10573 immI src3, immL_M1 src4) %{
10574 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10575 ins_cost(1.9 * INSN_COST);
10576 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10577
10578 ins_encode %{
10579 __ orn(as_Register($dst$$reg),
10580 as_Register($src1$$reg),
10581 as_Register($src2$$reg),
10582 Assembler::ROR,
10583 $src3$$constant & 0x3f);
10584 %}
10585
10586 ins_pipe(ialu_reg_reg_shift);
10587 %}
10588
10589 // This pattern is automatically generated from aarch64_ad.m4
10590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10591 // val | (-1 ^ (val << shift)) ==> ornw
10592 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10593 iRegIorL2I src1, iRegIorL2I src2,
10594 immI src3, immI_M1 src4) %{
10595 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10596 ins_cost(1.9 * INSN_COST);
10597 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10598
10599 ins_encode %{
10600 __ ornw(as_Register($dst$$reg),
10601 as_Register($src1$$reg),
10602 as_Register($src2$$reg),
10603 Assembler::LSL,
10604 $src3$$constant & 0x1f);
10605 %}
10606
10607 ins_pipe(ialu_reg_reg_shift);
10608 %}
10609
10610 // This pattern is automatically generated from aarch64_ad.m4
10611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10612 // val | (-1 ^ (val << shift)) ==> orn
10613 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10614 iRegL src1, iRegL src2,
10615 immI src3, immL_M1 src4) %{
10616 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10617 ins_cost(1.9 * INSN_COST);
10618 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10619
10620 ins_encode %{
10621 __ orn(as_Register($dst$$reg),
10622 as_Register($src1$$reg),
10623 as_Register($src2$$reg),
10624 Assembler::LSL,
10625 $src3$$constant & 0x3f);
10626 %}
10627
10628 ins_pipe(ialu_reg_reg_shift);
10629 %}
10630
10631 // This pattern is automatically generated from aarch64_ad.m4
10632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10633 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10634 iRegIorL2I src1, iRegIorL2I src2,
10635 immI src3) %{
10636 match(Set dst (AndI src1 (URShiftI src2 src3)));
10637
10638 ins_cost(1.9 * INSN_COST);
10639 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10640
10641 ins_encode %{
10642 __ andw(as_Register($dst$$reg),
10643 as_Register($src1$$reg),
10644 as_Register($src2$$reg),
10645 Assembler::LSR,
10646 $src3$$constant & 0x1f);
10647 %}
10648
10649 ins_pipe(ialu_reg_reg_shift);
10650 %}
10651
10652 // This pattern is automatically generated from aarch64_ad.m4
10653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10654 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10655 iRegL src1, iRegL src2,
10656 immI src3) %{
10657 match(Set dst (AndL src1 (URShiftL src2 src3)));
10658
10659 ins_cost(1.9 * INSN_COST);
10660 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10661
10662 ins_encode %{
10663 __ andr(as_Register($dst$$reg),
10664 as_Register($src1$$reg),
10665 as_Register($src2$$reg),
10666 Assembler::LSR,
10667 $src3$$constant & 0x3f);
10668 %}
10669
10670 ins_pipe(ialu_reg_reg_shift);
10671 %}
10672
10673 // This pattern is automatically generated from aarch64_ad.m4
10674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10675 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10676 iRegIorL2I src1, iRegIorL2I src2,
10677 immI src3) %{
10678 match(Set dst (AndI src1 (RShiftI src2 src3)));
10679
10680 ins_cost(1.9 * INSN_COST);
10681 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10682
10683 ins_encode %{
10684 __ andw(as_Register($dst$$reg),
10685 as_Register($src1$$reg),
10686 as_Register($src2$$reg),
10687 Assembler::ASR,
10688 $src3$$constant & 0x1f);
10689 %}
10690
10691 ins_pipe(ialu_reg_reg_shift);
10692 %}
10693
10694 // This pattern is automatically generated from aarch64_ad.m4
10695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10696 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10697 iRegL src1, iRegL src2,
10698 immI src3) %{
10699 match(Set dst (AndL src1 (RShiftL src2 src3)));
10700
10701 ins_cost(1.9 * INSN_COST);
10702 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10703
10704 ins_encode %{
10705 __ andr(as_Register($dst$$reg),
10706 as_Register($src1$$reg),
10707 as_Register($src2$$reg),
10708 Assembler::ASR,
10709 $src3$$constant & 0x3f);
10710 %}
10711
10712 ins_pipe(ialu_reg_reg_shift);
10713 %}
10714
10715 // This pattern is automatically generated from aarch64_ad.m4
10716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10717 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10718 iRegIorL2I src1, iRegIorL2I src2,
10719 immI src3) %{
10720 match(Set dst (AndI src1 (LShiftI src2 src3)));
10721
10722 ins_cost(1.9 * INSN_COST);
10723 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10724
10725 ins_encode %{
10726 __ andw(as_Register($dst$$reg),
10727 as_Register($src1$$reg),
10728 as_Register($src2$$reg),
10729 Assembler::LSL,
10730 $src3$$constant & 0x1f);
10731 %}
10732
10733 ins_pipe(ialu_reg_reg_shift);
10734 %}
10735
10736 // This pattern is automatically generated from aarch64_ad.m4
10737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10738 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10739 iRegL src1, iRegL src2,
10740 immI src3) %{
10741 match(Set dst (AndL src1 (LShiftL src2 src3)));
10742
10743 ins_cost(1.9 * INSN_COST);
10744 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10745
10746 ins_encode %{
10747 __ andr(as_Register($dst$$reg),
10748 as_Register($src1$$reg),
10749 as_Register($src2$$reg),
10750 Assembler::LSL,
10751 $src3$$constant & 0x3f);
10752 %}
10753
10754 ins_pipe(ialu_reg_reg_shift);
10755 %}
10756
10757 // This pattern is automatically generated from aarch64_ad.m4
10758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10759 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10760 iRegIorL2I src1, iRegIorL2I src2,
10761 immI src3) %{
10762 match(Set dst (AndI src1 (RotateRight src2 src3)));
10763
10764 ins_cost(1.9 * INSN_COST);
10765 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10766
10767 ins_encode %{
10768 __ andw(as_Register($dst$$reg),
10769 as_Register($src1$$reg),
10770 as_Register($src2$$reg),
10771 Assembler::ROR,
10772 $src3$$constant & 0x1f);
10773 %}
10774
10775 ins_pipe(ialu_reg_reg_shift);
10776 %}
10777
10778 // This pattern is automatically generated from aarch64_ad.m4
10779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10780 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10781 iRegL src1, iRegL src2,
10782 immI src3) %{
10783 match(Set dst (AndL src1 (RotateRight src2 src3)));
10784
10785 ins_cost(1.9 * INSN_COST);
10786 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10787
10788 ins_encode %{
10789 __ andr(as_Register($dst$$reg),
10790 as_Register($src1$$reg),
10791 as_Register($src2$$reg),
10792 Assembler::ROR,
10793 $src3$$constant & 0x3f);
10794 %}
10795
10796 ins_pipe(ialu_reg_reg_shift);
10797 %}
10798
10799 // This pattern is automatically generated from aarch64_ad.m4
10800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10801 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10802 iRegIorL2I src1, iRegIorL2I src2,
10803 immI src3) %{
10804 match(Set dst (XorI src1 (URShiftI src2 src3)));
10805
10806 ins_cost(1.9 * INSN_COST);
10807 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10808
10809 ins_encode %{
10810 __ eorw(as_Register($dst$$reg),
10811 as_Register($src1$$reg),
10812 as_Register($src2$$reg),
10813 Assembler::LSR,
10814 $src3$$constant & 0x1f);
10815 %}
10816
10817 ins_pipe(ialu_reg_reg_shift);
10818 %}
10819
10820 // This pattern is automatically generated from aarch64_ad.m4
10821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10822 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10823 iRegL src1, iRegL src2,
10824 immI src3) %{
10825 match(Set dst (XorL src1 (URShiftL src2 src3)));
10826
10827 ins_cost(1.9 * INSN_COST);
10828 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10829
10830 ins_encode %{
10831 __ eor(as_Register($dst$$reg),
10832 as_Register($src1$$reg),
10833 as_Register($src2$$reg),
10834 Assembler::LSR,
10835 $src3$$constant & 0x3f);
10836 %}
10837
10838 ins_pipe(ialu_reg_reg_shift);
10839 %}
10840
10841 // This pattern is automatically generated from aarch64_ad.m4
10842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10843 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10844 iRegIorL2I src1, iRegIorL2I src2,
10845 immI src3) %{
10846 match(Set dst (XorI src1 (RShiftI src2 src3)));
10847
10848 ins_cost(1.9 * INSN_COST);
10849 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10850
10851 ins_encode %{
10852 __ eorw(as_Register($dst$$reg),
10853 as_Register($src1$$reg),
10854 as_Register($src2$$reg),
10855 Assembler::ASR,
10856 $src3$$constant & 0x1f);
10857 %}
10858
10859 ins_pipe(ialu_reg_reg_shift);
10860 %}
10861
10862 // This pattern is automatically generated from aarch64_ad.m4
10863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10864 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10865 iRegL src1, iRegL src2,
10866 immI src3) %{
10867 match(Set dst (XorL src1 (RShiftL src2 src3)));
10868
10869 ins_cost(1.9 * INSN_COST);
10870 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10871
10872 ins_encode %{
10873 __ eor(as_Register($dst$$reg),
10874 as_Register($src1$$reg),
10875 as_Register($src2$$reg),
10876 Assembler::ASR,
10877 $src3$$constant & 0x3f);
10878 %}
10879
10880 ins_pipe(ialu_reg_reg_shift);
10881 %}
10882
10883 // This pattern is automatically generated from aarch64_ad.m4
10884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10885 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10886 iRegIorL2I src1, iRegIorL2I src2,
10887 immI src3) %{
10888 match(Set dst (XorI src1 (LShiftI src2 src3)));
10889
10890 ins_cost(1.9 * INSN_COST);
10891 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10892
10893 ins_encode %{
10894 __ eorw(as_Register($dst$$reg),
10895 as_Register($src1$$reg),
10896 as_Register($src2$$reg),
10897 Assembler::LSL,
10898 $src3$$constant & 0x1f);
10899 %}
10900
10901 ins_pipe(ialu_reg_reg_shift);
10902 %}
10903
10904 // This pattern is automatically generated from aarch64_ad.m4
10905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10906 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10907 iRegL src1, iRegL src2,
10908 immI src3) %{
10909 match(Set dst (XorL src1 (LShiftL src2 src3)));
10910
10911 ins_cost(1.9 * INSN_COST);
10912 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10913
10914 ins_encode %{
10915 __ eor(as_Register($dst$$reg),
10916 as_Register($src1$$reg),
10917 as_Register($src2$$reg),
10918 Assembler::LSL,
10919 $src3$$constant & 0x3f);
10920 %}
10921
10922 ins_pipe(ialu_reg_reg_shift);
10923 %}
10924
10925 // This pattern is automatically generated from aarch64_ad.m4
10926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10927 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
10928 iRegIorL2I src1, iRegIorL2I src2,
10929 immI src3) %{
10930 match(Set dst (XorI src1 (RotateRight src2 src3)));
10931
10932 ins_cost(1.9 * INSN_COST);
10933 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
10934
10935 ins_encode %{
10936 __ eorw(as_Register($dst$$reg),
10937 as_Register($src1$$reg),
10938 as_Register($src2$$reg),
10939 Assembler::ROR,
10940 $src3$$constant & 0x1f);
10941 %}
10942
10943 ins_pipe(ialu_reg_reg_shift);
10944 %}
10945
10946 // This pattern is automatically generated from aarch64_ad.m4
10947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10948 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
10949 iRegL src1, iRegL src2,
10950 immI src3) %{
10951 match(Set dst (XorL src1 (RotateRight src2 src3)));
10952
10953 ins_cost(1.9 * INSN_COST);
10954 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
10955
10956 ins_encode %{
10957 __ eor(as_Register($dst$$reg),
10958 as_Register($src1$$reg),
10959 as_Register($src2$$reg),
10960 Assembler::ROR,
10961 $src3$$constant & 0x3f);
10962 %}
10963
10964 ins_pipe(ialu_reg_reg_shift);
10965 %}
10966
10967 // This pattern is automatically generated from aarch64_ad.m4
10968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10969 instruct OrI_reg_URShift_reg(iRegINoSp dst,
10970 iRegIorL2I src1, iRegIorL2I src2,
10971 immI src3) %{
10972 match(Set dst (OrI src1 (URShiftI src2 src3)));
10973
10974 ins_cost(1.9 * INSN_COST);
10975 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
10976
10977 ins_encode %{
10978 __ orrw(as_Register($dst$$reg),
10979 as_Register($src1$$reg),
10980 as_Register($src2$$reg),
10981 Assembler::LSR,
10982 $src3$$constant & 0x1f);
10983 %}
10984
10985 ins_pipe(ialu_reg_reg_shift);
10986 %}
10987
10988 // This pattern is automatically generated from aarch64_ad.m4
10989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10990 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
10991 iRegL src1, iRegL src2,
10992 immI src3) %{
10993 match(Set dst (OrL src1 (URShiftL src2 src3)));
10994
10995 ins_cost(1.9 * INSN_COST);
10996 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
10997
10998 ins_encode %{
10999 __ orr(as_Register($dst$$reg),
11000 as_Register($src1$$reg),
11001 as_Register($src2$$reg),
11002 Assembler::LSR,
11003 $src3$$constant & 0x3f);
11004 %}
11005
11006 ins_pipe(ialu_reg_reg_shift);
11007 %}
11008
11009 // This pattern is automatically generated from aarch64_ad.m4
11010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11011 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11012 iRegIorL2I src1, iRegIorL2I src2,
11013 immI src3) %{
11014 match(Set dst (OrI src1 (RShiftI src2 src3)));
11015
11016 ins_cost(1.9 * INSN_COST);
11017 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11018
11019 ins_encode %{
11020 __ orrw(as_Register($dst$$reg),
11021 as_Register($src1$$reg),
11022 as_Register($src2$$reg),
11023 Assembler::ASR,
11024 $src3$$constant & 0x1f);
11025 %}
11026
11027 ins_pipe(ialu_reg_reg_shift);
11028 %}
11029
11030 // This pattern is automatically generated from aarch64_ad.m4
11031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11032 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11033 iRegL src1, iRegL src2,
11034 immI src3) %{
11035 match(Set dst (OrL src1 (RShiftL src2 src3)));
11036
11037 ins_cost(1.9 * INSN_COST);
11038 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11039
11040 ins_encode %{
11041 __ orr(as_Register($dst$$reg),
11042 as_Register($src1$$reg),
11043 as_Register($src2$$reg),
11044 Assembler::ASR,
11045 $src3$$constant & 0x3f);
11046 %}
11047
11048 ins_pipe(ialu_reg_reg_shift);
11049 %}
11050
11051 // This pattern is automatically generated from aarch64_ad.m4
11052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11053 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11054 iRegIorL2I src1, iRegIorL2I src2,
11055 immI src3) %{
11056 match(Set dst (OrI src1 (LShiftI src2 src3)));
11057
11058 ins_cost(1.9 * INSN_COST);
11059 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11060
11061 ins_encode %{
11062 __ orrw(as_Register($dst$$reg),
11063 as_Register($src1$$reg),
11064 as_Register($src2$$reg),
11065 Assembler::LSL,
11066 $src3$$constant & 0x1f);
11067 %}
11068
11069 ins_pipe(ialu_reg_reg_shift);
11070 %}
11071
11072 // This pattern is automatically generated from aarch64_ad.m4
11073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11074 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11075 iRegL src1, iRegL src2,
11076 immI src3) %{
11077 match(Set dst (OrL src1 (LShiftL src2 src3)));
11078
11079 ins_cost(1.9 * INSN_COST);
11080 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11081
11082 ins_encode %{
11083 __ orr(as_Register($dst$$reg),
11084 as_Register($src1$$reg),
11085 as_Register($src2$$reg),
11086 Assembler::LSL,
11087 $src3$$constant & 0x3f);
11088 %}
11089
11090 ins_pipe(ialu_reg_reg_shift);
11091 %}
11092
11093 // This pattern is automatically generated from aarch64_ad.m4
11094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11095 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11096 iRegIorL2I src1, iRegIorL2I src2,
11097 immI src3) %{
11098 match(Set dst (OrI src1 (RotateRight src2 src3)));
11099
11100 ins_cost(1.9 * INSN_COST);
11101 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11102
11103 ins_encode %{
11104 __ orrw(as_Register($dst$$reg),
11105 as_Register($src1$$reg),
11106 as_Register($src2$$reg),
11107 Assembler::ROR,
11108 $src3$$constant & 0x1f);
11109 %}
11110
11111 ins_pipe(ialu_reg_reg_shift);
11112 %}
11113
11114 // This pattern is automatically generated from aarch64_ad.m4
11115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11116 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11117 iRegL src1, iRegL src2,
11118 immI src3) %{
11119 match(Set dst (OrL src1 (RotateRight src2 src3)));
11120
11121 ins_cost(1.9 * INSN_COST);
11122 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11123
11124 ins_encode %{
11125 __ orr(as_Register($dst$$reg),
11126 as_Register($src1$$reg),
11127 as_Register($src2$$reg),
11128 Assembler::ROR,
11129 $src3$$constant & 0x3f);
11130 %}
11131
11132 ins_pipe(ialu_reg_reg_shift);
11133 %}
11134
11135 // This pattern is automatically generated from aarch64_ad.m4
11136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11137 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11138 iRegIorL2I src1, iRegIorL2I src2,
11139 immI src3) %{
11140 match(Set dst (AddI src1 (URShiftI src2 src3)));
11141
11142 ins_cost(1.9 * INSN_COST);
11143 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11144
11145 ins_encode %{
11146 __ addw(as_Register($dst$$reg),
11147 as_Register($src1$$reg),
11148 as_Register($src2$$reg),
11149 Assembler::LSR,
11150 $src3$$constant & 0x1f);
11151 %}
11152
11153 ins_pipe(ialu_reg_reg_shift);
11154 %}
11155
11156 // This pattern is automatically generated from aarch64_ad.m4
11157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11158 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11159 iRegL src1, iRegL src2,
11160 immI src3) %{
11161 match(Set dst (AddL src1 (URShiftL src2 src3)));
11162
11163 ins_cost(1.9 * INSN_COST);
11164 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11165
11166 ins_encode %{
11167 __ add(as_Register($dst$$reg),
11168 as_Register($src1$$reg),
11169 as_Register($src2$$reg),
11170 Assembler::LSR,
11171 $src3$$constant & 0x3f);
11172 %}
11173
11174 ins_pipe(ialu_reg_reg_shift);
11175 %}
11176
11177 // This pattern is automatically generated from aarch64_ad.m4
11178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11179 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11180 iRegIorL2I src1, iRegIorL2I src2,
11181 immI src3) %{
11182 match(Set dst (AddI src1 (RShiftI src2 src3)));
11183
11184 ins_cost(1.9 * INSN_COST);
11185 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11186
11187 ins_encode %{
11188 __ addw(as_Register($dst$$reg),
11189 as_Register($src1$$reg),
11190 as_Register($src2$$reg),
11191 Assembler::ASR,
11192 $src3$$constant & 0x1f);
11193 %}
11194
11195 ins_pipe(ialu_reg_reg_shift);
11196 %}
11197
11198 // This pattern is automatically generated from aarch64_ad.m4
11199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11200 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11201 iRegL src1, iRegL src2,
11202 immI src3) %{
11203 match(Set dst (AddL src1 (RShiftL src2 src3)));
11204
11205 ins_cost(1.9 * INSN_COST);
11206 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11207
11208 ins_encode %{
11209 __ add(as_Register($dst$$reg),
11210 as_Register($src1$$reg),
11211 as_Register($src2$$reg),
11212 Assembler::ASR,
11213 $src3$$constant & 0x3f);
11214 %}
11215
11216 ins_pipe(ialu_reg_reg_shift);
11217 %}
11218
11219 // This pattern is automatically generated from aarch64_ad.m4
11220 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11221 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11222 iRegIorL2I src1, iRegIorL2I src2,
11223 immI src3) %{
11224 match(Set dst (AddI src1 (LShiftI src2 src3)));
11225
11226 ins_cost(1.9 * INSN_COST);
11227 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11228
11229 ins_encode %{
11230 __ addw(as_Register($dst$$reg),
11231 as_Register($src1$$reg),
11232 as_Register($src2$$reg),
11233 Assembler::LSL,
11234 $src3$$constant & 0x1f);
11235 %}
11236
11237 ins_pipe(ialu_reg_reg_shift);
11238 %}
11239
11240 // This pattern is automatically generated from aarch64_ad.m4
11241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11242 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11243 iRegL src1, iRegL src2,
11244 immI src3) %{
11245 match(Set dst (AddL src1 (LShiftL src2 src3)));
11246
11247 ins_cost(1.9 * INSN_COST);
11248 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11249
11250 ins_encode %{
11251 __ add(as_Register($dst$$reg),
11252 as_Register($src1$$reg),
11253 as_Register($src2$$reg),
11254 Assembler::LSL,
11255 $src3$$constant & 0x3f);
11256 %}
11257
11258 ins_pipe(ialu_reg_reg_shift);
11259 %}
11260
11261 // This pattern is automatically generated from aarch64_ad.m4
11262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11263 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11264 iRegIorL2I src1, iRegIorL2I src2,
11265 immI src3) %{
11266 match(Set dst (SubI src1 (URShiftI src2 src3)));
11267
11268 ins_cost(1.9 * INSN_COST);
11269 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11270
11271 ins_encode %{
11272 __ subw(as_Register($dst$$reg),
11273 as_Register($src1$$reg),
11274 as_Register($src2$$reg),
11275 Assembler::LSR,
11276 $src3$$constant & 0x1f);
11277 %}
11278
11279 ins_pipe(ialu_reg_reg_shift);
11280 %}
11281
11282 // This pattern is automatically generated from aarch64_ad.m4
11283 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11284 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11285 iRegL src1, iRegL src2,
11286 immI src3) %{
11287 match(Set dst (SubL src1 (URShiftL src2 src3)));
11288
11289 ins_cost(1.9 * INSN_COST);
11290 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11291
11292 ins_encode %{
11293 __ sub(as_Register($dst$$reg),
11294 as_Register($src1$$reg),
11295 as_Register($src2$$reg),
11296 Assembler::LSR,
11297 $src3$$constant & 0x3f);
11298 %}
11299
11300 ins_pipe(ialu_reg_reg_shift);
11301 %}
11302
11303 // This pattern is automatically generated from aarch64_ad.m4
11304 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11305 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11306 iRegIorL2I src1, iRegIorL2I src2,
11307 immI src3) %{
11308 match(Set dst (SubI src1 (RShiftI src2 src3)));
11309
11310 ins_cost(1.9 * INSN_COST);
11311 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11312
11313 ins_encode %{
11314 __ subw(as_Register($dst$$reg),
11315 as_Register($src1$$reg),
11316 as_Register($src2$$reg),
11317 Assembler::ASR,
11318 $src3$$constant & 0x1f);
11319 %}
11320
11321 ins_pipe(ialu_reg_reg_shift);
11322 %}
11323
11324 // This pattern is automatically generated from aarch64_ad.m4
11325 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11326 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11327 iRegL src1, iRegL src2,
11328 immI src3) %{
11329 match(Set dst (SubL src1 (RShiftL src2 src3)));
11330
11331 ins_cost(1.9 * INSN_COST);
11332 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11333
11334 ins_encode %{
11335 __ sub(as_Register($dst$$reg),
11336 as_Register($src1$$reg),
11337 as_Register($src2$$reg),
11338 Assembler::ASR,
11339 $src3$$constant & 0x3f);
11340 %}
11341
11342 ins_pipe(ialu_reg_reg_shift);
11343 %}
11344
11345 // This pattern is automatically generated from aarch64_ad.m4
11346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11347 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11348 iRegIorL2I src1, iRegIorL2I src2,
11349 immI src3) %{
11350 match(Set dst (SubI src1 (LShiftI src2 src3)));
11351
11352 ins_cost(1.9 * INSN_COST);
11353 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11354
11355 ins_encode %{
11356 __ subw(as_Register($dst$$reg),
11357 as_Register($src1$$reg),
11358 as_Register($src2$$reg),
11359 Assembler::LSL,
11360 $src3$$constant & 0x1f);
11361 %}
11362
11363 ins_pipe(ialu_reg_reg_shift);
11364 %}
11365
11366 // This pattern is automatically generated from aarch64_ad.m4
11367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11368 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11369 iRegL src1, iRegL src2,
11370 immI src3) %{
11371 match(Set dst (SubL src1 (LShiftL src2 src3)));
11372
11373 ins_cost(1.9 * INSN_COST);
11374 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11375
11376 ins_encode %{
11377 __ sub(as_Register($dst$$reg),
11378 as_Register($src1$$reg),
11379 as_Register($src2$$reg),
11380 Assembler::LSL,
11381 $src3$$constant & 0x3f);
11382 %}
11383
11384 ins_pipe(ialu_reg_reg_shift);
11385 %}
11386
11387 // This pattern is automatically generated from aarch64_ad.m4
11388 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11389
11390 // Shift Left followed by Shift Right.
11391 // This idiom is used by the compiler for the i2b bytecode etc.
11392 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11393 %{
11394 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11395 ins_cost(INSN_COST * 2);
11396 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11397 ins_encode %{
11398 int lshift = $lshift_count$$constant & 63;
11399 int rshift = $rshift_count$$constant & 63;
11400 int s = 63 - lshift;
11401 int r = (rshift - lshift) & 63;
11402 __ sbfm(as_Register($dst$$reg),
11403 as_Register($src$$reg),
11404 r, s);
11405 %}
11406
11407 ins_pipe(ialu_reg_shift);
11408 %}
11409
11410 // This pattern is automatically generated from aarch64_ad.m4
11411 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11412
11413 // Shift Left followed by Shift Right.
11414 // This idiom is used by the compiler for the i2b bytecode etc.
11415 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11416 %{
11417 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11418 ins_cost(INSN_COST * 2);
11419 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11420 ins_encode %{
11421 int lshift = $lshift_count$$constant & 31;
11422 int rshift = $rshift_count$$constant & 31;
11423 int s = 31 - lshift;
11424 int r = (rshift - lshift) & 31;
11425 __ sbfmw(as_Register($dst$$reg),
11426 as_Register($src$$reg),
11427 r, s);
11428 %}
11429
11430 ins_pipe(ialu_reg_shift);
11431 %}
11432
11433 // This pattern is automatically generated from aarch64_ad.m4
11434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11435
11436 // Shift Left followed by Shift Right.
11437 // This idiom is used by the compiler for the i2b bytecode etc.
11438 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11439 %{
11440 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11441 ins_cost(INSN_COST * 2);
11442 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11443 ins_encode %{
11444 int lshift = $lshift_count$$constant & 63;
11445 int rshift = $rshift_count$$constant & 63;
11446 int s = 63 - lshift;
11447 int r = (rshift - lshift) & 63;
11448 __ ubfm(as_Register($dst$$reg),
11449 as_Register($src$$reg),
11450 r, s);
11451 %}
11452
11453 ins_pipe(ialu_reg_shift);
11454 %}
11455
11456 // This pattern is automatically generated from aarch64_ad.m4
11457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11458
11459 // Shift Left followed by Shift Right.
11460 // This idiom is used by the compiler for the i2b bytecode etc.
11461 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11462 %{
11463 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11464 ins_cost(INSN_COST * 2);
11465 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11466 ins_encode %{
11467 int lshift = $lshift_count$$constant & 31;
11468 int rshift = $rshift_count$$constant & 31;
11469 int s = 31 - lshift;
11470 int r = (rshift - lshift) & 31;
11471 __ ubfmw(as_Register($dst$$reg),
11472 as_Register($src$$reg),
11473 r, s);
11474 %}
11475
11476 ins_pipe(ialu_reg_shift);
11477 %}
11478
11479 // Bitfield extract with shift & mask
11480
11481 // This pattern is automatically generated from aarch64_ad.m4
11482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11483 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11484 %{
11485 match(Set dst (AndI (URShiftI src rshift) mask));
11486 // Make sure we are not going to exceed what ubfxw can do.
11487 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11488
11489 ins_cost(INSN_COST);
11490 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11491 ins_encode %{
11492 int rshift = $rshift$$constant & 31;
11493 intptr_t mask = $mask$$constant;
11494 int width = exact_log2(mask+1);
11495 __ ubfxw(as_Register($dst$$reg),
11496 as_Register($src$$reg), rshift, width);
11497 %}
11498 ins_pipe(ialu_reg_shift);
11499 %}
11500
11501 // This pattern is automatically generated from aarch64_ad.m4
11502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11503 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11504 %{
11505 match(Set dst (AndL (URShiftL src rshift) mask));
11506 // Make sure we are not going to exceed what ubfx can do.
11507 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11508
11509 ins_cost(INSN_COST);
11510 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11511 ins_encode %{
11512 int rshift = $rshift$$constant & 63;
11513 intptr_t mask = $mask$$constant;
11514 int width = exact_log2_long(mask+1);
11515 __ ubfx(as_Register($dst$$reg),
11516 as_Register($src$$reg), rshift, width);
11517 %}
11518 ins_pipe(ialu_reg_shift);
11519 %}
11520
11521
11522 // This pattern is automatically generated from aarch64_ad.m4
11523 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11524
11525 // We can use ubfx when extending an And with a mask when we know mask
11526 // is positive. We know that because immI_bitmask guarantees it.
11527 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11528 %{
11529 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11530 // Make sure we are not going to exceed what ubfxw can do.
11531 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11532
11533 ins_cost(INSN_COST * 2);
11534 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11535 ins_encode %{
11536 int rshift = $rshift$$constant & 31;
11537 intptr_t mask = $mask$$constant;
11538 int width = exact_log2(mask+1);
11539 __ ubfx(as_Register($dst$$reg),
11540 as_Register($src$$reg), rshift, width);
11541 %}
11542 ins_pipe(ialu_reg_shift);
11543 %}
11544
11545
11546 // This pattern is automatically generated from aarch64_ad.m4
11547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11548
11549 // We can use ubfiz when masking by a positive number and then left shifting the result.
11550 // We know that the mask is positive because immI_bitmask guarantees it.
11551 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11552 %{
11553 match(Set dst (LShiftI (AndI src mask) lshift));
11554 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11555
11556 ins_cost(INSN_COST);
11557 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11558 ins_encode %{
11559 int lshift = $lshift$$constant & 31;
11560 intptr_t mask = $mask$$constant;
11561 int width = exact_log2(mask+1);
11562 __ ubfizw(as_Register($dst$$reg),
11563 as_Register($src$$reg), lshift, width);
11564 %}
11565 ins_pipe(ialu_reg_shift);
11566 %}
11567
11568 // This pattern is automatically generated from aarch64_ad.m4
11569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11570
11571 // We can use ubfiz when masking by a positive number and then left shifting the result.
11572 // We know that the mask is positive because immL_bitmask guarantees it.
11573 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11574 %{
11575 match(Set dst (LShiftL (AndL src mask) lshift));
11576 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11577
11578 ins_cost(INSN_COST);
11579 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11580 ins_encode %{
11581 int lshift = $lshift$$constant & 63;
11582 intptr_t mask = $mask$$constant;
11583 int width = exact_log2_long(mask+1);
11584 __ ubfiz(as_Register($dst$$reg),
11585 as_Register($src$$reg), lshift, width);
11586 %}
11587 ins_pipe(ialu_reg_shift);
11588 %}
11589
11590 // This pattern is automatically generated from aarch64_ad.m4
11591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11592
11593 // We can use ubfiz when masking by a positive number and then left shifting the result.
11594 // We know that the mask is positive because immI_bitmask guarantees it.
11595 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11596 %{
11597 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11598 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11599
11600 ins_cost(INSN_COST);
11601 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11602 ins_encode %{
11603 int lshift = $lshift$$constant & 31;
11604 intptr_t mask = $mask$$constant;
11605 int width = exact_log2(mask+1);
11606 __ ubfizw(as_Register($dst$$reg),
11607 as_Register($src$$reg), lshift, width);
11608 %}
11609 ins_pipe(ialu_reg_shift);
11610 %}
11611
11612 // This pattern is automatically generated from aarch64_ad.m4
11613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11614
11615 // We can use ubfiz when masking by a positive number and then left shifting the result.
11616 // We know that the mask is positive because immL_bitmask guarantees it.
11617 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11618 %{
11619 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11620 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11621
11622 ins_cost(INSN_COST);
11623 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11624 ins_encode %{
11625 int lshift = $lshift$$constant & 63;
11626 intptr_t mask = $mask$$constant;
11627 int width = exact_log2_long(mask+1);
11628 __ ubfiz(as_Register($dst$$reg),
11629 as_Register($src$$reg), lshift, width);
11630 %}
11631 ins_pipe(ialu_reg_shift);
11632 %}
11633
11634
11635 // This pattern is automatically generated from aarch64_ad.m4
11636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11637
11638 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11639 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11640 %{
11641 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11642 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11643
11644 ins_cost(INSN_COST);
11645 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11646 ins_encode %{
11647 int lshift = $lshift$$constant & 63;
11648 intptr_t mask = $mask$$constant;
11649 int width = exact_log2(mask+1);
11650 __ ubfiz(as_Register($dst$$reg),
11651 as_Register($src$$reg), lshift, width);
11652 %}
11653 ins_pipe(ialu_reg_shift);
11654 %}
11655
11656 // This pattern is automatically generated from aarch64_ad.m4
11657 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11658
11659 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11660 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11661 %{
11662 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11663 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11664
11665 ins_cost(INSN_COST);
11666 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11667 ins_encode %{
11668 int lshift = $lshift$$constant & 31;
11669 intptr_t mask = $mask$$constant;
11670 int width = exact_log2(mask+1);
11671 __ ubfiz(as_Register($dst$$reg),
11672 as_Register($src$$reg), lshift, width);
11673 %}
11674 ins_pipe(ialu_reg_shift);
11675 %}
11676
11677 // This pattern is automatically generated from aarch64_ad.m4
11678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11679
11680 // Can skip int2long conversions after AND with small bitmask
11681 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11682 %{
11683 match(Set dst (ConvI2L (AndI src msk)));
11684 ins_cost(INSN_COST);
11685 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11686 ins_encode %{
11687 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11688 %}
11689 ins_pipe(ialu_reg_shift);
11690 %}
11691
11692
11693 // Rotations
11694
11695 // This pattern is automatically generated from aarch64_ad.m4
11696 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11697 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11698 %{
11699 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11700 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11701
11702 ins_cost(INSN_COST);
11703 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11704
11705 ins_encode %{
11706 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11707 $rshift$$constant & 63);
11708 %}
11709 ins_pipe(ialu_reg_reg_extr);
11710 %}
11711
11712
11713 // This pattern is automatically generated from aarch64_ad.m4
11714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11715 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11716 %{
11717 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11718 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11719
11720 ins_cost(INSN_COST);
11721 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11722
11723 ins_encode %{
11724 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11725 $rshift$$constant & 31);
11726 %}
11727 ins_pipe(ialu_reg_reg_extr);
11728 %}
11729
11730
11731 // This pattern is automatically generated from aarch64_ad.m4
11732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11733 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11734 %{
11735 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11736 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11737
11738 ins_cost(INSN_COST);
11739 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11740
11741 ins_encode %{
11742 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11743 $rshift$$constant & 63);
11744 %}
11745 ins_pipe(ialu_reg_reg_extr);
11746 %}
11747
11748
11749 // This pattern is automatically generated from aarch64_ad.m4
11750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11751 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11752 %{
11753 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11754 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11755
11756 ins_cost(INSN_COST);
11757 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11758
11759 ins_encode %{
11760 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11761 $rshift$$constant & 31);
11762 %}
11763 ins_pipe(ialu_reg_reg_extr);
11764 %}
11765
11766 // This pattern is automatically generated from aarch64_ad.m4
11767 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11768 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11769 %{
11770 match(Set dst (RotateRight src shift));
11771
11772 ins_cost(INSN_COST);
11773 format %{ "ror $dst, $src, $shift" %}
11774
11775 ins_encode %{
11776 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11777 $shift$$constant & 0x1f);
11778 %}
11779 ins_pipe(ialu_reg_reg_vshift);
11780 %}
11781
11782 // This pattern is automatically generated from aarch64_ad.m4
11783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11784 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11785 %{
11786 match(Set dst (RotateRight src shift));
11787
11788 ins_cost(INSN_COST);
11789 format %{ "ror $dst, $src, $shift" %}
11790
11791 ins_encode %{
11792 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11793 $shift$$constant & 0x3f);
11794 %}
11795 ins_pipe(ialu_reg_reg_vshift);
11796 %}
11797
11798 // This pattern is automatically generated from aarch64_ad.m4
11799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11800 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11801 %{
11802 match(Set dst (RotateRight src shift));
11803
11804 ins_cost(INSN_COST);
11805 format %{ "ror $dst, $src, $shift" %}
11806
11807 ins_encode %{
11808 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11809 %}
11810 ins_pipe(ialu_reg_reg_vshift);
11811 %}
11812
11813 // This pattern is automatically generated from aarch64_ad.m4
11814 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11815 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11816 %{
11817 match(Set dst (RotateRight src shift));
11818
11819 ins_cost(INSN_COST);
11820 format %{ "ror $dst, $src, $shift" %}
11821
11822 ins_encode %{
11823 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11824 %}
11825 ins_pipe(ialu_reg_reg_vshift);
11826 %}
11827
11828 // This pattern is automatically generated from aarch64_ad.m4
11829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11830 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11831 %{
11832 match(Set dst (RotateLeft src shift));
11833
11834 ins_cost(INSN_COST);
11835 format %{ "rol $dst, $src, $shift" %}
11836
11837 ins_encode %{
11838 __ subw(rscratch1, zr, as_Register($shift$$reg));
11839 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11840 %}
11841 ins_pipe(ialu_reg_reg_vshift);
11842 %}
11843
11844 // This pattern is automatically generated from aarch64_ad.m4
11845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11846 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11847 %{
11848 match(Set dst (RotateLeft src shift));
11849
11850 ins_cost(INSN_COST);
11851 format %{ "rol $dst, $src, $shift" %}
11852
11853 ins_encode %{
11854 __ subw(rscratch1, zr, as_Register($shift$$reg));
11855 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11856 %}
11857 ins_pipe(ialu_reg_reg_vshift);
11858 %}
11859
11860
11861 // Add/subtract (extended)
11862
11863 // This pattern is automatically generated from aarch64_ad.m4
11864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11865 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11866 %{
11867 match(Set dst (AddL src1 (ConvI2L src2)));
11868 ins_cost(INSN_COST);
11869 format %{ "add $dst, $src1, $src2, sxtw" %}
11870
11871 ins_encode %{
11872 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11873 as_Register($src2$$reg), ext::sxtw);
11874 %}
11875 ins_pipe(ialu_reg_reg);
11876 %}
11877
11878 // This pattern is automatically generated from aarch64_ad.m4
11879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11880 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11881 %{
11882 match(Set dst (SubL src1 (ConvI2L src2)));
11883 ins_cost(INSN_COST);
11884 format %{ "sub $dst, $src1, $src2, sxtw" %}
11885
11886 ins_encode %{
11887 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11888 as_Register($src2$$reg), ext::sxtw);
11889 %}
11890 ins_pipe(ialu_reg_reg);
11891 %}
11892
11893 // This pattern is automatically generated from aarch64_ad.m4
11894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11895 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11896 %{
11897 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11898 ins_cost(INSN_COST);
11899 format %{ "add $dst, $src1, $src2, sxth" %}
11900
11901 ins_encode %{
11902 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11903 as_Register($src2$$reg), ext::sxth);
11904 %}
11905 ins_pipe(ialu_reg_reg);
11906 %}
11907
11908 // This pattern is automatically generated from aarch64_ad.m4
11909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11910 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11911 %{
11912 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11913 ins_cost(INSN_COST);
11914 format %{ "add $dst, $src1, $src2, sxtb" %}
11915
11916 ins_encode %{
11917 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11918 as_Register($src2$$reg), ext::sxtb);
11919 %}
11920 ins_pipe(ialu_reg_reg);
11921 %}
11922
11923 // This pattern is automatically generated from aarch64_ad.m4
11924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11925 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11926 %{
11927 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
11928 ins_cost(INSN_COST);
11929 format %{ "add $dst, $src1, $src2, uxtb" %}
11930
11931 ins_encode %{
11932 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11933 as_Register($src2$$reg), ext::uxtb);
11934 %}
11935 ins_pipe(ialu_reg_reg);
11936 %}
11937
11938 // This pattern is automatically generated from aarch64_ad.m4
11939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11940 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
11941 %{
11942 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11943 ins_cost(INSN_COST);
11944 format %{ "add $dst, $src1, $src2, sxth" %}
11945
11946 ins_encode %{
11947 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11948 as_Register($src2$$reg), ext::sxth);
11949 %}
11950 ins_pipe(ialu_reg_reg);
11951 %}
11952
11953 // This pattern is automatically generated from aarch64_ad.m4
11954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11955 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
11956 %{
11957 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11958 ins_cost(INSN_COST);
11959 format %{ "add $dst, $src1, $src2, sxtw" %}
11960
11961 ins_encode %{
11962 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11963 as_Register($src2$$reg), ext::sxtw);
11964 %}
11965 ins_pipe(ialu_reg_reg);
11966 %}
11967
11968 // This pattern is automatically generated from aarch64_ad.m4
11969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11970 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11971 %{
11972 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
11973 ins_cost(INSN_COST);
11974 format %{ "add $dst, $src1, $src2, sxtb" %}
11975
11976 ins_encode %{
11977 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11978 as_Register($src2$$reg), ext::sxtb);
11979 %}
11980 ins_pipe(ialu_reg_reg);
11981 %}
11982
11983 // This pattern is automatically generated from aarch64_ad.m4
11984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11985 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
11986 %{
11987 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
11988 ins_cost(INSN_COST);
11989 format %{ "add $dst, $src1, $src2, uxtb" %}
11990
11991 ins_encode %{
11992 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11993 as_Register($src2$$reg), ext::uxtb);
11994 %}
11995 ins_pipe(ialu_reg_reg);
11996 %}
11997
11998 // This pattern is automatically generated from aarch64_ad.m4
11999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12000 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12001 %{
12002 match(Set dst (AddI src1 (AndI src2 mask)));
12003 ins_cost(INSN_COST);
12004 format %{ "addw $dst, $src1, $src2, uxtb" %}
12005
12006 ins_encode %{
12007 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12008 as_Register($src2$$reg), ext::uxtb);
12009 %}
12010 ins_pipe(ialu_reg_reg);
12011 %}
12012
12013 // This pattern is automatically generated from aarch64_ad.m4
12014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12015 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12016 %{
12017 match(Set dst (AddI src1 (AndI src2 mask)));
12018 ins_cost(INSN_COST);
12019 format %{ "addw $dst, $src1, $src2, uxth" %}
12020
12021 ins_encode %{
12022 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12023 as_Register($src2$$reg), ext::uxth);
12024 %}
12025 ins_pipe(ialu_reg_reg);
12026 %}
12027
12028 // This pattern is automatically generated from aarch64_ad.m4
12029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12030 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12031 %{
12032 match(Set dst (AddL src1 (AndL src2 mask)));
12033 ins_cost(INSN_COST);
12034 format %{ "add $dst, $src1, $src2, uxtb" %}
12035
12036 ins_encode %{
12037 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12038 as_Register($src2$$reg), ext::uxtb);
12039 %}
12040 ins_pipe(ialu_reg_reg);
12041 %}
12042
12043 // This pattern is automatically generated from aarch64_ad.m4
12044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12045 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12046 %{
12047 match(Set dst (AddL src1 (AndL src2 mask)));
12048 ins_cost(INSN_COST);
12049 format %{ "add $dst, $src1, $src2, uxth" %}
12050
12051 ins_encode %{
12052 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12053 as_Register($src2$$reg), ext::uxth);
12054 %}
12055 ins_pipe(ialu_reg_reg);
12056 %}
12057
12058 // This pattern is automatically generated from aarch64_ad.m4
12059 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12060 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12061 %{
12062 match(Set dst (AddL src1 (AndL src2 mask)));
12063 ins_cost(INSN_COST);
12064 format %{ "add $dst, $src1, $src2, uxtw" %}
12065
12066 ins_encode %{
12067 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12068 as_Register($src2$$reg), ext::uxtw);
12069 %}
12070 ins_pipe(ialu_reg_reg);
12071 %}
12072
12073 // This pattern is automatically generated from aarch64_ad.m4
12074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12075 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12076 %{
12077 match(Set dst (SubI src1 (AndI src2 mask)));
12078 ins_cost(INSN_COST);
12079 format %{ "subw $dst, $src1, $src2, uxtb" %}
12080
12081 ins_encode %{
12082 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12083 as_Register($src2$$reg), ext::uxtb);
12084 %}
12085 ins_pipe(ialu_reg_reg);
12086 %}
12087
12088 // This pattern is automatically generated from aarch64_ad.m4
12089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12090 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12091 %{
12092 match(Set dst (SubI src1 (AndI src2 mask)));
12093 ins_cost(INSN_COST);
12094 format %{ "subw $dst, $src1, $src2, uxth" %}
12095
12096 ins_encode %{
12097 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12098 as_Register($src2$$reg), ext::uxth);
12099 %}
12100 ins_pipe(ialu_reg_reg);
12101 %}
12102
12103 // This pattern is automatically generated from aarch64_ad.m4
12104 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12105 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12106 %{
12107 match(Set dst (SubL src1 (AndL src2 mask)));
12108 ins_cost(INSN_COST);
12109 format %{ "sub $dst, $src1, $src2, uxtb" %}
12110
12111 ins_encode %{
12112 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12113 as_Register($src2$$reg), ext::uxtb);
12114 %}
12115 ins_pipe(ialu_reg_reg);
12116 %}
12117
12118 // This pattern is automatically generated from aarch64_ad.m4
12119 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12120 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12121 %{
12122 match(Set dst (SubL src1 (AndL src2 mask)));
12123 ins_cost(INSN_COST);
12124 format %{ "sub $dst, $src1, $src2, uxth" %}
12125
12126 ins_encode %{
12127 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12128 as_Register($src2$$reg), ext::uxth);
12129 %}
12130 ins_pipe(ialu_reg_reg);
12131 %}
12132
12133 // This pattern is automatically generated from aarch64_ad.m4
12134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12135 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12136 %{
12137 match(Set dst (SubL src1 (AndL src2 mask)));
12138 ins_cost(INSN_COST);
12139 format %{ "sub $dst, $src1, $src2, uxtw" %}
12140
12141 ins_encode %{
12142 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12143 as_Register($src2$$reg), ext::uxtw);
12144 %}
12145 ins_pipe(ialu_reg_reg);
12146 %}
12147
12148
12149 // This pattern is automatically generated from aarch64_ad.m4
12150 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12151 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12152 %{
12153 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12154 ins_cost(1.9 * INSN_COST);
12155 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12156
12157 ins_encode %{
12158 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12159 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12160 %}
12161 ins_pipe(ialu_reg_reg_shift);
12162 %}
12163
12164 // This pattern is automatically generated from aarch64_ad.m4
12165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12166 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12167 %{
12168 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12169 ins_cost(1.9 * INSN_COST);
12170 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12171
12172 ins_encode %{
12173 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12174 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12175 %}
12176 ins_pipe(ialu_reg_reg_shift);
12177 %}
12178
12179 // This pattern is automatically generated from aarch64_ad.m4
12180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12181 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12182 %{
12183 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12184 ins_cost(1.9 * INSN_COST);
12185 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12186
12187 ins_encode %{
12188 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12189 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12190 %}
12191 ins_pipe(ialu_reg_reg_shift);
12192 %}
12193
12194 // This pattern is automatically generated from aarch64_ad.m4
12195 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12196 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12197 %{
12198 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12199 ins_cost(1.9 * INSN_COST);
12200 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12201
12202 ins_encode %{
12203 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12204 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12205 %}
12206 ins_pipe(ialu_reg_reg_shift);
12207 %}
12208
12209 // This pattern is automatically generated from aarch64_ad.m4
12210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12211 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12212 %{
12213 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12214 ins_cost(1.9 * INSN_COST);
12215 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12216
12217 ins_encode %{
12218 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12219 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12220 %}
12221 ins_pipe(ialu_reg_reg_shift);
12222 %}
12223
12224 // This pattern is automatically generated from aarch64_ad.m4
12225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12226 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12227 %{
12228 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12229 ins_cost(1.9 * INSN_COST);
12230 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12231
12232 ins_encode %{
12233 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12234 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12235 %}
12236 ins_pipe(ialu_reg_reg_shift);
12237 %}
12238
12239 // This pattern is automatically generated from aarch64_ad.m4
12240 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12241 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12242 %{
12243 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12244 ins_cost(1.9 * INSN_COST);
12245 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12246
12247 ins_encode %{
12248 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12249 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12250 %}
12251 ins_pipe(ialu_reg_reg_shift);
12252 %}
12253
12254 // This pattern is automatically generated from aarch64_ad.m4
12255 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12256 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12257 %{
12258 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12259 ins_cost(1.9 * INSN_COST);
12260 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12261
12262 ins_encode %{
12263 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12264 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12265 %}
12266 ins_pipe(ialu_reg_reg_shift);
12267 %}
12268
12269 // This pattern is automatically generated from aarch64_ad.m4
12270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12271 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12272 %{
12273 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12274 ins_cost(1.9 * INSN_COST);
12275 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12276
12277 ins_encode %{
12278 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12279 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12280 %}
12281 ins_pipe(ialu_reg_reg_shift);
12282 %}
12283
12284 // This pattern is automatically generated from aarch64_ad.m4
12285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12286 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12287 %{
12288 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12289 ins_cost(1.9 * INSN_COST);
12290 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12291
12292 ins_encode %{
12293 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12294 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12295 %}
12296 ins_pipe(ialu_reg_reg_shift);
12297 %}
12298
12299 // This pattern is automatically generated from aarch64_ad.m4
12300 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12301 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12302 %{
12303 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12304 ins_cost(1.9 * INSN_COST);
12305 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12306
12307 ins_encode %{
12308 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12309 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12310 %}
12311 ins_pipe(ialu_reg_reg_shift);
12312 %}
12313
12314 // This pattern is automatically generated from aarch64_ad.m4
12315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12316 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12317 %{
12318 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12319 ins_cost(1.9 * INSN_COST);
12320 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12321
12322 ins_encode %{
12323 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12324 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12325 %}
12326 ins_pipe(ialu_reg_reg_shift);
12327 %}
12328
12329 // This pattern is automatically generated from aarch64_ad.m4
12330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12331 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12332 %{
12333 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12334 ins_cost(1.9 * INSN_COST);
12335 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12336
12337 ins_encode %{
12338 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12339 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12340 %}
12341 ins_pipe(ialu_reg_reg_shift);
12342 %}
12343
12344 // This pattern is automatically generated from aarch64_ad.m4
12345 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12346 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12347 %{
12348 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12349 ins_cost(1.9 * INSN_COST);
12350 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12351
12352 ins_encode %{
12353 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12354 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12355 %}
12356 ins_pipe(ialu_reg_reg_shift);
12357 %}
12358
12359 // This pattern is automatically generated from aarch64_ad.m4
12360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12361 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12362 %{
12363 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12364 ins_cost(1.9 * INSN_COST);
12365 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12366
12367 ins_encode %{
12368 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12369 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12370 %}
12371 ins_pipe(ialu_reg_reg_shift);
12372 %}
12373
12374 // This pattern is automatically generated from aarch64_ad.m4
12375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12376 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12377 %{
12378 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12379 ins_cost(1.9 * INSN_COST);
12380 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12381
12382 ins_encode %{
12383 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12384 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12385 %}
12386 ins_pipe(ialu_reg_reg_shift);
12387 %}
12388
12389 // This pattern is automatically generated from aarch64_ad.m4
12390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12391 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12392 %{
12393 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12394 ins_cost(1.9 * INSN_COST);
12395 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12396
12397 ins_encode %{
12398 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12399 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12400 %}
12401 ins_pipe(ialu_reg_reg_shift);
12402 %}
12403
12404 // This pattern is automatically generated from aarch64_ad.m4
12405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12406 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12407 %{
12408 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12409 ins_cost(1.9 * INSN_COST);
12410 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12411
12412 ins_encode %{
12413 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12414 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12415 %}
12416 ins_pipe(ialu_reg_reg_shift);
12417 %}
12418
12419 // This pattern is automatically generated from aarch64_ad.m4
12420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12421 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12422 %{
12423 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12424 ins_cost(1.9 * INSN_COST);
12425 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12426
12427 ins_encode %{
12428 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12429 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12430 %}
12431 ins_pipe(ialu_reg_reg_shift);
12432 %}
12433
12434 // This pattern is automatically generated from aarch64_ad.m4
12435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12436 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12437 %{
12438 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12439 ins_cost(1.9 * INSN_COST);
12440 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12441
12442 ins_encode %{
12443 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12444 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12445 %}
12446 ins_pipe(ialu_reg_reg_shift);
12447 %}
12448
12449 // This pattern is automatically generated from aarch64_ad.m4
12450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12451 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12452 %{
12453 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12454 ins_cost(1.9 * INSN_COST);
12455 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12456
12457 ins_encode %{
12458 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12459 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12460 %}
12461 ins_pipe(ialu_reg_reg_shift);
12462 %}
12463
12464 // This pattern is automatically generated from aarch64_ad.m4
12465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12466 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12467 %{
12468 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12469 ins_cost(1.9 * INSN_COST);
12470 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12471
12472 ins_encode %{
12473 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12474 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12475 %}
12476 ins_pipe(ialu_reg_reg_shift);
12477 %}
12478
12479 // This pattern is automatically generated from aarch64_ad.m4
12480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12481 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12482 %{
12483 effect(DEF dst, USE src1, USE src2, USE cr);
12484 ins_cost(INSN_COST * 2);
12485 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12486
12487 ins_encode %{
12488 __ cselw($dst$$Register,
12489 $src1$$Register,
12490 $src2$$Register,
12491 Assembler::LT);
12492 %}
12493 ins_pipe(icond_reg_reg);
12494 %}
12495
12496 // This pattern is automatically generated from aarch64_ad.m4
12497 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12498 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12499 %{
12500 effect(DEF dst, USE src1, USE src2, USE cr);
12501 ins_cost(INSN_COST * 2);
12502 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12503
12504 ins_encode %{
12505 __ cselw($dst$$Register,
12506 $src1$$Register,
12507 $src2$$Register,
12508 Assembler::GT);
12509 %}
12510 ins_pipe(icond_reg_reg);
12511 %}
12512
12513 // This pattern is automatically generated from aarch64_ad.m4
12514 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12515 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12516 %{
12517 effect(DEF dst, USE src1, USE cr);
12518 ins_cost(INSN_COST * 2);
12519 format %{ "cselw $dst, $src1, zr lt\t" %}
12520
12521 ins_encode %{
12522 __ cselw($dst$$Register,
12523 $src1$$Register,
12524 zr,
12525 Assembler::LT);
12526 %}
12527 ins_pipe(icond_reg);
12528 %}
12529
12530 // This pattern is automatically generated from aarch64_ad.m4
12531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12532 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12533 %{
12534 effect(DEF dst, USE src1, USE cr);
12535 ins_cost(INSN_COST * 2);
12536 format %{ "cselw $dst, $src1, zr gt\t" %}
12537
12538 ins_encode %{
12539 __ cselw($dst$$Register,
12540 $src1$$Register,
12541 zr,
12542 Assembler::GT);
12543 %}
12544 ins_pipe(icond_reg);
12545 %}
12546
12547 // This pattern is automatically generated from aarch64_ad.m4
12548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12549 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12550 %{
12551 effect(DEF dst, USE src1, USE cr);
12552 ins_cost(INSN_COST * 2);
12553 format %{ "csincw $dst, $src1, zr le\t" %}
12554
12555 ins_encode %{
12556 __ csincw($dst$$Register,
12557 $src1$$Register,
12558 zr,
12559 Assembler::LE);
12560 %}
12561 ins_pipe(icond_reg);
12562 %}
12563
12564 // This pattern is automatically generated from aarch64_ad.m4
12565 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12566 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12567 %{
12568 effect(DEF dst, USE src1, USE cr);
12569 ins_cost(INSN_COST * 2);
12570 format %{ "csincw $dst, $src1, zr gt\t" %}
12571
12572 ins_encode %{
12573 __ csincw($dst$$Register,
12574 $src1$$Register,
12575 zr,
12576 Assembler::GT);
12577 %}
12578 ins_pipe(icond_reg);
12579 %}
12580
12581 // This pattern is automatically generated from aarch64_ad.m4
12582 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12583 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12584 %{
12585 effect(DEF dst, USE src1, USE cr);
12586 ins_cost(INSN_COST * 2);
12587 format %{ "csinvw $dst, $src1, zr lt\t" %}
12588
12589 ins_encode %{
12590 __ csinvw($dst$$Register,
12591 $src1$$Register,
12592 zr,
12593 Assembler::LT);
12594 %}
12595 ins_pipe(icond_reg);
12596 %}
12597
12598 // This pattern is automatically generated from aarch64_ad.m4
12599 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12600 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12601 %{
12602 effect(DEF dst, USE src1, USE cr);
12603 ins_cost(INSN_COST * 2);
12604 format %{ "csinvw $dst, $src1, zr ge\t" %}
12605
12606 ins_encode %{
12607 __ csinvw($dst$$Register,
12608 $src1$$Register,
12609 zr,
12610 Assembler::GE);
12611 %}
12612 ins_pipe(icond_reg);
12613 %}
12614
12615 // This pattern is automatically generated from aarch64_ad.m4
12616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12617 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12618 %{
12619 match(Set dst (MinI src imm));
12620 ins_cost(INSN_COST * 3);
12621 expand %{
12622 rFlagsReg cr;
12623 compI_reg_imm0(cr, src);
12624 cmovI_reg_imm0_lt(dst, src, cr);
12625 %}
12626 %}
12627
12628 // This pattern is automatically generated from aarch64_ad.m4
12629 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12630 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12631 %{
12632 match(Set dst (MinI imm src));
12633 ins_cost(INSN_COST * 3);
12634 expand %{
12635 rFlagsReg cr;
12636 compI_reg_imm0(cr, src);
12637 cmovI_reg_imm0_lt(dst, src, cr);
12638 %}
12639 %}
12640
12641 // This pattern is automatically generated from aarch64_ad.m4
12642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12643 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12644 %{
12645 match(Set dst (MinI src imm));
12646 ins_cost(INSN_COST * 3);
12647 expand %{
12648 rFlagsReg cr;
12649 compI_reg_imm0(cr, src);
12650 cmovI_reg_imm1_le(dst, src, cr);
12651 %}
12652 %}
12653
12654 // This pattern is automatically generated from aarch64_ad.m4
12655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12656 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12657 %{
12658 match(Set dst (MinI imm src));
12659 ins_cost(INSN_COST * 3);
12660 expand %{
12661 rFlagsReg cr;
12662 compI_reg_imm0(cr, src);
12663 cmovI_reg_imm1_le(dst, src, cr);
12664 %}
12665 %}
12666
12667 // This pattern is automatically generated from aarch64_ad.m4
12668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12669 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12670 %{
12671 match(Set dst (MinI src imm));
12672 ins_cost(INSN_COST * 3);
12673 expand %{
12674 rFlagsReg cr;
12675 compI_reg_imm0(cr, src);
12676 cmovI_reg_immM1_lt(dst, src, cr);
12677 %}
12678 %}
12679
12680 // This pattern is automatically generated from aarch64_ad.m4
12681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12682 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12683 %{
12684 match(Set dst (MinI imm src));
12685 ins_cost(INSN_COST * 3);
12686 expand %{
12687 rFlagsReg cr;
12688 compI_reg_imm0(cr, src);
12689 cmovI_reg_immM1_lt(dst, src, cr);
12690 %}
12691 %}
12692
12693 // This pattern is automatically generated from aarch64_ad.m4
12694 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12695 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12696 %{
12697 match(Set dst (MaxI src imm));
12698 ins_cost(INSN_COST * 3);
12699 expand %{
12700 rFlagsReg cr;
12701 compI_reg_imm0(cr, src);
12702 cmovI_reg_imm0_gt(dst, src, cr);
12703 %}
12704 %}
12705
12706 // This pattern is automatically generated from aarch64_ad.m4
12707 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12708 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12709 %{
12710 match(Set dst (MaxI imm src));
12711 ins_cost(INSN_COST * 3);
12712 expand %{
12713 rFlagsReg cr;
12714 compI_reg_imm0(cr, src);
12715 cmovI_reg_imm0_gt(dst, src, cr);
12716 %}
12717 %}
12718
12719 // This pattern is automatically generated from aarch64_ad.m4
12720 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12721 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12722 %{
12723 match(Set dst (MaxI src imm));
12724 ins_cost(INSN_COST * 3);
12725 expand %{
12726 rFlagsReg cr;
12727 compI_reg_imm0(cr, src);
12728 cmovI_reg_imm1_gt(dst, src, cr);
12729 %}
12730 %}
12731
12732 // This pattern is automatically generated from aarch64_ad.m4
12733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12734 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12735 %{
12736 match(Set dst (MaxI imm src));
12737 ins_cost(INSN_COST * 3);
12738 expand %{
12739 rFlagsReg cr;
12740 compI_reg_imm0(cr, src);
12741 cmovI_reg_imm1_gt(dst, src, cr);
12742 %}
12743 %}
12744
12745 // This pattern is automatically generated from aarch64_ad.m4
12746 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12747 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12748 %{
12749 match(Set dst (MaxI src imm));
12750 ins_cost(INSN_COST * 3);
12751 expand %{
12752 rFlagsReg cr;
12753 compI_reg_imm0(cr, src);
12754 cmovI_reg_immM1_ge(dst, src, cr);
12755 %}
12756 %}
12757
12758 // This pattern is automatically generated from aarch64_ad.m4
12759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12760 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12761 %{
12762 match(Set dst (MaxI imm src));
12763 ins_cost(INSN_COST * 3);
12764 expand %{
12765 rFlagsReg cr;
12766 compI_reg_imm0(cr, src);
12767 cmovI_reg_immM1_ge(dst, src, cr);
12768 %}
12769 %}
12770
12771 // This pattern is automatically generated from aarch64_ad.m4
12772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12773 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12774 %{
12775 match(Set dst (ReverseI src));
12776 ins_cost(INSN_COST);
12777 format %{ "rbitw $dst, $src" %}
12778 ins_encode %{
12779 __ rbitw($dst$$Register, $src$$Register);
12780 %}
12781 ins_pipe(ialu_reg);
12782 %}
12783
12784 // This pattern is automatically generated from aarch64_ad.m4
12785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12786 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12787 %{
12788 match(Set dst (ReverseL src));
12789 ins_cost(INSN_COST);
12790 format %{ "rbit $dst, $src" %}
12791 ins_encode %{
12792 __ rbit($dst$$Register, $src$$Register);
12793 %}
12794 ins_pipe(ialu_reg);
12795 %}
12796
12797
12798 // END This section of the file is automatically generated. Do not edit --------------
12799
12800
12801 // ============================================================================
12802 // Floating Point Arithmetic Instructions
12803
12804 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12805 match(Set dst (AddHF src1 src2));
12806 format %{ "faddh $dst, $src1, $src2" %}
12807 ins_encode %{
12808 __ faddh($dst$$FloatRegister,
12809 $src1$$FloatRegister,
12810 $src2$$FloatRegister);
12811 %}
12812 ins_pipe(fp_dop_reg_reg_s);
12813 %}
12814
12815 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12816 match(Set dst (AddF src1 src2));
12817
12818 ins_cost(INSN_COST * 5);
12819 format %{ "fadds $dst, $src1, $src2" %}
12820
12821 ins_encode %{
12822 __ fadds(as_FloatRegister($dst$$reg),
12823 as_FloatRegister($src1$$reg),
12824 as_FloatRegister($src2$$reg));
12825 %}
12826
12827 ins_pipe(fp_dop_reg_reg_s);
12828 %}
12829
12830 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12831 match(Set dst (AddD src1 src2));
12832
12833 ins_cost(INSN_COST * 5);
12834 format %{ "faddd $dst, $src1, $src2" %}
12835
12836 ins_encode %{
12837 __ faddd(as_FloatRegister($dst$$reg),
12838 as_FloatRegister($src1$$reg),
12839 as_FloatRegister($src2$$reg));
12840 %}
12841
12842 ins_pipe(fp_dop_reg_reg_d);
12843 %}
12844
12845 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12846 match(Set dst (SubHF src1 src2));
12847 format %{ "fsubh $dst, $src1, $src2" %}
12848 ins_encode %{
12849 __ fsubh($dst$$FloatRegister,
12850 $src1$$FloatRegister,
12851 $src2$$FloatRegister);
12852 %}
12853 ins_pipe(fp_dop_reg_reg_s);
12854 %}
12855
12856 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12857 match(Set dst (SubF src1 src2));
12858
12859 ins_cost(INSN_COST * 5);
12860 format %{ "fsubs $dst, $src1, $src2" %}
12861
12862 ins_encode %{
12863 __ fsubs(as_FloatRegister($dst$$reg),
12864 as_FloatRegister($src1$$reg),
12865 as_FloatRegister($src2$$reg));
12866 %}
12867
12868 ins_pipe(fp_dop_reg_reg_s);
12869 %}
12870
12871 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12872 match(Set dst (SubD src1 src2));
12873
12874 ins_cost(INSN_COST * 5);
12875 format %{ "fsubd $dst, $src1, $src2" %}
12876
12877 ins_encode %{
12878 __ fsubd(as_FloatRegister($dst$$reg),
12879 as_FloatRegister($src1$$reg),
12880 as_FloatRegister($src2$$reg));
12881 %}
12882
12883 ins_pipe(fp_dop_reg_reg_d);
12884 %}
12885
12886 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12887 match(Set dst (MulHF src1 src2));
12888 format %{ "fmulh $dst, $src1, $src2" %}
12889 ins_encode %{
12890 __ fmulh($dst$$FloatRegister,
12891 $src1$$FloatRegister,
12892 $src2$$FloatRegister);
12893 %}
12894 ins_pipe(fp_dop_reg_reg_s);
12895 %}
12896
12897 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12898 match(Set dst (MulF src1 src2));
12899
12900 ins_cost(INSN_COST * 6);
12901 format %{ "fmuls $dst, $src1, $src2" %}
12902
12903 ins_encode %{
12904 __ fmuls(as_FloatRegister($dst$$reg),
12905 as_FloatRegister($src1$$reg),
12906 as_FloatRegister($src2$$reg));
12907 %}
12908
12909 ins_pipe(fp_dop_reg_reg_s);
12910 %}
12911
12912 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12913 match(Set dst (MulD src1 src2));
12914
12915 ins_cost(INSN_COST * 6);
12916 format %{ "fmuld $dst, $src1, $src2" %}
12917
12918 ins_encode %{
12919 __ fmuld(as_FloatRegister($dst$$reg),
12920 as_FloatRegister($src1$$reg),
12921 as_FloatRegister($src2$$reg));
12922 %}
12923
12924 ins_pipe(fp_dop_reg_reg_d);
12925 %}
12926
12927 // src1 * src2 + src3 (half-precision float)
12928 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12929 match(Set dst (FmaHF src3 (Binary src1 src2)));
12930 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
12931 ins_encode %{
12932 assert(UseFMA, "Needs FMA instructions support.");
12933 __ fmaddh($dst$$FloatRegister,
12934 $src1$$FloatRegister,
12935 $src2$$FloatRegister,
12936 $src3$$FloatRegister);
12937 %}
12938 ins_pipe(pipe_class_default);
12939 %}
12940
12941 // src1 * src2 + src3
12942 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12943 match(Set dst (FmaF src3 (Binary src1 src2)));
12944
12945 format %{ "fmadds $dst, $src1, $src2, $src3" %}
12946
12947 ins_encode %{
12948 assert(UseFMA, "Needs FMA instructions support.");
12949 __ fmadds(as_FloatRegister($dst$$reg),
12950 as_FloatRegister($src1$$reg),
12951 as_FloatRegister($src2$$reg),
12952 as_FloatRegister($src3$$reg));
12953 %}
12954
12955 ins_pipe(pipe_class_default);
12956 %}
12957
12958 // src1 * src2 + src3
12959 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12960 match(Set dst (FmaD src3 (Binary src1 src2)));
12961
12962 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
12963
12964 ins_encode %{
12965 assert(UseFMA, "Needs FMA instructions support.");
12966 __ fmaddd(as_FloatRegister($dst$$reg),
12967 as_FloatRegister($src1$$reg),
12968 as_FloatRegister($src2$$reg),
12969 as_FloatRegister($src3$$reg));
12970 %}
12971
12972 ins_pipe(pipe_class_default);
12973 %}
12974
12975 // src1 * (-src2) + src3
12976 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12977 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
12978 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
12979
12980 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
12981
12982 ins_encode %{
12983 assert(UseFMA, "Needs FMA instructions support.");
12984 __ fmsubs(as_FloatRegister($dst$$reg),
12985 as_FloatRegister($src1$$reg),
12986 as_FloatRegister($src2$$reg),
12987 as_FloatRegister($src3$$reg));
12988 %}
12989
12990 ins_pipe(pipe_class_default);
12991 %}
12992
12993 // src1 * (-src2) + src3
12994 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
12995 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
12996 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
12997
12998 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
12999
13000 ins_encode %{
13001 assert(UseFMA, "Needs FMA instructions support.");
13002 __ fmsubd(as_FloatRegister($dst$$reg),
13003 as_FloatRegister($src1$$reg),
13004 as_FloatRegister($src2$$reg),
13005 as_FloatRegister($src3$$reg));
13006 %}
13007
13008 ins_pipe(pipe_class_default);
13009 %}
13010
13011 // src1 * (-src2) - src3
13012 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13013 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13014 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13015
13016 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13017
13018 ins_encode %{
13019 assert(UseFMA, "Needs FMA instructions support.");
13020 __ fnmadds(as_FloatRegister($dst$$reg),
13021 as_FloatRegister($src1$$reg),
13022 as_FloatRegister($src2$$reg),
13023 as_FloatRegister($src3$$reg));
13024 %}
13025
13026 ins_pipe(pipe_class_default);
13027 %}
13028
13029 // src1 * (-src2) - src3
13030 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13031 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13032 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13033
13034 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13035
13036 ins_encode %{
13037 assert(UseFMA, "Needs FMA instructions support.");
13038 __ fnmaddd(as_FloatRegister($dst$$reg),
13039 as_FloatRegister($src1$$reg),
13040 as_FloatRegister($src2$$reg),
13041 as_FloatRegister($src3$$reg));
13042 %}
13043
13044 ins_pipe(pipe_class_default);
13045 %}
13046
13047 // src1 * src2 - src3
13048 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13049 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13050
13051 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13052
13053 ins_encode %{
13054 assert(UseFMA, "Needs FMA instructions support.");
13055 __ fnmsubs(as_FloatRegister($dst$$reg),
13056 as_FloatRegister($src1$$reg),
13057 as_FloatRegister($src2$$reg),
13058 as_FloatRegister($src3$$reg));
13059 %}
13060
13061 ins_pipe(pipe_class_default);
13062 %}
13063
13064 // src1 * src2 - src3
13065 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13066 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13067
13068 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13069
13070 ins_encode %{
13071 assert(UseFMA, "Needs FMA instructions support.");
13072 // n.b. insn name should be fnmsubd
13073 __ fnmsub(as_FloatRegister($dst$$reg),
13074 as_FloatRegister($src1$$reg),
13075 as_FloatRegister($src2$$reg),
13076 as_FloatRegister($src3$$reg));
13077 %}
13078
13079 ins_pipe(pipe_class_default);
13080 %}
13081
13082 // Math.max(HH)H (half-precision float)
13083 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13084 match(Set dst (MaxHF src1 src2));
13085 format %{ "fmaxh $dst, $src1, $src2" %}
13086 ins_encode %{
13087 __ fmaxh($dst$$FloatRegister,
13088 $src1$$FloatRegister,
13089 $src2$$FloatRegister);
13090 %}
13091 ins_pipe(fp_dop_reg_reg_s);
13092 %}
13093
13094 // Math.min(HH)H (half-precision float)
13095 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13096 match(Set dst (MinHF src1 src2));
13097 format %{ "fminh $dst, $src1, $src2" %}
13098 ins_encode %{
13099 __ fminh($dst$$FloatRegister,
13100 $src1$$FloatRegister,
13101 $src2$$FloatRegister);
13102 %}
13103 ins_pipe(fp_dop_reg_reg_s);
13104 %}
13105
13106 // Math.max(FF)F
13107 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13108 match(Set dst (MaxF src1 src2));
13109
13110 format %{ "fmaxs $dst, $src1, $src2" %}
13111 ins_encode %{
13112 __ fmaxs(as_FloatRegister($dst$$reg),
13113 as_FloatRegister($src1$$reg),
13114 as_FloatRegister($src2$$reg));
13115 %}
13116
13117 ins_pipe(fp_dop_reg_reg_s);
13118 %}
13119
13120 // Math.min(FF)F
13121 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13122 match(Set dst (MinF src1 src2));
13123
13124 format %{ "fmins $dst, $src1, $src2" %}
13125 ins_encode %{
13126 __ fmins(as_FloatRegister($dst$$reg),
13127 as_FloatRegister($src1$$reg),
13128 as_FloatRegister($src2$$reg));
13129 %}
13130
13131 ins_pipe(fp_dop_reg_reg_s);
13132 %}
13133
13134 // Math.max(DD)D
13135 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13136 match(Set dst (MaxD src1 src2));
13137
13138 format %{ "fmaxd $dst, $src1, $src2" %}
13139 ins_encode %{
13140 __ fmaxd(as_FloatRegister($dst$$reg),
13141 as_FloatRegister($src1$$reg),
13142 as_FloatRegister($src2$$reg));
13143 %}
13144
13145 ins_pipe(fp_dop_reg_reg_d);
13146 %}
13147
13148 // Math.min(DD)D
13149 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13150 match(Set dst (MinD src1 src2));
13151
13152 format %{ "fmind $dst, $src1, $src2" %}
13153 ins_encode %{
13154 __ fmind(as_FloatRegister($dst$$reg),
13155 as_FloatRegister($src1$$reg),
13156 as_FloatRegister($src2$$reg));
13157 %}
13158
13159 ins_pipe(fp_dop_reg_reg_d);
13160 %}
13161
13162 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13163 match(Set dst (DivHF src1 src2));
13164 format %{ "fdivh $dst, $src1, $src2" %}
13165 ins_encode %{
13166 __ fdivh($dst$$FloatRegister,
13167 $src1$$FloatRegister,
13168 $src2$$FloatRegister);
13169 %}
13170 ins_pipe(fp_div_s);
13171 %}
13172
13173 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13174 match(Set dst (DivF src1 src2));
13175
13176 ins_cost(INSN_COST * 18);
13177 format %{ "fdivs $dst, $src1, $src2" %}
13178
13179 ins_encode %{
13180 __ fdivs(as_FloatRegister($dst$$reg),
13181 as_FloatRegister($src1$$reg),
13182 as_FloatRegister($src2$$reg));
13183 %}
13184
13185 ins_pipe(fp_div_s);
13186 %}
13187
13188 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13189 match(Set dst (DivD src1 src2));
13190
13191 ins_cost(INSN_COST * 32);
13192 format %{ "fdivd $dst, $src1, $src2" %}
13193
13194 ins_encode %{
13195 __ fdivd(as_FloatRegister($dst$$reg),
13196 as_FloatRegister($src1$$reg),
13197 as_FloatRegister($src2$$reg));
13198 %}
13199
13200 ins_pipe(fp_div_d);
13201 %}
13202
13203 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13204 match(Set dst (NegF src));
13205
13206 ins_cost(INSN_COST * 3);
13207 format %{ "fneg $dst, $src" %}
13208
13209 ins_encode %{
13210 __ fnegs(as_FloatRegister($dst$$reg),
13211 as_FloatRegister($src$$reg));
13212 %}
13213
13214 ins_pipe(fp_uop_s);
13215 %}
13216
13217 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13218 match(Set dst (NegD src));
13219
13220 ins_cost(INSN_COST * 3);
13221 format %{ "fnegd $dst, $src" %}
13222
13223 ins_encode %{
13224 __ fnegd(as_FloatRegister($dst$$reg),
13225 as_FloatRegister($src$$reg));
13226 %}
13227
13228 ins_pipe(fp_uop_d);
13229 %}
13230
13231 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13232 %{
13233 match(Set dst (AbsI src));
13234
13235 effect(KILL cr);
13236 ins_cost(INSN_COST * 2);
13237 format %{ "cmpw $src, zr\n\t"
13238 "cnegw $dst, $src, Assembler::LT\t# int abs"
13239 %}
13240
13241 ins_encode %{
13242 __ cmpw(as_Register($src$$reg), zr);
13243 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13244 %}
13245 ins_pipe(pipe_class_default);
13246 %}
13247
13248 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13249 %{
13250 match(Set dst (AbsL src));
13251
13252 effect(KILL cr);
13253 ins_cost(INSN_COST * 2);
13254 format %{ "cmp $src, zr\n\t"
13255 "cneg $dst, $src, Assembler::LT\t# long abs"
13256 %}
13257
13258 ins_encode %{
13259 __ cmp(as_Register($src$$reg), zr);
13260 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13261 %}
13262 ins_pipe(pipe_class_default);
13263 %}
13264
13265 instruct absF_reg(vRegF dst, vRegF src) %{
13266 match(Set dst (AbsF src));
13267
13268 ins_cost(INSN_COST * 3);
13269 format %{ "fabss $dst, $src" %}
13270 ins_encode %{
13271 __ fabss(as_FloatRegister($dst$$reg),
13272 as_FloatRegister($src$$reg));
13273 %}
13274
13275 ins_pipe(fp_uop_s);
13276 %}
13277
13278 instruct absD_reg(vRegD dst, vRegD src) %{
13279 match(Set dst (AbsD src));
13280
13281 ins_cost(INSN_COST * 3);
13282 format %{ "fabsd $dst, $src" %}
13283 ins_encode %{
13284 __ fabsd(as_FloatRegister($dst$$reg),
13285 as_FloatRegister($src$$reg));
13286 %}
13287
13288 ins_pipe(fp_uop_d);
13289 %}
13290
13291 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13292 match(Set dst (AbsF (SubF src1 src2)));
13293
13294 ins_cost(INSN_COST * 3);
13295 format %{ "fabds $dst, $src1, $src2" %}
13296 ins_encode %{
13297 __ fabds(as_FloatRegister($dst$$reg),
13298 as_FloatRegister($src1$$reg),
13299 as_FloatRegister($src2$$reg));
13300 %}
13301
13302 ins_pipe(fp_uop_s);
13303 %}
13304
13305 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13306 match(Set dst (AbsD (SubD src1 src2)));
13307
13308 ins_cost(INSN_COST * 3);
13309 format %{ "fabdd $dst, $src1, $src2" %}
13310 ins_encode %{
13311 __ fabdd(as_FloatRegister($dst$$reg),
13312 as_FloatRegister($src1$$reg),
13313 as_FloatRegister($src2$$reg));
13314 %}
13315
13316 ins_pipe(fp_uop_d);
13317 %}
13318
13319 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13320 match(Set dst (SqrtD src));
13321
13322 ins_cost(INSN_COST * 50);
13323 format %{ "fsqrtd $dst, $src" %}
13324 ins_encode %{
13325 __ fsqrtd(as_FloatRegister($dst$$reg),
13326 as_FloatRegister($src$$reg));
13327 %}
13328
13329 ins_pipe(fp_div_s);
13330 %}
13331
13332 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13333 match(Set dst (SqrtF src));
13334
13335 ins_cost(INSN_COST * 50);
13336 format %{ "fsqrts $dst, $src" %}
13337 ins_encode %{
13338 __ fsqrts(as_FloatRegister($dst$$reg),
13339 as_FloatRegister($src$$reg));
13340 %}
13341
13342 ins_pipe(fp_div_d);
13343 %}
13344
13345 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13346 match(Set dst (SqrtHF src));
13347 format %{ "fsqrth $dst, $src" %}
13348 ins_encode %{
13349 __ fsqrth($dst$$FloatRegister,
13350 $src$$FloatRegister);
13351 %}
13352 ins_pipe(fp_div_s);
13353 %}
13354
13355 // Math.rint, floor, ceil
13356 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13357 match(Set dst (RoundDoubleMode src rmode));
13358 format %{ "frint $dst, $src, $rmode" %}
13359 ins_encode %{
13360 switch ($rmode$$constant) {
13361 case RoundDoubleModeNode::rmode_rint:
13362 __ frintnd(as_FloatRegister($dst$$reg),
13363 as_FloatRegister($src$$reg));
13364 break;
13365 case RoundDoubleModeNode::rmode_floor:
13366 __ frintmd(as_FloatRegister($dst$$reg),
13367 as_FloatRegister($src$$reg));
13368 break;
13369 case RoundDoubleModeNode::rmode_ceil:
13370 __ frintpd(as_FloatRegister($dst$$reg),
13371 as_FloatRegister($src$$reg));
13372 break;
13373 }
13374 %}
13375 ins_pipe(fp_uop_d);
13376 %}
13377
13378 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13379 match(Set dst (CopySignD src1 (Binary src2 zero)));
13380 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13381 format %{ "CopySignD $dst $src1 $src2" %}
13382 ins_encode %{
13383 FloatRegister dst = as_FloatRegister($dst$$reg),
13384 src1 = as_FloatRegister($src1$$reg),
13385 src2 = as_FloatRegister($src2$$reg),
13386 zero = as_FloatRegister($zero$$reg);
13387 __ fnegd(dst, zero);
13388 __ bsl(dst, __ T8B, src2, src1);
13389 %}
13390 ins_pipe(fp_uop_d);
13391 %}
13392
13393 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13394 match(Set dst (CopySignF src1 src2));
13395 effect(TEMP_DEF dst, USE src1, USE src2);
13396 format %{ "CopySignF $dst $src1 $src2" %}
13397 ins_encode %{
13398 FloatRegister dst = as_FloatRegister($dst$$reg),
13399 src1 = as_FloatRegister($src1$$reg),
13400 src2 = as_FloatRegister($src2$$reg);
13401 __ movi(dst, __ T2S, 0x80, 24);
13402 __ bsl(dst, __ T8B, src2, src1);
13403 %}
13404 ins_pipe(fp_uop_d);
13405 %}
13406
13407 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13408 match(Set dst (SignumD src (Binary zero one)));
13409 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13410 format %{ "signumD $dst, $src" %}
13411 ins_encode %{
13412 FloatRegister src = as_FloatRegister($src$$reg),
13413 dst = as_FloatRegister($dst$$reg),
13414 zero = as_FloatRegister($zero$$reg),
13415 one = as_FloatRegister($one$$reg);
13416 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13417 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13418 // Bit selection instruction gets bit from "one" for each enabled bit in
13419 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13420 // NaN the whole "src" will be copied because "dst" is zero. For all other
13421 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13422 // from "src", and all other bits are copied from 1.0.
13423 __ bsl(dst, __ T8B, one, src);
13424 %}
13425 ins_pipe(fp_uop_d);
13426 %}
13427
13428 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13429 match(Set dst (SignumF src (Binary zero one)));
13430 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13431 format %{ "signumF $dst, $src" %}
13432 ins_encode %{
13433 FloatRegister src = as_FloatRegister($src$$reg),
13434 dst = as_FloatRegister($dst$$reg),
13435 zero = as_FloatRegister($zero$$reg),
13436 one = as_FloatRegister($one$$reg);
13437 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13438 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13439 // Bit selection instruction gets bit from "one" for each enabled bit in
13440 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13441 // NaN the whole "src" will be copied because "dst" is zero. For all other
13442 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13443 // from "src", and all other bits are copied from 1.0.
13444 __ bsl(dst, __ T8B, one, src);
13445 %}
13446 ins_pipe(fp_uop_d);
13447 %}
13448
13449 instruct onspinwait() %{
13450 match(OnSpinWait);
13451 ins_cost(INSN_COST);
13452
13453 format %{ "onspinwait" %}
13454
13455 ins_encode %{
13456 __ spin_wait();
13457 %}
13458 ins_pipe(pipe_class_empty);
13459 %}
13460
13461 // ============================================================================
13462 // Logical Instructions
13463
13464 // Integer Logical Instructions
13465
13466 // And Instructions
13467
13468
13469 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13470 match(Set dst (AndI src1 src2));
13471
13472 format %{ "andw $dst, $src1, $src2\t# int" %}
13473
13474 ins_cost(INSN_COST);
13475 ins_encode %{
13476 __ andw(as_Register($dst$$reg),
13477 as_Register($src1$$reg),
13478 as_Register($src2$$reg));
13479 %}
13480
13481 ins_pipe(ialu_reg_reg);
13482 %}
13483
13484 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13485 match(Set dst (AndI src1 src2));
13486
13487 format %{ "andsw $dst, $src1, $src2\t# int" %}
13488
13489 ins_cost(INSN_COST);
13490 ins_encode %{
13491 __ andw(as_Register($dst$$reg),
13492 as_Register($src1$$reg),
13493 (uint64_t)($src2$$constant));
13494 %}
13495
13496 ins_pipe(ialu_reg_imm);
13497 %}
13498
13499 // Or Instructions
13500
13501 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13502 match(Set dst (OrI src1 src2));
13503
13504 format %{ "orrw $dst, $src1, $src2\t# int" %}
13505
13506 ins_cost(INSN_COST);
13507 ins_encode %{
13508 __ orrw(as_Register($dst$$reg),
13509 as_Register($src1$$reg),
13510 as_Register($src2$$reg));
13511 %}
13512
13513 ins_pipe(ialu_reg_reg);
13514 %}
13515
13516 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13517 match(Set dst (OrI src1 src2));
13518
13519 format %{ "orrw $dst, $src1, $src2\t# int" %}
13520
13521 ins_cost(INSN_COST);
13522 ins_encode %{
13523 __ orrw(as_Register($dst$$reg),
13524 as_Register($src1$$reg),
13525 (uint64_t)($src2$$constant));
13526 %}
13527
13528 ins_pipe(ialu_reg_imm);
13529 %}
13530
13531 // Xor Instructions
13532
13533 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13534 match(Set dst (XorI src1 src2));
13535
13536 format %{ "eorw $dst, $src1, $src2\t# int" %}
13537
13538 ins_cost(INSN_COST);
13539 ins_encode %{
13540 __ eorw(as_Register($dst$$reg),
13541 as_Register($src1$$reg),
13542 as_Register($src2$$reg));
13543 %}
13544
13545 ins_pipe(ialu_reg_reg);
13546 %}
13547
13548 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13549 match(Set dst (XorI src1 src2));
13550
13551 format %{ "eorw $dst, $src1, $src2\t# int" %}
13552
13553 ins_cost(INSN_COST);
13554 ins_encode %{
13555 __ eorw(as_Register($dst$$reg),
13556 as_Register($src1$$reg),
13557 (uint64_t)($src2$$constant));
13558 %}
13559
13560 ins_pipe(ialu_reg_imm);
13561 %}
13562
13563 // Long Logical Instructions
13564 // TODO
13565
13566 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13567 match(Set dst (AndL src1 src2));
13568
13569 format %{ "and $dst, $src1, $src2\t# int" %}
13570
13571 ins_cost(INSN_COST);
13572 ins_encode %{
13573 __ andr(as_Register($dst$$reg),
13574 as_Register($src1$$reg),
13575 as_Register($src2$$reg));
13576 %}
13577
13578 ins_pipe(ialu_reg_reg);
13579 %}
13580
13581 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13582 match(Set dst (AndL src1 src2));
13583
13584 format %{ "and $dst, $src1, $src2\t# int" %}
13585
13586 ins_cost(INSN_COST);
13587 ins_encode %{
13588 __ andr(as_Register($dst$$reg),
13589 as_Register($src1$$reg),
13590 (uint64_t)($src2$$constant));
13591 %}
13592
13593 ins_pipe(ialu_reg_imm);
13594 %}
13595
13596 // Or Instructions
13597
13598 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13599 match(Set dst (OrL src1 src2));
13600
13601 format %{ "orr $dst, $src1, $src2\t# int" %}
13602
13603 ins_cost(INSN_COST);
13604 ins_encode %{
13605 __ orr(as_Register($dst$$reg),
13606 as_Register($src1$$reg),
13607 as_Register($src2$$reg));
13608 %}
13609
13610 ins_pipe(ialu_reg_reg);
13611 %}
13612
13613 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13614 match(Set dst (OrL src1 src2));
13615
13616 format %{ "orr $dst, $src1, $src2\t# int" %}
13617
13618 ins_cost(INSN_COST);
13619 ins_encode %{
13620 __ orr(as_Register($dst$$reg),
13621 as_Register($src1$$reg),
13622 (uint64_t)($src2$$constant));
13623 %}
13624
13625 ins_pipe(ialu_reg_imm);
13626 %}
13627
13628 // Xor Instructions
13629
13630 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13631 match(Set dst (XorL src1 src2));
13632
13633 format %{ "eor $dst, $src1, $src2\t# int" %}
13634
13635 ins_cost(INSN_COST);
13636 ins_encode %{
13637 __ eor(as_Register($dst$$reg),
13638 as_Register($src1$$reg),
13639 as_Register($src2$$reg));
13640 %}
13641
13642 ins_pipe(ialu_reg_reg);
13643 %}
13644
13645 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13646 match(Set dst (XorL src1 src2));
13647
13648 ins_cost(INSN_COST);
13649 format %{ "eor $dst, $src1, $src2\t# int" %}
13650
13651 ins_encode %{
13652 __ eor(as_Register($dst$$reg),
13653 as_Register($src1$$reg),
13654 (uint64_t)($src2$$constant));
13655 %}
13656
13657 ins_pipe(ialu_reg_imm);
13658 %}
13659
13660 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13661 %{
13662 match(Set dst (ConvI2L src));
13663
13664 ins_cost(INSN_COST);
13665 format %{ "sxtw $dst, $src\t# i2l" %}
13666 ins_encode %{
13667 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13668 %}
13669 ins_pipe(ialu_reg_shift);
13670 %}
13671
13672 // this pattern occurs in bigmath arithmetic
13673 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13674 %{
13675 match(Set dst (AndL (ConvI2L src) mask));
13676
13677 ins_cost(INSN_COST);
13678 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13679 ins_encode %{
13680 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13681 %}
13682
13683 ins_pipe(ialu_reg_shift);
13684 %}
13685
13686 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13687 match(Set dst (ConvL2I src));
13688
13689 ins_cost(INSN_COST);
13690 format %{ "movw $dst, $src \t// l2i" %}
13691
13692 ins_encode %{
13693 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13694 %}
13695
13696 ins_pipe(ialu_reg);
13697 %}
13698
13699 instruct convD2F_reg(vRegF dst, vRegD src) %{
13700 match(Set dst (ConvD2F src));
13701
13702 ins_cost(INSN_COST * 5);
13703 format %{ "fcvtd $dst, $src \t// d2f" %}
13704
13705 ins_encode %{
13706 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13707 %}
13708
13709 ins_pipe(fp_d2f);
13710 %}
13711
13712 instruct convF2D_reg(vRegD dst, vRegF src) %{
13713 match(Set dst (ConvF2D src));
13714
13715 ins_cost(INSN_COST * 5);
13716 format %{ "fcvts $dst, $src \t// f2d" %}
13717
13718 ins_encode %{
13719 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13720 %}
13721
13722 ins_pipe(fp_f2d);
13723 %}
13724
13725 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13726 match(Set dst (ConvF2I src));
13727
13728 ins_cost(INSN_COST * 5);
13729 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13730
13731 ins_encode %{
13732 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13733 %}
13734
13735 ins_pipe(fp_f2i);
13736 %}
13737
13738 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13739 match(Set dst (ConvF2L src));
13740
13741 ins_cost(INSN_COST * 5);
13742 format %{ "fcvtzs $dst, $src \t// f2l" %}
13743
13744 ins_encode %{
13745 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13746 %}
13747
13748 ins_pipe(fp_f2l);
13749 %}
13750
13751 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13752 match(Set dst (ConvF2HF src));
13753 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13754 "smov $dst, $tmp\t# move result from $tmp to $dst"
13755 %}
13756 effect(TEMP tmp);
13757 ins_encode %{
13758 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13759 %}
13760 ins_pipe(pipe_slow);
13761 %}
13762
13763 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13764 match(Set dst (ConvHF2F src));
13765 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13766 "fcvt $dst, $tmp\t# convert half to single precision"
13767 %}
13768 effect(TEMP tmp);
13769 ins_encode %{
13770 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13771 %}
13772 ins_pipe(pipe_slow);
13773 %}
13774
13775 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13776 match(Set dst (ConvI2F src));
13777
13778 ins_cost(INSN_COST * 5);
13779 format %{ "scvtfws $dst, $src \t// i2f" %}
13780
13781 ins_encode %{
13782 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13783 %}
13784
13785 ins_pipe(fp_i2f);
13786 %}
13787
13788 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13789 match(Set dst (ConvL2F src));
13790
13791 ins_cost(INSN_COST * 5);
13792 format %{ "scvtfs $dst, $src \t// l2f" %}
13793
13794 ins_encode %{
13795 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13796 %}
13797
13798 ins_pipe(fp_l2f);
13799 %}
13800
13801 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13802 match(Set dst (ConvD2I src));
13803
13804 ins_cost(INSN_COST * 5);
13805 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13806
13807 ins_encode %{
13808 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13809 %}
13810
13811 ins_pipe(fp_d2i);
13812 %}
13813
13814 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13815 match(Set dst (ConvD2L src));
13816
13817 ins_cost(INSN_COST * 5);
13818 format %{ "fcvtzd $dst, $src \t// d2l" %}
13819
13820 ins_encode %{
13821 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13822 %}
13823
13824 ins_pipe(fp_d2l);
13825 %}
13826
13827 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13828 match(Set dst (ConvI2D src));
13829
13830 ins_cost(INSN_COST * 5);
13831 format %{ "scvtfwd $dst, $src \t// i2d" %}
13832
13833 ins_encode %{
13834 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13835 %}
13836
13837 ins_pipe(fp_i2d);
13838 %}
13839
13840 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13841 match(Set dst (ConvL2D src));
13842
13843 ins_cost(INSN_COST * 5);
13844 format %{ "scvtfd $dst, $src \t// l2d" %}
13845
13846 ins_encode %{
13847 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13848 %}
13849
13850 ins_pipe(fp_l2d);
13851 %}
13852
13853 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13854 %{
13855 match(Set dst (RoundD src));
13856 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13857 format %{ "java_round_double $dst,$src"%}
13858 ins_encode %{
13859 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13860 as_FloatRegister($ftmp$$reg));
13861 %}
13862 ins_pipe(pipe_slow);
13863 %}
13864
13865 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13866 %{
13867 match(Set dst (RoundF src));
13868 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13869 format %{ "java_round_float $dst,$src"%}
13870 ins_encode %{
13871 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13872 as_FloatRegister($ftmp$$reg));
13873 %}
13874 ins_pipe(pipe_slow);
13875 %}
13876
13877 // stack <-> reg and reg <-> reg shuffles with no conversion
13878
13879 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13880
13881 match(Set dst (MoveF2I src));
13882
13883 effect(DEF dst, USE src);
13884
13885 ins_cost(4 * INSN_COST);
13886
13887 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13888
13889 ins_encode %{
13890 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13891 %}
13892
13893 ins_pipe(iload_reg_reg);
13894
13895 %}
13896
13897 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13898
13899 match(Set dst (MoveI2F src));
13900
13901 effect(DEF dst, USE src);
13902
13903 ins_cost(4 * INSN_COST);
13904
13905 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13906
13907 ins_encode %{
13908 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13909 %}
13910
13911 ins_pipe(pipe_class_memory);
13912
13913 %}
13914
13915 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13916
13917 match(Set dst (MoveD2L src));
13918
13919 effect(DEF dst, USE src);
13920
13921 ins_cost(4 * INSN_COST);
13922
13923 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13924
13925 ins_encode %{
13926 __ ldr($dst$$Register, Address(sp, $src$$disp));
13927 %}
13928
13929 ins_pipe(iload_reg_reg);
13930
13931 %}
13932
13933 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
13934
13935 match(Set dst (MoveL2D src));
13936
13937 effect(DEF dst, USE src);
13938
13939 ins_cost(4 * INSN_COST);
13940
13941 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
13942
13943 ins_encode %{
13944 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13945 %}
13946
13947 ins_pipe(pipe_class_memory);
13948
13949 %}
13950
13951 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
13952
13953 match(Set dst (MoveF2I src));
13954
13955 effect(DEF dst, USE src);
13956
13957 ins_cost(INSN_COST);
13958
13959 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
13960
13961 ins_encode %{
13962 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13963 %}
13964
13965 ins_pipe(pipe_class_memory);
13966
13967 %}
13968
13969 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
13970
13971 match(Set dst (MoveI2F src));
13972
13973 effect(DEF dst, USE src);
13974
13975 ins_cost(INSN_COST);
13976
13977 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
13978
13979 ins_encode %{
13980 __ strw($src$$Register, Address(sp, $dst$$disp));
13981 %}
13982
13983 ins_pipe(istore_reg_reg);
13984
13985 %}
13986
13987 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
13988
13989 match(Set dst (MoveD2L src));
13990
13991 effect(DEF dst, USE src);
13992
13993 ins_cost(INSN_COST);
13994
13995 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
13996
13997 ins_encode %{
13998 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
13999 %}
14000
14001 ins_pipe(pipe_class_memory);
14002
14003 %}
14004
14005 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14006
14007 match(Set dst (MoveL2D src));
14008
14009 effect(DEF dst, USE src);
14010
14011 ins_cost(INSN_COST);
14012
14013 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14014
14015 ins_encode %{
14016 __ str($src$$Register, Address(sp, $dst$$disp));
14017 %}
14018
14019 ins_pipe(istore_reg_reg);
14020
14021 %}
14022
14023 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14024
14025 match(Set dst (MoveF2I src));
14026
14027 effect(DEF dst, USE src);
14028
14029 ins_cost(INSN_COST);
14030
14031 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14032
14033 ins_encode %{
14034 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14035 %}
14036
14037 ins_pipe(fp_f2i);
14038
14039 %}
14040
14041 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14042
14043 match(Set dst (MoveI2F src));
14044
14045 effect(DEF dst, USE src);
14046
14047 ins_cost(INSN_COST);
14048
14049 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14050
14051 ins_encode %{
14052 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14053 %}
14054
14055 ins_pipe(fp_i2f);
14056
14057 %}
14058
14059 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14060
14061 match(Set dst (MoveD2L src));
14062
14063 effect(DEF dst, USE src);
14064
14065 ins_cost(INSN_COST);
14066
14067 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14068
14069 ins_encode %{
14070 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14071 %}
14072
14073 ins_pipe(fp_d2l);
14074
14075 %}
14076
14077 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14078
14079 match(Set dst (MoveL2D src));
14080
14081 effect(DEF dst, USE src);
14082
14083 ins_cost(INSN_COST);
14084
14085 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14086
14087 ins_encode %{
14088 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14089 %}
14090
14091 ins_pipe(fp_l2d);
14092
14093 %}
14094
14095 // ============================================================================
14096 // clearing of an array
14097
14098 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14099 %{
14100 match(Set dummy (ClearArray (Binary cnt base) zero));
14101 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14102
14103 ins_cost(4 * INSN_COST);
14104 format %{ "ClearArray $cnt, $base" %}
14105
14106 ins_encode %{
14107 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14108 if (tpc == nullptr) {
14109 ciEnv::current()->record_failure("CodeCache is full");
14110 return;
14111 }
14112 %}
14113
14114 ins_pipe(pipe_class_memory);
14115 %}
14116
14117 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14118 %{
14119 predicate(((ClearArrayNode*)n)->word_copy_only());
14120 match(Set dummy (ClearArray (Binary cnt base) val));
14121 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14122
14123 ins_cost(4 * INSN_COST);
14124 format %{ "ClearArray $cnt, $base, $val" %}
14125
14126 ins_encode %{
14127 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14128 %}
14129
14130 ins_pipe(pipe_class_memory);
14131 %}
14132
14133 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14134 %{
14135 predicate((uint64_t)n->in(2)->get_long()
14136 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14137 && !((ClearArrayNode*)n)->word_copy_only());
14138 match(Set dummy (ClearArray cnt base));
14139 effect(TEMP temp, USE_KILL base, KILL cr);
14140
14141 ins_cost(4 * INSN_COST);
14142 format %{ "ClearArray $cnt, $base" %}
14143
14144 ins_encode %{
14145 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14146 if (tpc == nullptr) {
14147 ciEnv::current()->record_failure("CodeCache is full");
14148 return;
14149 }
14150 %}
14151
14152 ins_pipe(pipe_class_memory);
14153 %}
14154
14155 // ============================================================================
14156 // Overflow Math Instructions
14157
14158 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14159 %{
14160 match(Set cr (OverflowAddI op1 op2));
14161
14162 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14163 ins_cost(INSN_COST);
14164 ins_encode %{
14165 __ cmnw($op1$$Register, $op2$$Register);
14166 %}
14167
14168 ins_pipe(icmp_reg_reg);
14169 %}
14170
14171 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14172 %{
14173 match(Set cr (OverflowAddI op1 op2));
14174
14175 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14176 ins_cost(INSN_COST);
14177 ins_encode %{
14178 __ cmnw($op1$$Register, $op2$$constant);
14179 %}
14180
14181 ins_pipe(icmp_reg_imm);
14182 %}
14183
14184 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14185 %{
14186 match(Set cr (OverflowAddL op1 op2));
14187
14188 format %{ "cmn $op1, $op2\t# overflow check long" %}
14189 ins_cost(INSN_COST);
14190 ins_encode %{
14191 __ cmn($op1$$Register, $op2$$Register);
14192 %}
14193
14194 ins_pipe(icmp_reg_reg);
14195 %}
14196
14197 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14198 %{
14199 match(Set cr (OverflowAddL op1 op2));
14200
14201 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14202 ins_cost(INSN_COST);
14203 ins_encode %{
14204 __ adds(zr, $op1$$Register, $op2$$constant);
14205 %}
14206
14207 ins_pipe(icmp_reg_imm);
14208 %}
14209
14210 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14211 %{
14212 match(Set cr (OverflowSubI op1 op2));
14213
14214 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14215 ins_cost(INSN_COST);
14216 ins_encode %{
14217 __ cmpw($op1$$Register, $op2$$Register);
14218 %}
14219
14220 ins_pipe(icmp_reg_reg);
14221 %}
14222
14223 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14224 %{
14225 match(Set cr (OverflowSubI op1 op2));
14226
14227 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14228 ins_cost(INSN_COST);
14229 ins_encode %{
14230 __ cmpw($op1$$Register, $op2$$constant);
14231 %}
14232
14233 ins_pipe(icmp_reg_imm);
14234 %}
14235
14236 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14237 %{
14238 match(Set cr (OverflowSubL op1 op2));
14239
14240 format %{ "cmp $op1, $op2\t# overflow check long" %}
14241 ins_cost(INSN_COST);
14242 ins_encode %{
14243 __ cmp($op1$$Register, $op2$$Register);
14244 %}
14245
14246 ins_pipe(icmp_reg_reg);
14247 %}
14248
14249 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14250 %{
14251 match(Set cr (OverflowSubL op1 op2));
14252
14253 format %{ "cmp $op1, $op2\t# overflow check long" %}
14254 ins_cost(INSN_COST);
14255 ins_encode %{
14256 __ subs(zr, $op1$$Register, $op2$$constant);
14257 %}
14258
14259 ins_pipe(icmp_reg_imm);
14260 %}
14261
14262 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14263 %{
14264 match(Set cr (OverflowSubI zero op1));
14265
14266 format %{ "cmpw zr, $op1\t# overflow check int" %}
14267 ins_cost(INSN_COST);
14268 ins_encode %{
14269 __ cmpw(zr, $op1$$Register);
14270 %}
14271
14272 ins_pipe(icmp_reg_imm);
14273 %}
14274
14275 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14276 %{
14277 match(Set cr (OverflowSubL zero op1));
14278
14279 format %{ "cmp zr, $op1\t# overflow check long" %}
14280 ins_cost(INSN_COST);
14281 ins_encode %{
14282 __ cmp(zr, $op1$$Register);
14283 %}
14284
14285 ins_pipe(icmp_reg_imm);
14286 %}
14287
14288 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14289 %{
14290 match(Set cr (OverflowMulI op1 op2));
14291
14292 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14293 "cmp rscratch1, rscratch1, sxtw\n\t"
14294 "movw rscratch1, #0x80000000\n\t"
14295 "cselw rscratch1, rscratch1, zr, NE\n\t"
14296 "cmpw rscratch1, #1" %}
14297 ins_cost(5 * INSN_COST);
14298 ins_encode %{
14299 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14300 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14301 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14302 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14303 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14304 %}
14305
14306 ins_pipe(pipe_slow);
14307 %}
14308
14309 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14310 %{
14311 match(If cmp (OverflowMulI op1 op2));
14312 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14313 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14314 effect(USE labl, KILL cr);
14315
14316 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14317 "cmp rscratch1, rscratch1, sxtw\n\t"
14318 "b$cmp $labl" %}
14319 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14320 ins_encode %{
14321 Label* L = $labl$$label;
14322 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14323 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14324 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14325 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14326 %}
14327
14328 ins_pipe(pipe_serial);
14329 %}
14330
14331 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14332 %{
14333 match(Set cr (OverflowMulL op1 op2));
14334
14335 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14336 "smulh rscratch2, $op1, $op2\n\t"
14337 "cmp rscratch2, rscratch1, ASR #63\n\t"
14338 "movw rscratch1, #0x80000000\n\t"
14339 "cselw rscratch1, rscratch1, zr, NE\n\t"
14340 "cmpw rscratch1, #1" %}
14341 ins_cost(6 * INSN_COST);
14342 ins_encode %{
14343 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14344 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14345 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14346 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14347 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14348 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14349 %}
14350
14351 ins_pipe(pipe_slow);
14352 %}
14353
14354 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14355 %{
14356 match(If cmp (OverflowMulL op1 op2));
14357 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14358 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14359 effect(USE labl, KILL cr);
14360
14361 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14362 "smulh rscratch2, $op1, $op2\n\t"
14363 "cmp rscratch2, rscratch1, ASR #63\n\t"
14364 "b$cmp $labl" %}
14365 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14366 ins_encode %{
14367 Label* L = $labl$$label;
14368 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14369 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14370 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14371 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14372 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14373 %}
14374
14375 ins_pipe(pipe_serial);
14376 %}
14377
14378 // ============================================================================
14379 // Compare Instructions
14380
14381 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14382 %{
14383 match(Set cr (CmpI op1 op2));
14384
14385 effect(DEF cr, USE op1, USE op2);
14386
14387 ins_cost(INSN_COST);
14388 format %{ "cmpw $op1, $op2" %}
14389
14390 ins_encode(aarch64_enc_cmpw(op1, op2));
14391
14392 ins_pipe(icmp_reg_reg);
14393 %}
14394
14395 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14396 %{
14397 match(Set cr (CmpI op1 zero));
14398
14399 effect(DEF cr, USE op1);
14400
14401 ins_cost(INSN_COST);
14402 format %{ "cmpw $op1, 0" %}
14403
14404 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14405
14406 ins_pipe(icmp_reg_imm);
14407 %}
14408
14409 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14410 %{
14411 match(Set cr (CmpI op1 op2));
14412
14413 effect(DEF cr, USE op1);
14414
14415 ins_cost(INSN_COST);
14416 format %{ "cmpw $op1, $op2" %}
14417
14418 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14419
14420 ins_pipe(icmp_reg_imm);
14421 %}
14422
14423 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14424 %{
14425 match(Set cr (CmpI op1 op2));
14426
14427 effect(DEF cr, USE op1);
14428
14429 ins_cost(INSN_COST * 2);
14430 format %{ "cmpw $op1, $op2" %}
14431
14432 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14433
14434 ins_pipe(icmp_reg_imm);
14435 %}
14436
14437 // Unsigned compare Instructions; really, same as signed compare
14438 // except it should only be used to feed an If or a CMovI which takes a
14439 // cmpOpU.
14440
14441 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14442 %{
14443 match(Set cr (CmpU op1 op2));
14444
14445 effect(DEF cr, USE op1, USE op2);
14446
14447 ins_cost(INSN_COST);
14448 format %{ "cmpw $op1, $op2\t# unsigned" %}
14449
14450 ins_encode(aarch64_enc_cmpw(op1, op2));
14451
14452 ins_pipe(icmp_reg_reg);
14453 %}
14454
14455 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14456 %{
14457 match(Set cr (CmpU op1 zero));
14458
14459 effect(DEF cr, USE op1);
14460
14461 ins_cost(INSN_COST);
14462 format %{ "cmpw $op1, #0\t# unsigned" %}
14463
14464 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14465
14466 ins_pipe(icmp_reg_imm);
14467 %}
14468
14469 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14470 %{
14471 match(Set cr (CmpU op1 op2));
14472
14473 effect(DEF cr, USE op1);
14474
14475 ins_cost(INSN_COST);
14476 format %{ "cmpw $op1, $op2\t# unsigned" %}
14477
14478 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14479
14480 ins_pipe(icmp_reg_imm);
14481 %}
14482
14483 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14484 %{
14485 match(Set cr (CmpU op1 op2));
14486
14487 effect(DEF cr, USE op1);
14488
14489 ins_cost(INSN_COST * 2);
14490 format %{ "cmpw $op1, $op2\t# unsigned" %}
14491
14492 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14493
14494 ins_pipe(icmp_reg_imm);
14495 %}
14496
14497 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14498 %{
14499 match(Set cr (CmpL op1 op2));
14500
14501 effect(DEF cr, USE op1, USE op2);
14502
14503 ins_cost(INSN_COST);
14504 format %{ "cmp $op1, $op2" %}
14505
14506 ins_encode(aarch64_enc_cmp(op1, op2));
14507
14508 ins_pipe(icmp_reg_reg);
14509 %}
14510
14511 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14512 %{
14513 match(Set cr (CmpL op1 zero));
14514
14515 effect(DEF cr, USE op1);
14516
14517 ins_cost(INSN_COST);
14518 format %{ "tst $op1" %}
14519
14520 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14521
14522 ins_pipe(icmp_reg_imm);
14523 %}
14524
14525 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14526 %{
14527 match(Set cr (CmpL op1 op2));
14528
14529 effect(DEF cr, USE op1);
14530
14531 ins_cost(INSN_COST);
14532 format %{ "cmp $op1, $op2" %}
14533
14534 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14535
14536 ins_pipe(icmp_reg_imm);
14537 %}
14538
14539 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14540 %{
14541 match(Set cr (CmpL op1 op2));
14542
14543 effect(DEF cr, USE op1);
14544
14545 ins_cost(INSN_COST * 2);
14546 format %{ "cmp $op1, $op2" %}
14547
14548 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14549
14550 ins_pipe(icmp_reg_imm);
14551 %}
14552
14553 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14554 %{
14555 match(Set cr (CmpUL op1 op2));
14556
14557 effect(DEF cr, USE op1, USE op2);
14558
14559 ins_cost(INSN_COST);
14560 format %{ "cmp $op1, $op2" %}
14561
14562 ins_encode(aarch64_enc_cmp(op1, op2));
14563
14564 ins_pipe(icmp_reg_reg);
14565 %}
14566
14567 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14568 %{
14569 match(Set cr (CmpUL op1 zero));
14570
14571 effect(DEF cr, USE op1);
14572
14573 ins_cost(INSN_COST);
14574 format %{ "tst $op1" %}
14575
14576 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14577
14578 ins_pipe(icmp_reg_imm);
14579 %}
14580
14581 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14582 %{
14583 match(Set cr (CmpUL op1 op2));
14584
14585 effect(DEF cr, USE op1);
14586
14587 ins_cost(INSN_COST);
14588 format %{ "cmp $op1, $op2" %}
14589
14590 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14591
14592 ins_pipe(icmp_reg_imm);
14593 %}
14594
14595 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14596 %{
14597 match(Set cr (CmpUL op1 op2));
14598
14599 effect(DEF cr, USE op1);
14600
14601 ins_cost(INSN_COST * 2);
14602 format %{ "cmp $op1, $op2" %}
14603
14604 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14605
14606 ins_pipe(icmp_reg_imm);
14607 %}
14608
14609 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14610 %{
14611 match(Set cr (CmpP op1 op2));
14612
14613 effect(DEF cr, USE op1, USE op2);
14614
14615 ins_cost(INSN_COST);
14616 format %{ "cmp $op1, $op2\t // ptr" %}
14617
14618 ins_encode(aarch64_enc_cmpp(op1, op2));
14619
14620 ins_pipe(icmp_reg_reg);
14621 %}
14622
14623 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14624 %{
14625 match(Set cr (CmpN op1 op2));
14626
14627 effect(DEF cr, USE op1, USE op2);
14628
14629 ins_cost(INSN_COST);
14630 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14631
14632 ins_encode(aarch64_enc_cmpn(op1, op2));
14633
14634 ins_pipe(icmp_reg_reg);
14635 %}
14636
14637 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14638 %{
14639 match(Set cr (CmpP op1 zero));
14640
14641 effect(DEF cr, USE op1, USE zero);
14642
14643 ins_cost(INSN_COST);
14644 format %{ "cmp $op1, 0\t // ptr" %}
14645
14646 ins_encode(aarch64_enc_testp(op1));
14647
14648 ins_pipe(icmp_reg_imm);
14649 %}
14650
14651 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14652 %{
14653 match(Set cr (CmpN op1 zero));
14654
14655 effect(DEF cr, USE op1, USE zero);
14656
14657 ins_cost(INSN_COST);
14658 format %{ "cmp $op1, 0\t // compressed ptr" %}
14659
14660 ins_encode(aarch64_enc_testn(op1));
14661
14662 ins_pipe(icmp_reg_imm);
14663 %}
14664
14665 // FP comparisons
14666 //
14667 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14668 // using normal cmpOp. See declaration of rFlagsReg for details.
14669
14670 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14671 %{
14672 match(Set cr (CmpF src1 src2));
14673
14674 ins_cost(3 * INSN_COST);
14675 format %{ "fcmps $src1, $src2" %}
14676
14677 ins_encode %{
14678 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14679 %}
14680
14681 ins_pipe(pipe_class_compare);
14682 %}
14683
14684 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14685 %{
14686 match(Set cr (CmpF src1 src2));
14687
14688 ins_cost(3 * INSN_COST);
14689 format %{ "fcmps $src1, 0.0" %}
14690
14691 ins_encode %{
14692 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14693 %}
14694
14695 ins_pipe(pipe_class_compare);
14696 %}
14697 // FROM HERE
14698
14699 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14700 %{
14701 match(Set cr (CmpD src1 src2));
14702
14703 ins_cost(3 * INSN_COST);
14704 format %{ "fcmpd $src1, $src2" %}
14705
14706 ins_encode %{
14707 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14708 %}
14709
14710 ins_pipe(pipe_class_compare);
14711 %}
14712
14713 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14714 %{
14715 match(Set cr (CmpD src1 src2));
14716
14717 ins_cost(3 * INSN_COST);
14718 format %{ "fcmpd $src1, 0.0" %}
14719
14720 ins_encode %{
14721 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14722 %}
14723
14724 ins_pipe(pipe_class_compare);
14725 %}
14726
14727 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14728 %{
14729 match(Set dst (CmpF3 src1 src2));
14730 effect(KILL cr);
14731
14732 ins_cost(5 * INSN_COST);
14733 format %{ "fcmps $src1, $src2\n\t"
14734 "csinvw($dst, zr, zr, eq\n\t"
14735 "csnegw($dst, $dst, $dst, lt)"
14736 %}
14737
14738 ins_encode %{
14739 Label done;
14740 FloatRegister s1 = as_FloatRegister($src1$$reg);
14741 FloatRegister s2 = as_FloatRegister($src2$$reg);
14742 Register d = as_Register($dst$$reg);
14743 __ fcmps(s1, s2);
14744 // installs 0 if EQ else -1
14745 __ csinvw(d, zr, zr, Assembler::EQ);
14746 // keeps -1 if less or unordered else installs 1
14747 __ csnegw(d, d, d, Assembler::LT);
14748 __ bind(done);
14749 %}
14750
14751 ins_pipe(pipe_class_default);
14752
14753 %}
14754
14755 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14756 %{
14757 match(Set dst (CmpD3 src1 src2));
14758 effect(KILL cr);
14759
14760 ins_cost(5 * INSN_COST);
14761 format %{ "fcmpd $src1, $src2\n\t"
14762 "csinvw($dst, zr, zr, eq\n\t"
14763 "csnegw($dst, $dst, $dst, lt)"
14764 %}
14765
14766 ins_encode %{
14767 Label done;
14768 FloatRegister s1 = as_FloatRegister($src1$$reg);
14769 FloatRegister s2 = as_FloatRegister($src2$$reg);
14770 Register d = as_Register($dst$$reg);
14771 __ fcmpd(s1, s2);
14772 // installs 0 if EQ else -1
14773 __ csinvw(d, zr, zr, Assembler::EQ);
14774 // keeps -1 if less or unordered else installs 1
14775 __ csnegw(d, d, d, Assembler::LT);
14776 __ bind(done);
14777 %}
14778 ins_pipe(pipe_class_default);
14779
14780 %}
14781
14782 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14783 %{
14784 match(Set dst (CmpF3 src1 zero));
14785 effect(KILL cr);
14786
14787 ins_cost(5 * INSN_COST);
14788 format %{ "fcmps $src1, 0.0\n\t"
14789 "csinvw($dst, zr, zr, eq\n\t"
14790 "csnegw($dst, $dst, $dst, lt)"
14791 %}
14792
14793 ins_encode %{
14794 Label done;
14795 FloatRegister s1 = as_FloatRegister($src1$$reg);
14796 Register d = as_Register($dst$$reg);
14797 __ fcmps(s1, 0.0);
14798 // installs 0 if EQ else -1
14799 __ csinvw(d, zr, zr, Assembler::EQ);
14800 // keeps -1 if less or unordered else installs 1
14801 __ csnegw(d, d, d, Assembler::LT);
14802 __ bind(done);
14803 %}
14804
14805 ins_pipe(pipe_class_default);
14806
14807 %}
14808
14809 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14810 %{
14811 match(Set dst (CmpD3 src1 zero));
14812 effect(KILL cr);
14813
14814 ins_cost(5 * INSN_COST);
14815 format %{ "fcmpd $src1, 0.0\n\t"
14816 "csinvw($dst, zr, zr, eq\n\t"
14817 "csnegw($dst, $dst, $dst, lt)"
14818 %}
14819
14820 ins_encode %{
14821 Label done;
14822 FloatRegister s1 = as_FloatRegister($src1$$reg);
14823 Register d = as_Register($dst$$reg);
14824 __ fcmpd(s1, 0.0);
14825 // installs 0 if EQ else -1
14826 __ csinvw(d, zr, zr, Assembler::EQ);
14827 // keeps -1 if less or unordered else installs 1
14828 __ csnegw(d, d, d, Assembler::LT);
14829 __ bind(done);
14830 %}
14831 ins_pipe(pipe_class_default);
14832
14833 %}
14834
14835 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14836 %{
14837 match(Set dst (CmpLTMask p q));
14838 effect(KILL cr);
14839
14840 ins_cost(3 * INSN_COST);
14841
14842 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14843 "csetw $dst, lt\n\t"
14844 "subw $dst, zr, $dst"
14845 %}
14846
14847 ins_encode %{
14848 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14849 __ csetw(as_Register($dst$$reg), Assembler::LT);
14850 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14851 %}
14852
14853 ins_pipe(ialu_reg_reg);
14854 %}
14855
14856 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14857 %{
14858 match(Set dst (CmpLTMask src zero));
14859 effect(KILL cr);
14860
14861 ins_cost(INSN_COST);
14862
14863 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14864
14865 ins_encode %{
14866 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14867 %}
14868
14869 ins_pipe(ialu_reg_shift);
14870 %}
14871
14872 // ============================================================================
14873 // Max and Min
14874
14875 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14876
14877 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14878 %{
14879 effect(DEF cr, USE src);
14880 ins_cost(INSN_COST);
14881 format %{ "cmpw $src, 0" %}
14882
14883 ins_encode %{
14884 __ cmpw($src$$Register, 0);
14885 %}
14886 ins_pipe(icmp_reg_imm);
14887 %}
14888
14889 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14890 %{
14891 match(Set dst (MinI src1 src2));
14892 ins_cost(INSN_COST * 3);
14893
14894 expand %{
14895 rFlagsReg cr;
14896 compI_reg_reg(cr, src1, src2);
14897 cmovI_reg_reg_lt(dst, src1, src2, cr);
14898 %}
14899 %}
14900
14901 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14902 %{
14903 match(Set dst (MaxI src1 src2));
14904 ins_cost(INSN_COST * 3);
14905
14906 expand %{
14907 rFlagsReg cr;
14908 compI_reg_reg(cr, src1, src2);
14909 cmovI_reg_reg_gt(dst, src1, src2, cr);
14910 %}
14911 %}
14912
14913
14914 // ============================================================================
14915 // Branch Instructions
14916
14917 // Direct Branch.
14918 instruct branch(label lbl)
14919 %{
14920 match(Goto);
14921
14922 effect(USE lbl);
14923
14924 ins_cost(BRANCH_COST);
14925 format %{ "b $lbl" %}
14926
14927 ins_encode(aarch64_enc_b(lbl));
14928
14929 ins_pipe(pipe_branch);
14930 %}
14931
14932 // Conditional Near Branch
14933 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
14934 %{
14935 // Same match rule as `branchConFar'.
14936 match(If cmp cr);
14937
14938 effect(USE lbl);
14939
14940 ins_cost(BRANCH_COST);
14941 // If set to 1 this indicates that the current instruction is a
14942 // short variant of a long branch. This avoids using this
14943 // instruction in first-pass matching. It will then only be used in
14944 // the `Shorten_branches' pass.
14945 // ins_short_branch(1);
14946 format %{ "b$cmp $lbl" %}
14947
14948 ins_encode(aarch64_enc_br_con(cmp, lbl));
14949
14950 ins_pipe(pipe_branch_cond);
14951 %}
14952
14953 // Conditional Near Branch Unsigned
14954 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
14955 %{
14956 // Same match rule as `branchConFar'.
14957 match(If cmp cr);
14958
14959 effect(USE lbl);
14960
14961 ins_cost(BRANCH_COST);
14962 // If set to 1 this indicates that the current instruction is a
14963 // short variant of a long branch. This avoids using this
14964 // instruction in first-pass matching. It will then only be used in
14965 // the `Shorten_branches' pass.
14966 // ins_short_branch(1);
14967 format %{ "b$cmp $lbl\t# unsigned" %}
14968
14969 ins_encode(aarch64_enc_br_conU(cmp, lbl));
14970
14971 ins_pipe(pipe_branch_cond);
14972 %}
14973
14974 // Make use of CBZ and CBNZ. These instructions, as well as being
14975 // shorter than (cmp; branch), have the additional benefit of not
14976 // killing the flags.
14977
14978 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
14979 match(If cmp (CmpI op1 op2));
14980 effect(USE labl);
14981
14982 ins_cost(BRANCH_COST);
14983 format %{ "cbw$cmp $op1, $labl" %}
14984 ins_encode %{
14985 Label* L = $labl$$label;
14986 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14987 if (cond == Assembler::EQ)
14988 __ cbzw($op1$$Register, *L);
14989 else
14990 __ cbnzw($op1$$Register, *L);
14991 %}
14992 ins_pipe(pipe_cmp_branch);
14993 %}
14994
14995 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
14996 match(If cmp (CmpL op1 op2));
14997 effect(USE labl);
14998
14999 ins_cost(BRANCH_COST);
15000 format %{ "cb$cmp $op1, $labl" %}
15001 ins_encode %{
15002 Label* L = $labl$$label;
15003 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15004 if (cond == Assembler::EQ)
15005 __ cbz($op1$$Register, *L);
15006 else
15007 __ cbnz($op1$$Register, *L);
15008 %}
15009 ins_pipe(pipe_cmp_branch);
15010 %}
15011
15012 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15013 match(If cmp (CmpP op1 op2));
15014 effect(USE labl);
15015
15016 ins_cost(BRANCH_COST);
15017 format %{ "cb$cmp $op1, $labl" %}
15018 ins_encode %{
15019 Label* L = $labl$$label;
15020 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15021 if (cond == Assembler::EQ)
15022 __ cbz($op1$$Register, *L);
15023 else
15024 __ cbnz($op1$$Register, *L);
15025 %}
15026 ins_pipe(pipe_cmp_branch);
15027 %}
15028
15029 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15030 match(If cmp (CmpN op1 op2));
15031 effect(USE labl);
15032
15033 ins_cost(BRANCH_COST);
15034 format %{ "cbw$cmp $op1, $labl" %}
15035 ins_encode %{
15036 Label* L = $labl$$label;
15037 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15038 if (cond == Assembler::EQ)
15039 __ cbzw($op1$$Register, *L);
15040 else
15041 __ cbnzw($op1$$Register, *L);
15042 %}
15043 ins_pipe(pipe_cmp_branch);
15044 %}
15045
15046 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15047 match(If cmp (CmpP (DecodeN oop) zero));
15048 effect(USE labl);
15049
15050 ins_cost(BRANCH_COST);
15051 format %{ "cb$cmp $oop, $labl" %}
15052 ins_encode %{
15053 Label* L = $labl$$label;
15054 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15055 if (cond == Assembler::EQ)
15056 __ cbzw($oop$$Register, *L);
15057 else
15058 __ cbnzw($oop$$Register, *L);
15059 %}
15060 ins_pipe(pipe_cmp_branch);
15061 %}
15062
15063 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15064 match(If cmp (CmpU op1 op2));
15065 effect(USE labl);
15066
15067 ins_cost(BRANCH_COST);
15068 format %{ "cbw$cmp $op1, $labl" %}
15069 ins_encode %{
15070 Label* L = $labl$$label;
15071 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15072 if (cond == Assembler::EQ || cond == Assembler::LS) {
15073 __ cbzw($op1$$Register, *L);
15074 } else {
15075 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15076 __ cbnzw($op1$$Register, *L);
15077 }
15078 %}
15079 ins_pipe(pipe_cmp_branch);
15080 %}
15081
15082 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15083 match(If cmp (CmpUL op1 op2));
15084 effect(USE labl);
15085
15086 ins_cost(BRANCH_COST);
15087 format %{ "cb$cmp $op1, $labl" %}
15088 ins_encode %{
15089 Label* L = $labl$$label;
15090 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15091 if (cond == Assembler::EQ || cond == Assembler::LS) {
15092 __ cbz($op1$$Register, *L);
15093 } else {
15094 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15095 __ cbnz($op1$$Register, *L);
15096 }
15097 %}
15098 ins_pipe(pipe_cmp_branch);
15099 %}
15100
15101 // Test bit and Branch
15102
15103 // Patterns for short (< 32KiB) variants
15104 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15105 match(If cmp (CmpL op1 op2));
15106 effect(USE labl);
15107
15108 ins_cost(BRANCH_COST);
15109 format %{ "cb$cmp $op1, $labl # long" %}
15110 ins_encode %{
15111 Label* L = $labl$$label;
15112 Assembler::Condition cond =
15113 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15114 __ tbr(cond, $op1$$Register, 63, *L);
15115 %}
15116 ins_pipe(pipe_cmp_branch);
15117 ins_short_branch(1);
15118 %}
15119
15120 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15121 match(If cmp (CmpI op1 op2));
15122 effect(USE labl);
15123
15124 ins_cost(BRANCH_COST);
15125 format %{ "cb$cmp $op1, $labl # int" %}
15126 ins_encode %{
15127 Label* L = $labl$$label;
15128 Assembler::Condition cond =
15129 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15130 __ tbr(cond, $op1$$Register, 31, *L);
15131 %}
15132 ins_pipe(pipe_cmp_branch);
15133 ins_short_branch(1);
15134 %}
15135
15136 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15137 match(If cmp (CmpL (AndL op1 op2) op3));
15138 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15139 effect(USE labl);
15140
15141 ins_cost(BRANCH_COST);
15142 format %{ "tb$cmp $op1, $op2, $labl" %}
15143 ins_encode %{
15144 Label* L = $labl$$label;
15145 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15146 int bit = exact_log2_long($op2$$constant);
15147 __ tbr(cond, $op1$$Register, bit, *L);
15148 %}
15149 ins_pipe(pipe_cmp_branch);
15150 ins_short_branch(1);
15151 %}
15152
15153 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15154 match(If cmp (CmpI (AndI op1 op2) op3));
15155 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15156 effect(USE labl);
15157
15158 ins_cost(BRANCH_COST);
15159 format %{ "tb$cmp $op1, $op2, $labl" %}
15160 ins_encode %{
15161 Label* L = $labl$$label;
15162 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15163 int bit = exact_log2((juint)$op2$$constant);
15164 __ tbr(cond, $op1$$Register, bit, *L);
15165 %}
15166 ins_pipe(pipe_cmp_branch);
15167 ins_short_branch(1);
15168 %}
15169
15170 // And far variants
15171 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15172 match(If cmp (CmpL op1 op2));
15173 effect(USE labl);
15174
15175 ins_cost(BRANCH_COST);
15176 format %{ "cb$cmp $op1, $labl # long" %}
15177 ins_encode %{
15178 Label* L = $labl$$label;
15179 Assembler::Condition cond =
15180 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15181 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15182 %}
15183 ins_pipe(pipe_cmp_branch);
15184 %}
15185
15186 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15187 match(If cmp (CmpI op1 op2));
15188 effect(USE labl);
15189
15190 ins_cost(BRANCH_COST);
15191 format %{ "cb$cmp $op1, $labl # int" %}
15192 ins_encode %{
15193 Label* L = $labl$$label;
15194 Assembler::Condition cond =
15195 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15196 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15197 %}
15198 ins_pipe(pipe_cmp_branch);
15199 %}
15200
15201 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15202 match(If cmp (CmpL (AndL op1 op2) op3));
15203 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15204 effect(USE labl);
15205
15206 ins_cost(BRANCH_COST);
15207 format %{ "tb$cmp $op1, $op2, $labl" %}
15208 ins_encode %{
15209 Label* L = $labl$$label;
15210 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15211 int bit = exact_log2_long($op2$$constant);
15212 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15213 %}
15214 ins_pipe(pipe_cmp_branch);
15215 %}
15216
15217 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15218 match(If cmp (CmpI (AndI op1 op2) op3));
15219 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15220 effect(USE labl);
15221
15222 ins_cost(BRANCH_COST);
15223 format %{ "tb$cmp $op1, $op2, $labl" %}
15224 ins_encode %{
15225 Label* L = $labl$$label;
15226 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15227 int bit = exact_log2((juint)$op2$$constant);
15228 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15229 %}
15230 ins_pipe(pipe_cmp_branch);
15231 %}
15232
15233 // Test bits
15234
15235 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15236 match(Set cr (CmpL (AndL op1 op2) op3));
15237 predicate(Assembler::operand_valid_for_logical_immediate
15238 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15239
15240 ins_cost(INSN_COST);
15241 format %{ "tst $op1, $op2 # long" %}
15242 ins_encode %{
15243 __ tst($op1$$Register, $op2$$constant);
15244 %}
15245 ins_pipe(ialu_reg_reg);
15246 %}
15247
15248 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15249 match(Set cr (CmpI (AndI op1 op2) op3));
15250 predicate(Assembler::operand_valid_for_logical_immediate
15251 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15252
15253 ins_cost(INSN_COST);
15254 format %{ "tst $op1, $op2 # int" %}
15255 ins_encode %{
15256 __ tstw($op1$$Register, $op2$$constant);
15257 %}
15258 ins_pipe(ialu_reg_reg);
15259 %}
15260
15261 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15262 match(Set cr (CmpL (AndL op1 op2) op3));
15263
15264 ins_cost(INSN_COST);
15265 format %{ "tst $op1, $op2 # long" %}
15266 ins_encode %{
15267 __ tst($op1$$Register, $op2$$Register);
15268 %}
15269 ins_pipe(ialu_reg_reg);
15270 %}
15271
15272 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15273 match(Set cr (CmpI (AndI op1 op2) op3));
15274
15275 ins_cost(INSN_COST);
15276 format %{ "tstw $op1, $op2 # int" %}
15277 ins_encode %{
15278 __ tstw($op1$$Register, $op2$$Register);
15279 %}
15280 ins_pipe(ialu_reg_reg);
15281 %}
15282
15283
15284 // Conditional Far Branch
15285 // Conditional Far Branch Unsigned
15286 // TODO: fixme
15287
15288 // counted loop end branch near
15289 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15290 %{
15291 match(CountedLoopEnd cmp cr);
15292
15293 effect(USE lbl);
15294
15295 ins_cost(BRANCH_COST);
15296 // short variant.
15297 // ins_short_branch(1);
15298 format %{ "b$cmp $lbl \t// counted loop end" %}
15299
15300 ins_encode(aarch64_enc_br_con(cmp, lbl));
15301
15302 ins_pipe(pipe_branch);
15303 %}
15304
15305 // counted loop end branch far
15306 // TODO: fixme
15307
15308 // ============================================================================
15309 // inlined locking and unlocking
15310
15311 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15312 %{
15313 match(Set cr (FastLock object box));
15314 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15315
15316 ins_cost(5 * INSN_COST);
15317 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15318
15319 ins_encode %{
15320 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15321 %}
15322
15323 ins_pipe(pipe_serial);
15324 %}
15325
15326 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15327 %{
15328 match(Set cr (FastUnlock object box));
15329 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15330
15331 ins_cost(5 * INSN_COST);
15332 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15333
15334 ins_encode %{
15335 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15336 %}
15337
15338 ins_pipe(pipe_serial);
15339 %}
15340
15341 // ============================================================================
15342 // Safepoint Instructions
15343
15344 // TODO
15345 // provide a near and far version of this code
15346
15347 instruct safePoint(rFlagsReg cr, iRegP poll)
15348 %{
15349 match(SafePoint poll);
15350 effect(KILL cr);
15351
15352 format %{
15353 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15354 %}
15355 ins_encode %{
15356 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15357 %}
15358 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15359 %}
15360
15361
15362 // ============================================================================
15363 // Procedure Call/Return Instructions
15364
15365 // Call Java Static Instruction
15366
15367 instruct CallStaticJavaDirect(method meth)
15368 %{
15369 match(CallStaticJava);
15370
15371 effect(USE meth);
15372
15373 ins_cost(CALL_COST);
15374
15375 format %{ "call,static $meth \t// ==> " %}
15376
15377 ins_encode(aarch64_enc_java_static_call(meth),
15378 aarch64_enc_call_epilog);
15379
15380 ins_pipe(pipe_class_call);
15381 %}
15382
15383 // TO HERE
15384
15385 // Call Java Dynamic Instruction
15386 instruct CallDynamicJavaDirect(method meth)
15387 %{
15388 match(CallDynamicJava);
15389
15390 effect(USE meth);
15391
15392 ins_cost(CALL_COST);
15393
15394 format %{ "CALL,dynamic $meth \t// ==> " %}
15395
15396 ins_encode(aarch64_enc_java_dynamic_call(meth),
15397 aarch64_enc_call_epilog);
15398
15399 ins_pipe(pipe_class_call);
15400 %}
15401
15402 // Call Runtime Instruction
15403
15404 instruct CallRuntimeDirect(method meth)
15405 %{
15406 match(CallRuntime);
15407
15408 effect(USE meth);
15409
15410 ins_cost(CALL_COST);
15411
15412 format %{ "CALL, runtime $meth" %}
15413
15414 ins_encode( aarch64_enc_java_to_runtime(meth) );
15415
15416 ins_pipe(pipe_class_call);
15417 %}
15418
15419 // Call Runtime Instruction
15420
15421 instruct CallLeafDirect(method meth)
15422 %{
15423 match(CallLeaf);
15424
15425 effect(USE meth);
15426
15427 ins_cost(CALL_COST);
15428
15429 format %{ "CALL, runtime leaf $meth" %}
15430
15431 ins_encode( aarch64_enc_java_to_runtime(meth) );
15432
15433 ins_pipe(pipe_class_call);
15434 %}
15435
15436 // Call Runtime Instruction without safepoint and with vector arguments
15437 instruct CallLeafDirectVector(method meth)
15438 %{
15439 match(CallLeafVector);
15440
15441 effect(USE meth);
15442
15443 ins_cost(CALL_COST);
15444
15445 format %{ "CALL, runtime leaf vector $meth" %}
15446
15447 ins_encode(aarch64_enc_java_to_runtime(meth));
15448
15449 ins_pipe(pipe_class_call);
15450 %}
15451
15452 // Call Runtime Instruction
15453
15454 // entry point is null, target holds the address to call
15455 instruct CallLeafNoFPIndirect(iRegP target)
15456 %{
15457 predicate(n->as_Call()->entry_point() == nullptr);
15458
15459 match(CallLeafNoFP target);
15460
15461 ins_cost(CALL_COST);
15462
15463 format %{ "CALL, runtime leaf nofp indirect $target" %}
15464
15465 ins_encode %{
15466 __ blr($target$$Register);
15467 %}
15468
15469 ins_pipe(pipe_class_call);
15470 %}
15471
15472 instruct CallLeafNoFPDirect(method meth)
15473 %{
15474 predicate(n->as_Call()->entry_point() != nullptr);
15475
15476 match(CallLeafNoFP);
15477
15478 effect(USE meth);
15479
15480 ins_cost(CALL_COST);
15481
15482 format %{ "CALL, runtime leaf nofp $meth" %}
15483
15484 ins_encode( aarch64_enc_java_to_runtime(meth) );
15485
15486 ins_pipe(pipe_class_call);
15487 %}
15488
15489 // Tail Call; Jump from runtime stub to Java code.
15490 // Also known as an 'interprocedural jump'.
15491 // Target of jump will eventually return to caller.
15492 // TailJump below removes the return address.
15493 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15494 // emitted just above the TailCall which has reset rfp to the caller state.
15495 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15496 %{
15497 match(TailCall jump_target method_ptr);
15498
15499 ins_cost(CALL_COST);
15500
15501 format %{ "br $jump_target\t# $method_ptr holds method" %}
15502
15503 ins_encode(aarch64_enc_tail_call(jump_target));
15504
15505 ins_pipe(pipe_class_call);
15506 %}
15507
15508 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15509 %{
15510 match(TailJump jump_target ex_oop);
15511
15512 ins_cost(CALL_COST);
15513
15514 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15515
15516 ins_encode(aarch64_enc_tail_jmp(jump_target));
15517
15518 ins_pipe(pipe_class_call);
15519 %}
15520
15521 // Forward exception.
15522 instruct ForwardExceptionjmp()
15523 %{
15524 match(ForwardException);
15525 ins_cost(CALL_COST);
15526
15527 format %{ "b forward_exception_stub" %}
15528 ins_encode %{
15529 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15530 %}
15531 ins_pipe(pipe_class_call);
15532 %}
15533
15534 // Create exception oop: created by stack-crawling runtime code.
15535 // Created exception is now available to this handler, and is setup
15536 // just prior to jumping to this handler. No code emitted.
15537 // TODO check
15538 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15539 instruct CreateException(iRegP_R0 ex_oop)
15540 %{
15541 match(Set ex_oop (CreateEx));
15542
15543 format %{ " -- \t// exception oop; no code emitted" %}
15544
15545 size(0);
15546
15547 ins_encode( /*empty*/ );
15548
15549 ins_pipe(pipe_class_empty);
15550 %}
15551
15552 // Rethrow exception: The exception oop will come in the first
15553 // argument position. Then JUMP (not call) to the rethrow stub code.
15554 instruct RethrowException() %{
15555 match(Rethrow);
15556 ins_cost(CALL_COST);
15557
15558 format %{ "b rethrow_stub" %}
15559
15560 ins_encode( aarch64_enc_rethrow() );
15561
15562 ins_pipe(pipe_class_call);
15563 %}
15564
15565
15566 // Return Instruction
15567 // epilog node loads ret address into lr as part of frame pop
15568 instruct Ret()
15569 %{
15570 match(Return);
15571
15572 format %{ "ret\t// return register" %}
15573
15574 ins_encode( aarch64_enc_ret() );
15575
15576 ins_pipe(pipe_branch);
15577 %}
15578
15579 // Die now.
15580 instruct ShouldNotReachHere() %{
15581 match(Halt);
15582
15583 ins_cost(CALL_COST);
15584 format %{ "ShouldNotReachHere" %}
15585
15586 ins_encode %{
15587 if (is_reachable()) {
15588 const char* str = __ code_string(_halt_reason);
15589 __ stop(str);
15590 }
15591 %}
15592
15593 ins_pipe(pipe_class_default);
15594 %}
15595
15596 // ============================================================================
15597 // Partial Subtype Check
15598 //
15599 // superklass array for an instance of the superklass. Set a hidden
15600 // internal cache on a hit (cache is checked with exposed code in
15601 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15602 // encoding ALSO sets flags.
15603
15604 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15605 %{
15606 match(Set result (PartialSubtypeCheck sub super));
15607 predicate(!UseSecondarySupersTable);
15608 effect(KILL cr, KILL temp);
15609
15610 ins_cost(20 * INSN_COST); // slightly larger than the next version
15611 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15612
15613 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15614
15615 opcode(0x1); // Force zero of result reg on hit
15616
15617 ins_pipe(pipe_class_memory);
15618 %}
15619
15620 // Two versions of partialSubtypeCheck, both used when we need to
15621 // search for a super class in the secondary supers array. The first
15622 // is used when we don't know _a priori_ the class being searched
15623 // for. The second, far more common, is used when we do know: this is
15624 // used for instanceof, checkcast, and any case where C2 can determine
15625 // it by constant propagation.
15626
15627 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15628 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15629 rFlagsReg cr)
15630 %{
15631 match(Set result (PartialSubtypeCheck sub super));
15632 predicate(UseSecondarySupersTable);
15633 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15634
15635 ins_cost(10 * INSN_COST); // slightly larger than the next version
15636 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15637
15638 ins_encode %{
15639 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15640 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15641 $vtemp$$FloatRegister,
15642 $result$$Register, /*L_success*/nullptr);
15643 %}
15644
15645 ins_pipe(pipe_class_memory);
15646 %}
15647
15648 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15649 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15650 rFlagsReg cr)
15651 %{
15652 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15653 predicate(UseSecondarySupersTable);
15654 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15655
15656 ins_cost(5 * INSN_COST); // smaller than the next version
15657 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15658
15659 ins_encode %{
15660 bool success = false;
15661 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15662 if (InlineSecondarySupersTest) {
15663 success =
15664 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15665 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15666 $vtemp$$FloatRegister,
15667 $result$$Register,
15668 super_klass_slot);
15669 } else {
15670 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15671 success = (call != nullptr);
15672 }
15673 if (!success) {
15674 ciEnv::current()->record_failure("CodeCache is full");
15675 return;
15676 }
15677 %}
15678
15679 ins_pipe(pipe_class_memory);
15680 %}
15681
15682 // Intrisics for String.compareTo()
15683
15684 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15685 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15686 %{
15687 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15688 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15689 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15690
15691 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15692 ins_encode %{
15693 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15694 __ string_compare($str1$$Register, $str2$$Register,
15695 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15696 $tmp1$$Register, $tmp2$$Register,
15697 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15698 %}
15699 ins_pipe(pipe_class_memory);
15700 %}
15701
15702 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15703 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15704 %{
15705 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15706 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15707 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15708
15709 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15710 ins_encode %{
15711 __ string_compare($str1$$Register, $str2$$Register,
15712 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15713 $tmp1$$Register, $tmp2$$Register,
15714 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15715 %}
15716 ins_pipe(pipe_class_memory);
15717 %}
15718
15719 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15720 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15721 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15722 %{
15723 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15724 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15725 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15726 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15727
15728 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15729 ins_encode %{
15730 __ string_compare($str1$$Register, $str2$$Register,
15731 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15732 $tmp1$$Register, $tmp2$$Register,
15733 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15734 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15735 %}
15736 ins_pipe(pipe_class_memory);
15737 %}
15738
15739 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15740 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15741 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15742 %{
15743 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15744 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15745 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15746 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15747
15748 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15749 ins_encode %{
15750 __ string_compare($str1$$Register, $str2$$Register,
15751 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15752 $tmp1$$Register, $tmp2$$Register,
15753 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15754 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15755 %}
15756 ins_pipe(pipe_class_memory);
15757 %}
15758
15759 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15760 // these string_compare variants as NEON register type for convenience so that the prototype of
15761 // string_compare can be shared with all variants.
15762
15763 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15764 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15765 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15766 pRegGov_P1 pgtmp2, rFlagsReg cr)
15767 %{
15768 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15769 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15770 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15771 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15772
15773 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15774 ins_encode %{
15775 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15776 __ string_compare($str1$$Register, $str2$$Register,
15777 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15778 $tmp1$$Register, $tmp2$$Register,
15779 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15780 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15781 StrIntrinsicNode::LL);
15782 %}
15783 ins_pipe(pipe_class_memory);
15784 %}
15785
15786 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15787 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15788 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15789 pRegGov_P1 pgtmp2, rFlagsReg cr)
15790 %{
15791 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15792 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15793 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15794 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15795
15796 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15797 ins_encode %{
15798 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15799 __ string_compare($str1$$Register, $str2$$Register,
15800 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15801 $tmp1$$Register, $tmp2$$Register,
15802 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15803 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15804 StrIntrinsicNode::LU);
15805 %}
15806 ins_pipe(pipe_class_memory);
15807 %}
15808
15809 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15810 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15811 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15812 pRegGov_P1 pgtmp2, rFlagsReg cr)
15813 %{
15814 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15815 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15816 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15817 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15818
15819 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15820 ins_encode %{
15821 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15822 __ string_compare($str1$$Register, $str2$$Register,
15823 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15824 $tmp1$$Register, $tmp2$$Register,
15825 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15826 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15827 StrIntrinsicNode::UL);
15828 %}
15829 ins_pipe(pipe_class_memory);
15830 %}
15831
15832 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15833 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15834 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15835 pRegGov_P1 pgtmp2, rFlagsReg cr)
15836 %{
15837 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15838 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15839 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15840 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15841
15842 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15843 ins_encode %{
15844 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15845 __ string_compare($str1$$Register, $str2$$Register,
15846 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15847 $tmp1$$Register, $tmp2$$Register,
15848 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15849 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15850 StrIntrinsicNode::UU);
15851 %}
15852 ins_pipe(pipe_class_memory);
15853 %}
15854
15855 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15856 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15857 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15858 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15859 %{
15860 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15861 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15862 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15863 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15864 TEMP vtmp0, TEMP vtmp1, KILL cr);
15865 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15866 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15867
15868 ins_encode %{
15869 __ string_indexof($str1$$Register, $str2$$Register,
15870 $cnt1$$Register, $cnt2$$Register,
15871 $tmp1$$Register, $tmp2$$Register,
15872 $tmp3$$Register, $tmp4$$Register,
15873 $tmp5$$Register, $tmp6$$Register,
15874 -1, $result$$Register, StrIntrinsicNode::UU);
15875 %}
15876 ins_pipe(pipe_class_memory);
15877 %}
15878
15879 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15880 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15881 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15882 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15883 %{
15884 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15885 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15886 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15887 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15888 TEMP vtmp0, TEMP vtmp1, KILL cr);
15889 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15890 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15891
15892 ins_encode %{
15893 __ string_indexof($str1$$Register, $str2$$Register,
15894 $cnt1$$Register, $cnt2$$Register,
15895 $tmp1$$Register, $tmp2$$Register,
15896 $tmp3$$Register, $tmp4$$Register,
15897 $tmp5$$Register, $tmp6$$Register,
15898 -1, $result$$Register, StrIntrinsicNode::LL);
15899 %}
15900 ins_pipe(pipe_class_memory);
15901 %}
15902
15903 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15904 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15905 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15906 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15907 %{
15908 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15909 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15910 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15911 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15912 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15913 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15914 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15915
15916 ins_encode %{
15917 __ string_indexof($str1$$Register, $str2$$Register,
15918 $cnt1$$Register, $cnt2$$Register,
15919 $tmp1$$Register, $tmp2$$Register,
15920 $tmp3$$Register, $tmp4$$Register,
15921 $tmp5$$Register, $tmp6$$Register,
15922 -1, $result$$Register, StrIntrinsicNode::UL);
15923 %}
15924 ins_pipe(pipe_class_memory);
15925 %}
15926
15927 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15928 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15929 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15930 %{
15931 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15932 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15933 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15934 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15935 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
15936 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15937
15938 ins_encode %{
15939 int icnt2 = (int)$int_cnt2$$constant;
15940 __ string_indexof($str1$$Register, $str2$$Register,
15941 $cnt1$$Register, zr,
15942 $tmp1$$Register, $tmp2$$Register,
15943 $tmp3$$Register, $tmp4$$Register, zr, zr,
15944 icnt2, $result$$Register, StrIntrinsicNode::UU);
15945 %}
15946 ins_pipe(pipe_class_memory);
15947 %}
15948
15949 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15950 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15951 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15952 %{
15953 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15954 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15955 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15956 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15957 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
15958 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15959
15960 ins_encode %{
15961 int icnt2 = (int)$int_cnt2$$constant;
15962 __ string_indexof($str1$$Register, $str2$$Register,
15963 $cnt1$$Register, zr,
15964 $tmp1$$Register, $tmp2$$Register,
15965 $tmp3$$Register, $tmp4$$Register, zr, zr,
15966 icnt2, $result$$Register, StrIntrinsicNode::LL);
15967 %}
15968 ins_pipe(pipe_class_memory);
15969 %}
15970
15971 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
15972 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
15973 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
15974 %{
15975 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15976 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
15977 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
15978 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
15979 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
15980 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
15981
15982 ins_encode %{
15983 int icnt2 = (int)$int_cnt2$$constant;
15984 __ string_indexof($str1$$Register, $str2$$Register,
15985 $cnt1$$Register, zr,
15986 $tmp1$$Register, $tmp2$$Register,
15987 $tmp3$$Register, $tmp4$$Register, zr, zr,
15988 icnt2, $result$$Register, StrIntrinsicNode::UL);
15989 %}
15990 ins_pipe(pipe_class_memory);
15991 %}
15992
15993 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
15994 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15995 iRegINoSp tmp3, rFlagsReg cr)
15996 %{
15997 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
15998 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
15999 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16000 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16001
16002 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16003
16004 ins_encode %{
16005 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16006 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16007 $tmp3$$Register);
16008 %}
16009 ins_pipe(pipe_class_memory);
16010 %}
16011
16012 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16013 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16014 iRegINoSp tmp3, rFlagsReg cr)
16015 %{
16016 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16017 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16018 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16019 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16020
16021 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16022
16023 ins_encode %{
16024 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16025 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16026 $tmp3$$Register);
16027 %}
16028 ins_pipe(pipe_class_memory);
16029 %}
16030
16031 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16032 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16033 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16034 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16035 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16036 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16037 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16038 ins_encode %{
16039 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16040 $result$$Register, $ztmp1$$FloatRegister,
16041 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16042 $ptmp$$PRegister, true /* isL */);
16043 %}
16044 ins_pipe(pipe_class_memory);
16045 %}
16046
16047 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16048 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16049 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16050 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16051 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16052 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16053 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16054 ins_encode %{
16055 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16056 $result$$Register, $ztmp1$$FloatRegister,
16057 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16058 $ptmp$$PRegister, false /* isL */);
16059 %}
16060 ins_pipe(pipe_class_memory);
16061 %}
16062
16063 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16064 iRegI_R0 result, rFlagsReg cr)
16065 %{
16066 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16067 match(Set result (StrEquals (Binary str1 str2) cnt));
16068 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16069
16070 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16071 ins_encode %{
16072 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16073 __ string_equals($str1$$Register, $str2$$Register,
16074 $result$$Register, $cnt$$Register);
16075 %}
16076 ins_pipe(pipe_class_memory);
16077 %}
16078
16079 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16080 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16081 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16082 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16083 iRegP_R10 tmp, rFlagsReg cr)
16084 %{
16085 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16086 match(Set result (AryEq ary1 ary2));
16087 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16088 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16089 TEMP vtmp6, TEMP vtmp7, KILL cr);
16090
16091 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16092 ins_encode %{
16093 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16094 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16095 $result$$Register, $tmp$$Register, 1);
16096 if (tpc == nullptr) {
16097 ciEnv::current()->record_failure("CodeCache is full");
16098 return;
16099 }
16100 %}
16101 ins_pipe(pipe_class_memory);
16102 %}
16103
16104 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16105 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16106 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16107 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16108 iRegP_R10 tmp, rFlagsReg cr)
16109 %{
16110 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16111 match(Set result (AryEq ary1 ary2));
16112 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16113 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16114 TEMP vtmp6, TEMP vtmp7, KILL cr);
16115
16116 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16117 ins_encode %{
16118 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16119 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16120 $result$$Register, $tmp$$Register, 2);
16121 if (tpc == nullptr) {
16122 ciEnv::current()->record_failure("CodeCache is full");
16123 return;
16124 }
16125 %}
16126 ins_pipe(pipe_class_memory);
16127 %}
16128
16129 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16130 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16131 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16132 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16133 %{
16134 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16135 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16136 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16137
16138 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16139 ins_encode %{
16140 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16141 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16142 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16143 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16144 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16145 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16146 (BasicType)$basic_type$$constant);
16147 if (tpc == nullptr) {
16148 ciEnv::current()->record_failure("CodeCache is full");
16149 return;
16150 }
16151 %}
16152 ins_pipe(pipe_class_memory);
16153 %}
16154
16155 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16156 %{
16157 match(Set result (CountPositives ary1 len));
16158 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16159 format %{ "count positives byte[] $ary1,$len -> $result" %}
16160 ins_encode %{
16161 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16162 if (tpc == nullptr) {
16163 ciEnv::current()->record_failure("CodeCache is full");
16164 return;
16165 }
16166 %}
16167 ins_pipe( pipe_slow );
16168 %}
16169
16170 // fast char[] to byte[] compression
16171 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16172 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16173 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16174 iRegI_R0 result, rFlagsReg cr)
16175 %{
16176 match(Set result (StrCompressedCopy src (Binary dst len)));
16177 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16178 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16179
16180 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16181 ins_encode %{
16182 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16183 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16184 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16185 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16186 %}
16187 ins_pipe(pipe_slow);
16188 %}
16189
16190 // fast byte[] to char[] inflation
16191 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16192 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16193 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16194 %{
16195 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16196 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16197 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16198 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16199
16200 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16201 ins_encode %{
16202 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16203 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16204 $vtmp2$$FloatRegister, $tmp$$Register);
16205 if (tpc == nullptr) {
16206 ciEnv::current()->record_failure("CodeCache is full");
16207 return;
16208 }
16209 %}
16210 ins_pipe(pipe_class_memory);
16211 %}
16212
16213 // encode char[] to byte[] in ISO_8859_1
16214 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16215 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16216 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16217 iRegI_R0 result, rFlagsReg cr)
16218 %{
16219 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16220 match(Set result (EncodeISOArray src (Binary dst len)));
16221 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16222 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16223
16224 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16225 ins_encode %{
16226 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16227 $result$$Register, false,
16228 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16229 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16230 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16231 %}
16232 ins_pipe(pipe_class_memory);
16233 %}
16234
16235 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16236 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16237 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16238 iRegI_R0 result, rFlagsReg cr)
16239 %{
16240 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16241 match(Set result (EncodeISOArray src (Binary dst len)));
16242 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16243 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16244
16245 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16246 ins_encode %{
16247 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16248 $result$$Register, true,
16249 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16250 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16251 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16252 %}
16253 ins_pipe(pipe_class_memory);
16254 %}
16255
16256 //----------------------------- CompressBits/ExpandBits ------------------------
16257
16258 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16259 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16260 match(Set dst (CompressBits src mask));
16261 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16262 format %{ "mov $tsrc, $src\n\t"
16263 "mov $tmask, $mask\n\t"
16264 "bext $tdst, $tsrc, $tmask\n\t"
16265 "mov $dst, $tdst"
16266 %}
16267 ins_encode %{
16268 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16269 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16270 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16271 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16272 %}
16273 ins_pipe(pipe_slow);
16274 %}
16275
16276 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16277 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16278 match(Set dst (CompressBits (LoadI mem) mask));
16279 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16280 format %{ "ldrs $tsrc, $mem\n\t"
16281 "ldrs $tmask, $mask\n\t"
16282 "bext $tdst, $tsrc, $tmask\n\t"
16283 "mov $dst, $tdst"
16284 %}
16285 ins_encode %{
16286 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16287 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16288 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16289 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16290 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16291 %}
16292 ins_pipe(pipe_slow);
16293 %}
16294
16295 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16296 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16297 match(Set dst (CompressBits src mask));
16298 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16299 format %{ "mov $tsrc, $src\n\t"
16300 "mov $tmask, $mask\n\t"
16301 "bext $tdst, $tsrc, $tmask\n\t"
16302 "mov $dst, $tdst"
16303 %}
16304 ins_encode %{
16305 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16306 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16307 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16308 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16309 %}
16310 ins_pipe(pipe_slow);
16311 %}
16312
16313 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16314 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16315 match(Set dst (CompressBits (LoadL mem) mask));
16316 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16317 format %{ "ldrd $tsrc, $mem\n\t"
16318 "ldrd $tmask, $mask\n\t"
16319 "bext $tdst, $tsrc, $tmask\n\t"
16320 "mov $dst, $tdst"
16321 %}
16322 ins_encode %{
16323 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16324 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16325 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16326 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16327 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16328 %}
16329 ins_pipe(pipe_slow);
16330 %}
16331
16332 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16333 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16334 match(Set dst (ExpandBits src mask));
16335 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16336 format %{ "mov $tsrc, $src\n\t"
16337 "mov $tmask, $mask\n\t"
16338 "bdep $tdst, $tsrc, $tmask\n\t"
16339 "mov $dst, $tdst"
16340 %}
16341 ins_encode %{
16342 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16343 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16344 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16345 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16346 %}
16347 ins_pipe(pipe_slow);
16348 %}
16349
16350 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16351 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16352 match(Set dst (ExpandBits (LoadI mem) mask));
16353 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16354 format %{ "ldrs $tsrc, $mem\n\t"
16355 "ldrs $tmask, $mask\n\t"
16356 "bdep $tdst, $tsrc, $tmask\n\t"
16357 "mov $dst, $tdst"
16358 %}
16359 ins_encode %{
16360 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16361 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16362 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16363 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16364 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16365 %}
16366 ins_pipe(pipe_slow);
16367 %}
16368
16369 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16370 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16371 match(Set dst (ExpandBits src mask));
16372 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16373 format %{ "mov $tsrc, $src\n\t"
16374 "mov $tmask, $mask\n\t"
16375 "bdep $tdst, $tsrc, $tmask\n\t"
16376 "mov $dst, $tdst"
16377 %}
16378 ins_encode %{
16379 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16380 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16381 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16382 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16383 %}
16384 ins_pipe(pipe_slow);
16385 %}
16386
16387
16388 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16389 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16390 match(Set dst (ExpandBits (LoadL mem) mask));
16391 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16392 format %{ "ldrd $tsrc, $mem\n\t"
16393 "ldrd $tmask, $mask\n\t"
16394 "bdep $tdst, $tsrc, $tmask\n\t"
16395 "mov $dst, $tdst"
16396 %}
16397 ins_encode %{
16398 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16399 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16400 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16401 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16402 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16403 %}
16404 ins_pipe(pipe_slow);
16405 %}
16406
16407 //----------------------------- Reinterpret ----------------------------------
16408 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16409 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16410 match(Set dst (ReinterpretHF2S src));
16411 format %{ "reinterpretHF2S $dst, $src" %}
16412 ins_encode %{
16413 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16414 %}
16415 ins_pipe(pipe_slow);
16416 %}
16417
16418 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16419 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16420 match(Set dst (ReinterpretS2HF src));
16421 format %{ "reinterpretS2HF $dst, $src" %}
16422 ins_encode %{
16423 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16424 %}
16425 ins_pipe(pipe_slow);
16426 %}
16427
16428 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16429 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16430 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16431 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16432 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16433 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16434 // can be omitted in this pattern, resulting in -
16435 // fcvt $dst, $src // Convert float to half-precision float
16436 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16437 %{
16438 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16439 format %{ "convF2HFAndS2HF $dst, $src" %}
16440 ins_encode %{
16441 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16442 %}
16443 ins_pipe(pipe_slow);
16444 %}
16445
16446 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16447 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16448 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16449 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16450 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16451 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16452 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16453 // resulting in -
16454 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16455 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16456 %{
16457 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16458 format %{ "convHF2SAndHF2F $dst, $src" %}
16459 ins_encode %{
16460 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16461 %}
16462 ins_pipe(pipe_slow);
16463 %}
16464
16465 // ============================================================================
16466 // This name is KNOWN by the ADLC and cannot be changed.
16467 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16468 // for this guy.
16469 instruct tlsLoadP(thread_RegP dst)
16470 %{
16471 match(Set dst (ThreadLocal));
16472
16473 ins_cost(0);
16474
16475 format %{ " -- \t// $dst=Thread::current(), empty" %}
16476
16477 size(0);
16478
16479 ins_encode( /*empty*/ );
16480
16481 ins_pipe(pipe_class_empty);
16482 %}
16483
16484 //----------PEEPHOLE RULES-----------------------------------------------------
16485 // These must follow all instruction definitions as they use the names
16486 // defined in the instructions definitions.
16487 //
16488 // peepmatch ( root_instr_name [preceding_instruction]* );
16489 //
16490 // peepconstraint %{
16491 // (instruction_number.operand_name relational_op instruction_number.operand_name
16492 // [, ...] );
16493 // // instruction numbers are zero-based using left to right order in peepmatch
16494 //
16495 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16496 // // provide an instruction_number.operand_name for each operand that appears
16497 // // in the replacement instruction's match rule
16498 //
16499 // ---------VM FLAGS---------------------------------------------------------
16500 //
16501 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16502 //
16503 // Each peephole rule is given an identifying number starting with zero and
16504 // increasing by one in the order seen by the parser. An individual peephole
16505 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16506 // on the command-line.
16507 //
16508 // ---------CURRENT LIMITATIONS----------------------------------------------
16509 //
16510 // Only match adjacent instructions in same basic block
16511 // Only equality constraints
16512 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16513 // Only one replacement instruction
16514 //
16515 // ---------EXAMPLE----------------------------------------------------------
16516 //
16517 // // pertinent parts of existing instructions in architecture description
16518 // instruct movI(iRegINoSp dst, iRegI src)
16519 // %{
16520 // match(Set dst (CopyI src));
16521 // %}
16522 //
16523 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16524 // %{
16525 // match(Set dst (AddI dst src));
16526 // effect(KILL cr);
16527 // %}
16528 //
16529 // // Change (inc mov) to lea
16530 // peephole %{
16531 // // increment preceded by register-register move
16532 // peepmatch ( incI_iReg movI );
16533 // // require that the destination register of the increment
16534 // // match the destination register of the move
16535 // peepconstraint ( 0.dst == 1.dst );
16536 // // construct a replacement instruction that sets
16537 // // the destination to ( move's source register + one )
16538 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16539 // %}
16540 //
16541
16542 // Implementation no longer uses movX instructions since
16543 // machine-independent system no longer uses CopyX nodes.
16544 //
16545 // peephole
16546 // %{
16547 // peepmatch (incI_iReg movI);
16548 // peepconstraint (0.dst == 1.dst);
16549 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16550 // %}
16551
16552 // peephole
16553 // %{
16554 // peepmatch (decI_iReg movI);
16555 // peepconstraint (0.dst == 1.dst);
16556 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16557 // %}
16558
16559 // peephole
16560 // %{
16561 // peepmatch (addI_iReg_imm movI);
16562 // peepconstraint (0.dst == 1.dst);
16563 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16564 // %}
16565
16566 // peephole
16567 // %{
16568 // peepmatch (incL_iReg movL);
16569 // peepconstraint (0.dst == 1.dst);
16570 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16571 // %}
16572
16573 // peephole
16574 // %{
16575 // peepmatch (decL_iReg movL);
16576 // peepconstraint (0.dst == 1.dst);
16577 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16578 // %}
16579
16580 // peephole
16581 // %{
16582 // peepmatch (addL_iReg_imm movL);
16583 // peepconstraint (0.dst == 1.dst);
16584 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16585 // %}
16586
16587 // peephole
16588 // %{
16589 // peepmatch (addP_iReg_imm movP);
16590 // peepconstraint (0.dst == 1.dst);
16591 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16592 // %}
16593
16594 // // Change load of spilled value to only a spill
16595 // instruct storeI(memory mem, iRegI src)
16596 // %{
16597 // match(Set mem (StoreI mem src));
16598 // %}
16599 //
16600 // instruct loadI(iRegINoSp dst, memory mem)
16601 // %{
16602 // match(Set dst (LoadI mem));
16603 // %}
16604 //
16605
16606 //----------SMARTSPILL RULES---------------------------------------------------
16607 // These must follow all instruction definitions as they use the names
16608 // defined in the instructions definitions.
16609
16610 // Local Variables:
16611 // mode: c++
16612 // End: