1 //
2 // Copyright (c) 2003, 2026, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return MacroAssembler::max_trampoline_stub_size();
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 5; // metadata; call dest; trampoline address; trampoline destination; trampoline_owner_metadata
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BoolTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else if (_entry_point == nullptr) {
1694 // See CallLeafNoFPIndirect
1695 return 1 * NativeInstruction::instruction_size;
1696 } else {
1697 return 6 * NativeInstruction::instruction_size;
1698 }
1699 }
1700
1701 //=============================================================================
1702
1703 #ifndef PRODUCT
1704 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1705 st->print("BREAKPOINT");
1706 }
1707 #endif
1708
1709 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1710 __ brk(0);
1711 }
1712
1713 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1714 return MachNode::size(ra_);
1715 }
1716
1717 //=============================================================================
1718
1719 #ifndef PRODUCT
1720 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1721 st->print("nop \t# %d bytes pad for loops and calls", _count);
1722 }
1723 #endif
1724
1725 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1726 for (int i = 0; i < _count; i++) {
1727 __ nop();
1728 }
1729 }
1730
1731 uint MachNopNode::size(PhaseRegAlloc*) const {
1732 return _count * NativeInstruction::instruction_size;
1733 }
1734
1735 //=============================================================================
1736 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1737
1738 int ConstantTable::calculate_table_base_offset() const {
1739 return 0; // absolute addressing, no offset
1740 }
1741
1742 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1743 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1744 ShouldNotReachHere();
1745 }
1746
1747 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1748 // Empty encoding
1749 }
1750
1751 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1752 return 0;
1753 }
1754
1755 #ifndef PRODUCT
1756 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1757 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1758 }
1759 #endif
1760
1761 #ifndef PRODUCT
1762 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1763 Compile* C = ra_->C;
1764
1765 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1766
1767 if (C->output()->need_stack_bang(framesize))
1768 st->print("# stack bang size=%d\n\t", framesize);
1769
1770 if (VM_Version::use_rop_protection()) {
1771 st->print("ldr zr, [lr]\n\t");
1772 st->print("paciaz\n\t");
1773 }
1774 if (framesize < ((1 << 9) + 2 * wordSize)) {
1775 st->print("sub sp, sp, #%d\n\t", framesize);
1776 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1777 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1778 } else {
1779 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1780 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1781 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1782 st->print("sub sp, sp, rscratch1");
1783 }
1784 if (C->stub_function() == nullptr) {
1785 st->print("\n\t");
1786 st->print("ldr rscratch1, [guard]\n\t");
1787 st->print("dmb ishld\n\t");
1788 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1789 st->print("cmp rscratch1, rscratch2\n\t");
1790 st->print("b.eq skip");
1791 st->print("\n\t");
1792 st->print("blr #nmethod_entry_barrier_stub\n\t");
1793 st->print("b skip\n\t");
1794 st->print("guard: int\n\t");
1795 st->print("\n\t");
1796 st->print("skip:\n\t");
1797 }
1798 }
1799 #endif
1800
1801 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1802 Compile* C = ra_->C;
1803
1804
1805 __ verified_entry(C, 0);
1806
1807 if (C->stub_function() == nullptr) {
1808 __ entry_barrier();
1809 }
1810
1811 if (!Compile::current()->output()->in_scratch_emit_size()) {
1812 __ bind(*_verified_entry);
1813 }
1814
1815 if (VerifyStackAtCalls) {
1816 Unimplemented();
1817 }
1818
1819 C->output()->set_frame_complete(__ offset());
1820
1821 if (C->has_mach_constant_base_node()) {
1822 // NOTE: We set the table base offset here because users might be
1823 // emitted before MachConstantBaseNode.
1824 ConstantTable& constant_table = C->output()->constant_table();
1825 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1826 }
1827 }
1828
1829 int MachPrologNode::reloc() const
1830 {
1831 return 0;
1832 }
1833
1834 //=============================================================================
1835
1836 #ifndef PRODUCT
1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1838 Compile* C = ra_->C;
1839 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1840
1841 st->print("# pop frame %d\n\t",framesize);
1842
1843 if (framesize == 0) {
1844 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1845 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1846 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1847 st->print("add sp, sp, #%d\n\t", framesize);
1848 } else {
1849 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1850 st->print("add sp, sp, rscratch1\n\t");
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 }
1853 if (VM_Version::use_rop_protection()) {
1854 st->print("autiaz\n\t");
1855 st->print("ldr zr, [lr]\n\t");
1856 }
1857
1858 if (do_polling() && C->is_method_compilation()) {
1859 st->print("# test polling word\n\t");
1860 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1861 st->print("cmp sp, rscratch1\n\t");
1862 st->print("bhi #slow_path");
1863 }
1864 }
1865 #endif
1866
1867 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1868 Compile* C = ra_->C;
1869 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1870
1871 __ remove_frame(framesize, C->needs_stack_repair());
1872
1873 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1874 __ reserved_stack_check();
1875 }
1876
1877 if (do_polling() && C->is_method_compilation()) {
1878 Label dummy_label;
1879 Label* code_stub = &dummy_label;
1880 if (!C->output()->in_scratch_emit_size()) {
1881 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1882 C->output()->add_stub(stub);
1883 code_stub = &stub->entry();
1884 }
1885 __ relocate(relocInfo::poll_return_type);
1886 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1887 }
1888 }
1889
1890 int MachEpilogNode::reloc() const {
1891 // Return number of relocatable values contained in this instruction.
1892 return 1; // 1 for polling page.
1893 }
1894
1895 const Pipeline * MachEpilogNode::pipeline() const {
1896 return MachNode::pipeline_class();
1897 }
1898
1899 //=============================================================================
1900
1901 static enum RC rc_class(OptoReg::Name reg) {
1902
1903 if (reg == OptoReg::Bad) {
1904 return rc_bad;
1905 }
1906
1907 // we have 32 int registers * 2 halves
1908 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1909
1910 if (reg < slots_of_int_registers) {
1911 return rc_int;
1912 }
1913
1914 // we have 32 float register * 8 halves
1915 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1916 if (reg < slots_of_int_registers + slots_of_float_registers) {
1917 return rc_float;
1918 }
1919
1920 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1921 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1922 return rc_predicate;
1923 }
1924
1925 // Between predicate regs & stack is the flags.
1926 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1927
1928 return rc_stack;
1929 }
1930
1931 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1932 Compile* C = ra_->C;
1933
1934 // Get registers to move.
1935 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1936 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1937 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1938 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1939
1940 enum RC src_hi_rc = rc_class(src_hi);
1941 enum RC src_lo_rc = rc_class(src_lo);
1942 enum RC dst_hi_rc = rc_class(dst_hi);
1943 enum RC dst_lo_rc = rc_class(dst_lo);
1944
1945 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1946
1947 if (src_hi != OptoReg::Bad && !bottom_type()->isa_pvectmask()) {
1948 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1949 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1950 "expected aligned-adjacent pairs");
1951 }
1952
1953 if (src_lo == dst_lo && src_hi == dst_hi) {
1954 return 0; // Self copy, no move.
1955 }
1956
1957 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1958 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1959 int src_offset = ra_->reg2offset(src_lo);
1960 int dst_offset = ra_->reg2offset(dst_lo);
1961
1962 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
1963 uint ireg = ideal_reg();
1964 DEBUG_ONLY(int algm = MIN2(RegMask::num_registers(ireg), (int)Matcher::stack_alignment_in_slots()) * VMRegImpl::stack_slot_size);
1965 assert((src_lo_rc != rc_stack) || is_aligned(src_offset, algm), "unaligned vector spill sp offset %d (src)", src_offset);
1966 assert((dst_lo_rc != rc_stack) || is_aligned(dst_offset, algm), "unaligned vector spill sp offset %d (dst)", dst_offset);
1967 if (ireg == Op_VecA && masm) {
1968 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1969 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1970 // stack->stack
1971 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1972 sve_vector_reg_size_in_bytes);
1973 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1974 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1975 sve_vector_reg_size_in_bytes);
1976 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1977 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1978 sve_vector_reg_size_in_bytes);
1979 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1980 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1981 as_FloatRegister(Matcher::_regEncode[src_lo]),
1982 as_FloatRegister(Matcher::_regEncode[src_lo]));
1983 } else {
1984 ShouldNotReachHere();
1985 }
1986 } else if (masm) {
1987 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1988 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1989 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1990 // stack->stack
1991 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1992 if (ireg == Op_VecD) {
1993 __ unspill(rscratch1, true, src_offset);
1994 __ spill(rscratch1, true, dst_offset);
1995 } else {
1996 __ spill_copy128(src_offset, dst_offset);
1997 }
1998 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1999 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2000 ireg == Op_VecD ? __ T8B : __ T16B,
2001 as_FloatRegister(Matcher::_regEncode[src_lo]));
2002 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2003 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2004 ireg == Op_VecD ? __ D : __ Q,
2005 ra_->reg2offset(dst_lo));
2006 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2007 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2008 ireg == Op_VecD ? __ D : __ Q,
2009 ra_->reg2offset(src_lo));
2010 } else {
2011 ShouldNotReachHere();
2012 }
2013 }
2014 } else if (masm) {
2015 switch (src_lo_rc) {
2016 case rc_int:
2017 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2018 if (is64) {
2019 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2020 as_Register(Matcher::_regEncode[src_lo]));
2021 } else {
2022 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2023 as_Register(Matcher::_regEncode[src_lo]));
2024 }
2025 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2026 if (is64) {
2027 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2028 as_Register(Matcher::_regEncode[src_lo]));
2029 } else {
2030 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2031 as_Register(Matcher::_regEncode[src_lo]));
2032 }
2033 } else { // gpr --> stack spill
2034 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2035 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2036 }
2037 break;
2038 case rc_float:
2039 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2040 if (is64) {
2041 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2042 as_FloatRegister(Matcher::_regEncode[src_lo]));
2043 } else {
2044 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2045 as_FloatRegister(Matcher::_regEncode[src_lo]));
2046 }
2047 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2048 if (is64) {
2049 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 as_FloatRegister(Matcher::_regEncode[src_lo]));
2051 } else {
2052 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2053 as_FloatRegister(Matcher::_regEncode[src_lo]));
2054 }
2055 } else { // fpr --> stack spill
2056 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2057 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2058 is64 ? __ D : __ S, dst_offset);
2059 }
2060 break;
2061 case rc_stack:
2062 if (dst_lo_rc == rc_int) { // stack --> gpr load
2063 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2064 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2065 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2066 is64 ? __ D : __ S, src_offset);
2067 } else if (dst_lo_rc == rc_predicate) {
2068 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2069 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2070 } else { // stack --> stack copy
2071 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2072 if (ideal_reg() == Op_RegVectMask) {
2073 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2074 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2075 } else {
2076 __ unspill(rscratch1, is64, src_offset);
2077 __ spill(rscratch1, is64, dst_offset);
2078 }
2079 }
2080 break;
2081 case rc_predicate:
2082 if (dst_lo_rc == rc_predicate) {
2083 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2084 } else if (dst_lo_rc == rc_stack) {
2085 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2086 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2087 } else {
2088 assert(false, "bad src and dst rc_class combination.");
2089 ShouldNotReachHere();
2090 }
2091 break;
2092 default:
2093 assert(false, "bad rc_class for spill");
2094 ShouldNotReachHere();
2095 }
2096 }
2097
2098 if (st) {
2099 st->print("spill ");
2100 if (src_lo_rc == rc_stack) {
2101 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2102 } else {
2103 st->print("%s -> ", Matcher::regName[src_lo]);
2104 }
2105 if (dst_lo_rc == rc_stack) {
2106 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2107 } else {
2108 st->print("%s", Matcher::regName[dst_lo]);
2109 }
2110 if (bottom_type()->isa_vect() && !bottom_type()->isa_pvectmask()) {
2111 int vsize = 0;
2112 switch (ideal_reg()) {
2113 case Op_VecD:
2114 vsize = 64;
2115 break;
2116 case Op_VecX:
2117 vsize = 128;
2118 break;
2119 case Op_VecA:
2120 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2121 break;
2122 default:
2123 assert(false, "bad register type for spill");
2124 ShouldNotReachHere();
2125 }
2126 st->print("\t# vector spill size = %d", vsize);
2127 } else if (ideal_reg() == Op_RegVectMask) {
2128 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2129 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2130 st->print("\t# predicate spill size = %d", vsize);
2131 } else {
2132 st->print("\t# spill size = %d", is64 ? 64 : 32);
2133 }
2134 }
2135
2136 return 0;
2137
2138 }
2139
2140 #ifndef PRODUCT
2141 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2142 if (!ra_)
2143 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2144 else
2145 implementation(nullptr, ra_, false, st);
2146 }
2147 #endif
2148
2149 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2150 implementation(masm, ra_, false, nullptr);
2151 }
2152
2153 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2154 return MachNode::size(ra_);
2155 }
2156
2157 //=============================================================================
2158
2159 #ifndef PRODUCT
2160 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2161 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2162 int reg = ra_->get_reg_first(this);
2163 st->print("add %s, rsp, #%d]\t# box lock",
2164 Matcher::regName[reg], offset);
2165 }
2166 #endif
2167
2168 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2169 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2170 int reg = ra_->get_encode(this);
2171
2172 // This add will handle any 24-bit signed offset. 24 bits allows an
2173 // 8 megabyte stack frame.
2174 __ add(as_Register(reg), sp, offset);
2175 }
2176
2177 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2178 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2179 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2180
2181 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2182 return NativeInstruction::instruction_size;
2183 } else {
2184 return 2 * NativeInstruction::instruction_size;
2185 }
2186 }
2187
2188 ///=============================================================================
2189 #ifndef PRODUCT
2190 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2191 {
2192 st->print_cr("# MachVEPNode");
2193 if (!_verified) {
2194 st->print_cr("\t load_class");
2195 } else {
2196 st->print_cr("\t unpack_inline_arg");
2197 }
2198 }
2199 #endif
2200
2201 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2202 {
2203 if (!_verified) {
2204 __ ic_check(1);
2205 } else {
2206 if (ra_->C->stub_function() == nullptr) {
2207 // Emit the entry barrier in a temporary frame before unpacking because
2208 // it can deopt, which would require packing the scalarized args again.
2209 __ verified_entry(ra_->C, 0);
2210 __ entry_barrier();
2211 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2212 __ remove_frame(framesize, false);
2213 }
2214 // Unpack inline type args passed as oop and then jump to
2215 // the verified entry point (skipping the unverified entry).
2216 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2217 // Emit code for verified entry and save increment for stack repair on return
2218 __ verified_entry(ra_->C, sp_inc);
2219 if (Compile::current()->output()->in_scratch_emit_size()) {
2220 Label dummy_verified_entry;
2221 __ b(dummy_verified_entry);
2222 } else {
2223 __ b(*_verified_entry);
2224 }
2225 }
2226 }
2227
2228 //=============================================================================
2229 #ifndef PRODUCT
2230 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2231 {
2232 st->print_cr("# MachUEPNode");
2233 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2234 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2235 st->print_cr("\tcmpw rscratch1, r10");
2236 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2237 }
2238 #endif
2239
2240 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2241 {
2242 __ ic_check(InteriorEntryAlignment);
2243 }
2244
2245 // REQUIRED EMIT CODE
2246
2247 //=============================================================================
2248
2249 // Emit deopt handler code.
2250 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2251 {
2252 // Note that the code buffer's insts_mark is always relative to insts.
2253 // That's why we must use the macroassembler to generate a handler.
2254 address base = __ start_a_stub(size_deopt_handler());
2255 if (base == nullptr) {
2256 ciEnv::current()->record_failure("CodeCache is full");
2257 return 0; // CodeBuffer::expand failed
2258 }
2259
2260 int offset = __ offset();
2261 Label start;
2262 __ bind(start);
2263 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2264
2265 int entry_offset = __ offset();
2266 __ b(start);
2267
2268 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2269 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2270 "out of bounds read in post-call NOP check");
2271 __ end_a_stub();
2272 return entry_offset;
2273 }
2274
2275 // REQUIRED MATCHER CODE
2276
2277 //=============================================================================
2278
2279 bool Matcher::match_rule_supported(int opcode) {
2280 if (!has_match_rule(opcode))
2281 return false;
2282
2283 switch (opcode) {
2284 case Op_OnSpinWait:
2285 return VM_Version::supports_on_spin_wait();
2286 case Op_CacheWB:
2287 case Op_CacheWBPreSync:
2288 case Op_CacheWBPostSync:
2289 if (!VM_Version::supports_data_cache_line_flush()) {
2290 return false;
2291 }
2292 break;
2293 case Op_ExpandBits:
2294 case Op_CompressBits:
2295 if (!VM_Version::supports_svebitperm()) {
2296 return false;
2297 }
2298 break;
2299 case Op_FmaF:
2300 case Op_FmaD:
2301 case Op_FmaVF:
2302 case Op_FmaVD:
2303 if (!UseFMA) {
2304 return false;
2305 }
2306 break;
2307 case Op_FmaHF:
2308 // UseFMA flag also needs to be checked along with FEAT_FP16
2309 if (!UseFMA || !is_feat_fp16_supported()) {
2310 return false;
2311 }
2312 break;
2313 case Op_AddHF:
2314 case Op_SubHF:
2315 case Op_MulHF:
2316 case Op_DivHF:
2317 case Op_MinHF:
2318 case Op_MaxHF:
2319 case Op_SqrtHF:
2320 // Half-precision floating point scalar operations require FEAT_FP16
2321 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2322 // features are supported.
2323 if (!is_feat_fp16_supported()) {
2324 return false;
2325 }
2326 break;
2327 }
2328
2329 return true; // Per default match rules are supported.
2330 }
2331
2332 const RegMask* Matcher::predicate_reg_mask(void) {
2333 return &_PR_REG_mask;
2334 }
2335
2336 bool Matcher::supports_vector_calling_convention(void) {
2337 return EnableVectorSupport;
2338 }
2339
2340 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2341 assert(EnableVectorSupport, "sanity");
2342 int lo = V0_num;
2343 int hi = V0_H_num;
2344 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2345 hi = V0_K_num;
2346 }
2347 return OptoRegPair(hi, lo);
2348 }
2349
2350 // Is this branch offset short enough that a short branch can be used?
2351 //
2352 // NOTE: If the platform does not provide any short branch variants, then
2353 // this method should return false for offset 0.
2354 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2355 // The passed offset is relative to address of the branch.
2356
2357 return (-32768 <= offset && offset < 32768);
2358 }
2359
2360 // Vector width in bytes.
2361 int Matcher::vector_width_in_bytes(BasicType bt) {
2362 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2363 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2364 // Minimum 2 values in vector
2365 if (size < 2*type2aelembytes(bt)) size = 0;
2366 // But never < 4
2367 if (size < 4) size = 0;
2368 return size;
2369 }
2370
2371 // Limits on vector size (number of elements) loaded into vector.
2372 int Matcher::max_vector_size(const BasicType bt) {
2373 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2374 }
2375
2376 int Matcher::min_vector_size(const BasicType bt) {
2377 // Usually, the shortest vector length supported by AArch64 ISA and
2378 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2379 // vectors in a few special cases.
2380 int size;
2381 switch(bt) {
2382 case T_BOOLEAN:
2383 // Load/store a vector mask with only 2 elements for vector types
2384 // such as "2I/2F/2L/2D".
2385 size = 2;
2386 break;
2387 case T_BYTE:
2388 // Generate a "4B" vector, to support vector cast between "8B/16B"
2389 // and "4S/4I/4L/4F/4D".
2390 size = 4;
2391 break;
2392 case T_SHORT:
2393 // Generate a "2S" vector, to support vector cast between "4S/8S"
2394 // and "2I/2L/2F/2D".
2395 size = 2;
2396 break;
2397 default:
2398 // Limit the min vector length to 64-bit.
2399 size = 8 / type2aelembytes(bt);
2400 // The number of elements in a vector should be at least 2.
2401 size = MAX2(size, 2);
2402 }
2403
2404 int max_size = max_vector_size(bt);
2405 return MIN2(size, max_size);
2406 }
2407
2408 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2409 return Matcher::max_vector_size(bt);
2410 }
2411
2412 // Actual max scalable vector register length.
2413 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2414 return Matcher::max_vector_size(bt);
2415 }
2416
2417 // Vector ideal reg.
2418 uint Matcher::vector_ideal_reg(int len) {
2419 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2420 return Op_VecA;
2421 }
2422 switch(len) {
2423 // For 16-bit/32-bit mask vector, reuse VecD.
2424 case 2:
2425 case 4:
2426 case 8: return Op_VecD;
2427 case 16: return Op_VecX;
2428 }
2429 ShouldNotReachHere();
2430 return 0;
2431 }
2432
2433 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2434 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2435 switch (ideal_reg) {
2436 case Op_VecA: return new vecAOper();
2437 case Op_VecD: return new vecDOper();
2438 case Op_VecX: return new vecXOper();
2439 }
2440 ShouldNotReachHere();
2441 return nullptr;
2442 }
2443
2444 bool Matcher::is_reg2reg_move(MachNode* m) {
2445 return false;
2446 }
2447
2448 bool Matcher::is_register_biasing_candidate(const MachNode* mdef, int oper_index) {
2449 return false;
2450 }
2451
2452 bool Matcher::is_generic_vector(MachOper* opnd) {
2453 return opnd->opcode() == VREG;
2454 }
2455
2456 #ifdef ASSERT
2457 // Return whether or not this register is ever used as an argument.
2458 bool Matcher::can_be_java_arg(int reg)
2459 {
2460 return
2461 reg == R0_num || reg == R0_H_num ||
2462 reg == R1_num || reg == R1_H_num ||
2463 reg == R2_num || reg == R2_H_num ||
2464 reg == R3_num || reg == R3_H_num ||
2465 reg == R4_num || reg == R4_H_num ||
2466 reg == R5_num || reg == R5_H_num ||
2467 reg == R6_num || reg == R6_H_num ||
2468 reg == R7_num || reg == R7_H_num ||
2469 reg == V0_num || reg == V0_H_num ||
2470 reg == V1_num || reg == V1_H_num ||
2471 reg == V2_num || reg == V2_H_num ||
2472 reg == V3_num || reg == V3_H_num ||
2473 reg == V4_num || reg == V4_H_num ||
2474 reg == V5_num || reg == V5_H_num ||
2475 reg == V6_num || reg == V6_H_num ||
2476 reg == V7_num || reg == V7_H_num;
2477 }
2478 #endif
2479
2480 uint Matcher::int_pressure_limit()
2481 {
2482 // JDK-8183543: When taking the number of available registers as int
2483 // register pressure threshold, the jtreg test:
2484 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2485 // failed due to C2 compilation failure with
2486 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2487 //
2488 // A derived pointer is live at CallNode and then is flagged by RA
2489 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2490 // derived pointers and lastly fail to spill after reaching maximum
2491 // number of iterations. Lowering the default pressure threshold to
2492 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2493 // a high register pressure area of the code so that split_DEF can
2494 // generate DefinitionSpillCopy for the derived pointer.
2495 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2496 if (!PreserveFramePointer) {
2497 // When PreserveFramePointer is off, frame pointer is allocatable,
2498 // but different from other SOC registers, it is excluded from
2499 // fatproj's mask because its save type is No-Save. Decrease 1 to
2500 // ensure high pressure at fatproj when PreserveFramePointer is off.
2501 // See check_pressure_at_fatproj().
2502 default_int_pressure_threshold--;
2503 }
2504 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2505 }
2506
2507 uint Matcher::float_pressure_limit()
2508 {
2509 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2510 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2511 }
2512
2513 const RegMask& Matcher::divI_proj_mask() {
2514 ShouldNotReachHere();
2515 return RegMask::EMPTY;
2516 }
2517
2518 // Register for MODI projection of divmodI.
2519 const RegMask& Matcher::modI_proj_mask() {
2520 ShouldNotReachHere();
2521 return RegMask::EMPTY;
2522 }
2523
2524 // Register for DIVL projection of divmodL.
2525 const RegMask& Matcher::divL_proj_mask() {
2526 ShouldNotReachHere();
2527 return RegMask::EMPTY;
2528 }
2529
2530 // Register for MODL projection of divmodL.
2531 const RegMask& Matcher::modL_proj_mask() {
2532 ShouldNotReachHere();
2533 return RegMask::EMPTY;
2534 }
2535
2536 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2537 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2538 Node* u = addp->fast_out(i);
2539 if (u->is_LoadStore()) {
2540 // On AArch64, LoadStoreNodes (i.e. compare and swap
2541 // instructions) only take register indirect as an operand, so
2542 // any attempt to use an AddPNode as an input to a LoadStoreNode
2543 // must fail.
2544 return false;
2545 }
2546 if (u->is_Mem()) {
2547 int opsize = u->as_Mem()->memory_size();
2548 assert(opsize > 0, "unexpected memory operand size");
2549 if (u->as_Mem()->memory_size() != (1<<shift)) {
2550 return false;
2551 }
2552 }
2553 }
2554 return true;
2555 }
2556
2557 // Convert BoolTest condition to Assembler condition.
2558 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2559 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2560 Assembler::Condition result;
2561 switch(cond) {
2562 case BoolTest::eq:
2563 result = Assembler::EQ; break;
2564 case BoolTest::ne:
2565 result = Assembler::NE; break;
2566 case BoolTest::le:
2567 result = Assembler::LE; break;
2568 case BoolTest::ge:
2569 result = Assembler::GE; break;
2570 case BoolTest::lt:
2571 result = Assembler::LT; break;
2572 case BoolTest::gt:
2573 result = Assembler::GT; break;
2574 case BoolTest::ule:
2575 result = Assembler::LS; break;
2576 case BoolTest::uge:
2577 result = Assembler::HS; break;
2578 case BoolTest::ult:
2579 result = Assembler::LO; break;
2580 case BoolTest::ugt:
2581 result = Assembler::HI; break;
2582 case BoolTest::overflow:
2583 result = Assembler::VS; break;
2584 case BoolTest::no_overflow:
2585 result = Assembler::VC; break;
2586 default:
2587 ShouldNotReachHere();
2588 return Assembler::Condition(-1);
2589 }
2590
2591 // Check conversion
2592 if (cond & BoolTest::unsigned_compare) {
2593 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2594 } else {
2595 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2596 }
2597
2598 return result;
2599 }
2600
2601 // Binary src (Replicate con)
2602 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2603 if (n == nullptr || m == nullptr) {
2604 return false;
2605 }
2606
2607 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2608 return false;
2609 }
2610
2611 Node* imm_node = m->in(1);
2612 if (!imm_node->is_Con()) {
2613 return false;
2614 }
2615
2616 const Type* t = imm_node->bottom_type();
2617 if (!(t->isa_int() || t->isa_long())) {
2618 return false;
2619 }
2620
2621 switch (n->Opcode()) {
2622 case Op_AndV:
2623 case Op_OrV:
2624 case Op_XorV: {
2625 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2626 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2627 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2628 }
2629 case Op_AddVB:
2630 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2631 case Op_AddVS:
2632 case Op_AddVI:
2633 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2634 case Op_AddVL:
2635 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2636 default:
2637 return false;
2638 }
2639 }
2640
2641 // (XorV src (Replicate m1))
2642 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2643 if (n != nullptr && m != nullptr) {
2644 return n->Opcode() == Op_XorV &&
2645 VectorNode::is_all_ones_vector(m);
2646 }
2647 return false;
2648 }
2649
2650 // Returns true if (n, m) matches "(XorVMask vm2 (MaskAll m1))" and that XorVMask
2651 // is used only by an AndVMask. In that case, cloning m (the MaskAll) lets the
2652 // matcher avoid sharing the MaskAll node and subsume the pattern into rule:
2653 // "(AndVMask vm1 (XorVMask vm2 (MaskAll m1)))".
2654 //
2655 // Limitation: the "andNot" rule still cannot be matched if "m" has other
2656 // uses outside this pattern.
2657 static bool is_vector_mask_not_operand_in_andnot_pattern(Node* n, Node* m) {
2658 if (n == nullptr || m == nullptr) {
2659 return false;
2660 }
2661
2662 if (VectorNode::is_all_ones_vector(m) &&
2663 n->Opcode() == Op_XorVMask &&
2664 n->outcnt() == 1 &&
2665 n->unique_out()->Opcode() == Op_AndVMask) {
2666 // If another input of the AndVMask is also a mask-not pattern that would
2667 // qualify for the `maskAll` cloning, do not clone the "maskAll" here,
2668 // because the match rule can only consume one such pattern.
2669 Node* use = n->unique_out();
2670 Node* other_input = use->in(1) == n ? use->in(2) : use->in(1);
2671 return !VectorNode::is_vectormask_bitwise_not_pattern(other_input);
2672 }
2673 return false;
2674 }
2675
2676 // Should the matcher clone input 'm' of node 'n'?
2677 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2678 if (is_vshift_con_pattern(n, m) ||
2679 is_vector_bitwise_not_pattern(n, m) ||
2680 is_vector_mask_not_operand_in_andnot_pattern(n, m) ||
2681 is_valid_sve_arith_imm_pattern(n, m) ||
2682 is_encode_and_store_pattern(n, m)) {
2683 mstack.push(m, Visit);
2684 return true;
2685 }
2686 return false;
2687 }
2688
2689 // Should the Matcher clone shifts on addressing modes, expecting them
2690 // to be subsumed into complex addressing expressions or compute them
2691 // into registers?
2692 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2693
2694 // Loads and stores with indirect memory input (e.g., volatile loads and
2695 // stores) do not subsume the input into complex addressing expressions. If
2696 // the addressing expression is input to at least one such load or store, do
2697 // not clone the addressing expression. Query needs_acquiring_load and
2698 // needs_releasing_store as a proxy for indirect memory input, as it is not
2699 // possible to directly query for indirect memory input at this stage.
2700 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2701 Node* n = m->fast_out(i);
2702 if (n->is_Load() && needs_acquiring_load(n)) {
2703 return false;
2704 }
2705 if (n->is_Store() && needs_releasing_store(n)) {
2706 return false;
2707 }
2708 }
2709
2710 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2711 return true;
2712 }
2713
2714 Node *off = m->in(AddPNode::Offset);
2715 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2716 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2717 // Are there other uses besides address expressions?
2718 !is_visited(off)) {
2719 address_visited.set(off->_idx); // Flag as address_visited
2720 mstack.push(off->in(2), Visit);
2721 Node *conv = off->in(1);
2722 if (conv->Opcode() == Op_ConvI2L &&
2723 // Are there other uses besides address expressions?
2724 !is_visited(conv)) {
2725 address_visited.set(conv->_idx); // Flag as address_visited
2726 mstack.push(conv->in(1), Pre_Visit);
2727 } else {
2728 mstack.push(conv, Pre_Visit);
2729 }
2730 address_visited.test_set(m->_idx); // Flag as address_visited
2731 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2732 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2733 return true;
2734 } else if (off->Opcode() == Op_ConvI2L &&
2735 // Are there other uses besides address expressions?
2736 !is_visited(off)) {
2737 address_visited.test_set(m->_idx); // Flag as address_visited
2738 address_visited.set(off->_idx); // Flag as address_visited
2739 mstack.push(off->in(1), Pre_Visit);
2740 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2741 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2742 return true;
2743 }
2744 return false;
2745 }
2746
2747 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2748 { \
2749 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2750 guarantee(DISP == 0, "mode not permitted for volatile"); \
2751 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2752 __ INSN(REG, as_Register(BASE)); \
2753 }
2754
2755
2756 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2757 {
2758 Address::extend scale;
2759
2760 // Hooboy, this is fugly. We need a way to communicate to the
2761 // encoder that the index needs to be sign extended, so we have to
2762 // enumerate all the cases.
2763 switch (opcode) {
2764 case INDINDEXSCALEDI2L:
2765 case INDINDEXSCALEDI2LN:
2766 case INDINDEXI2L:
2767 case INDINDEXI2LN:
2768 scale = Address::sxtw(size);
2769 break;
2770 default:
2771 scale = Address::lsl(size);
2772 }
2773
2774 if (index == -1) {
2775 return Address(base, disp);
2776 } else {
2777 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2778 return Address(base, as_Register(index), scale);
2779 }
2780 }
2781
2782
2783 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2784 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2785 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2786 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2787 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2788
2789 // Used for all non-volatile memory accesses. The use of
2790 // $mem->opcode() to discover whether this pattern uses sign-extended
2791 // offsets is something of a kludge.
2792 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2793 Register reg, int opcode,
2794 Register base, int index, int scale, int disp,
2795 int size_in_memory)
2796 {
2797 Address addr = mem2address(opcode, base, index, scale, disp);
2798 if (addr.getMode() == Address::base_plus_offset) {
2799 /* Fix up any out-of-range offsets. */
2800 assert_different_registers(rscratch1, base);
2801 assert_different_registers(rscratch1, reg);
2802 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2803 }
2804 (masm->*insn)(reg, addr);
2805 }
2806
2807 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2808 FloatRegister reg, int opcode,
2809 Register base, int index, int size, int disp,
2810 int size_in_memory)
2811 {
2812 Address::extend scale;
2813
2814 switch (opcode) {
2815 case INDINDEXSCALEDI2L:
2816 case INDINDEXSCALEDI2LN:
2817 scale = Address::sxtw(size);
2818 break;
2819 default:
2820 scale = Address::lsl(size);
2821 }
2822
2823 if (index == -1) {
2824 // Fix up any out-of-range offsets.
2825 assert_different_registers(rscratch1, base);
2826 Address addr = Address(base, disp);
2827 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2828 (masm->*insn)(reg, addr);
2829 } else {
2830 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2831 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2832 }
2833 }
2834
2835 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2836 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2837 int opcode, Register base, int index, int size, int disp)
2838 {
2839 if (index == -1) {
2840 (masm->*insn)(reg, T, Address(base, disp));
2841 } else {
2842 assert(disp == 0, "unsupported address mode");
2843 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2844 }
2845 }
2846
2847 %}
2848
2849
2850
2851 //----------ENCODING BLOCK-----------------------------------------------------
2852 // This block specifies the encoding classes used by the compiler to
2853 // output byte streams. Encoding classes are parameterized macros
2854 // used by Machine Instruction Nodes in order to generate the bit
2855 // encoding of the instruction. Operands specify their base encoding
2856 // interface with the interface keyword. There are currently
2857 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2858 // COND_INTER. REG_INTER causes an operand to generate a function
2859 // which returns its register number when queried. CONST_INTER causes
2860 // an operand to generate a function which returns the value of the
2861 // constant when queried. MEMORY_INTER causes an operand to generate
2862 // four functions which return the Base Register, the Index Register,
2863 // the Scale Value, and the Offset Value of the operand when queried.
2864 // COND_INTER causes an operand to generate six functions which return
2865 // the encoding code (ie - encoding bits for the instruction)
2866 // associated with each basic boolean condition for a conditional
2867 // instruction.
2868 //
2869 // Instructions specify two basic values for encoding. Again, a
2870 // function is available to check if the constant displacement is an
2871 // oop. They use the ins_encode keyword to specify their encoding
2872 // classes (which must be a sequence of enc_class names, and their
2873 // parameters, specified in the encoding block), and they use the
2874 // opcode keyword to specify, in order, their primary, secondary, and
2875 // tertiary opcode. Only the opcode sections which a particular
2876 // instruction needs for encoding need to be specified.
2877 encode %{
2878 // Build emit functions for each basic byte or larger field in the
2879 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2880 // from C++ code in the enc_class source block. Emit functions will
2881 // live in the main source block for now. In future, we can
2882 // generalize this by adding a syntax that specifies the sizes of
2883 // fields in an order, so that the adlc can build the emit functions
2884 // automagically
2885
2886 // catch all for unimplemented encodings
2887 enc_class enc_unimplemented %{
2888 __ unimplemented("C2 catch all");
2889 %}
2890
2891 // BEGIN Non-volatile memory access
2892
2893 // This encoding class is generated automatically from ad_encode.m4.
2894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2895 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2896 Register dst_reg = as_Register($dst$$reg);
2897 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2898 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2899 %}
2900
2901 // This encoding class is generated automatically from ad_encode.m4.
2902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2903 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2904 Register dst_reg = as_Register($dst$$reg);
2905 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2906 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2907 %}
2908
2909 // This encoding class is generated automatically from ad_encode.m4.
2910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2911 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2912 Register dst_reg = as_Register($dst$$reg);
2913 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2914 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2915 %}
2916
2917 // This encoding class is generated automatically from ad_encode.m4.
2918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2919 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2920 Register dst_reg = as_Register($dst$$reg);
2921 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2922 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2923 %}
2924
2925 // This encoding class is generated automatically from ad_encode.m4.
2926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2927 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2928 Register dst_reg = as_Register($dst$$reg);
2929 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2931 %}
2932
2933 // This encoding class is generated automatically from ad_encode.m4.
2934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2935 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2936 Register dst_reg = as_Register($dst$$reg);
2937 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2944 Register dst_reg = as_Register($dst$$reg);
2945 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2946 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2947 %}
2948
2949 // This encoding class is generated automatically from ad_encode.m4.
2950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2951 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2952 Register dst_reg = as_Register($dst$$reg);
2953 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2954 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2955 %}
2956
2957 // This encoding class is generated automatically from ad_encode.m4.
2958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2959 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2960 Register dst_reg = as_Register($dst$$reg);
2961 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2962 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2963 %}
2964
2965 // This encoding class is generated automatically from ad_encode.m4.
2966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2967 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2968 Register dst_reg = as_Register($dst$$reg);
2969 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2970 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2971 %}
2972
2973 // This encoding class is generated automatically from ad_encode.m4.
2974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2975 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2976 Register dst_reg = as_Register($dst$$reg);
2977 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2978 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2979 %}
2980
2981 // This encoding class is generated automatically from ad_encode.m4.
2982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2983 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2984 Register dst_reg = as_Register($dst$$reg);
2985 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2986 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2987 %}
2988
2989 // This encoding class is generated automatically from ad_encode.m4.
2990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2991 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2992 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2993 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2994 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2995 %}
2996
2997 // This encoding class is generated automatically from ad_encode.m4.
2998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2999 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
3000 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3001 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
3002 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3003 %}
3004
3005 // This encoding class is generated automatically from ad_encode.m4.
3006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3007 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3008 Register src_reg = as_Register($src$$reg);
3009 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3010 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3011 %}
3012
3013 // This encoding class is generated automatically from ad_encode.m4.
3014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3015 enc_class aarch64_enc_strb0(memory1 mem) %{
3016 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3017 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3018 %}
3019
3020 // This encoding class is generated automatically from ad_encode.m4.
3021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3022 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3023 Register src_reg = as_Register($src$$reg);
3024 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3025 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3026 %}
3027
3028 // This encoding class is generated automatically from ad_encode.m4.
3029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3030 enc_class aarch64_enc_strh0(memory2 mem) %{
3031 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3032 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3033 %}
3034
3035 // This encoding class is generated automatically from ad_encode.m4.
3036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3037 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3038 Register src_reg = as_Register($src$$reg);
3039 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3040 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3041 %}
3042
3043 // This encoding class is generated automatically from ad_encode.m4.
3044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3045 enc_class aarch64_enc_strw0(memory4 mem) %{
3046 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3047 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3048 %}
3049
3050 // This encoding class is generated automatically from ad_encode.m4.
3051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3052 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3053 Register src_reg = as_Register($src$$reg);
3054 // we sometimes get asked to store the stack pointer into the
3055 // current thread -- we cannot do that directly on AArch64
3056 if (src_reg == r31_sp) {
3057 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3058 __ mov(rscratch2, sp);
3059 src_reg = rscratch2;
3060 }
3061 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3062 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3063 %}
3064
3065 // This encoding class is generated automatically from ad_encode.m4.
3066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3067 enc_class aarch64_enc_str0(memory8 mem) %{
3068 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3069 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3070 %}
3071
3072 // This encoding class is generated automatically from ad_encode.m4.
3073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3074 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3075 FloatRegister src_reg = as_FloatRegister($src$$reg);
3076 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3077 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3078 %}
3079
3080 // This encoding class is generated automatically from ad_encode.m4.
3081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3082 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3083 FloatRegister src_reg = as_FloatRegister($src$$reg);
3084 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3085 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3086 %}
3087
3088 // This encoding class is generated automatically from ad_encode.m4.
3089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3090 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3091 __ membar(Assembler::StoreStore);
3092 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3093 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3094 %}
3095
3096 // END Non-volatile memory access
3097
3098 // Vector loads and stores
3099 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3100 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3101 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3102 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3103 %}
3104
3105 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3106 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3107 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3108 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3109 %}
3110
3111 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3112 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3113 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3114 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3115 %}
3116
3117 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3118 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3119 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3120 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3121 %}
3122
3123 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3124 FloatRegister src_reg = as_FloatRegister($src$$reg);
3125 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3126 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3127 %}
3128
3129 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3130 FloatRegister src_reg = as_FloatRegister($src$$reg);
3131 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3132 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3133 %}
3134
3135 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3136 FloatRegister src_reg = as_FloatRegister($src$$reg);
3137 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3138 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3139 %}
3140
3141 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3142 FloatRegister src_reg = as_FloatRegister($src$$reg);
3143 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3144 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3145 %}
3146
3147 // volatile loads and stores
3148
3149 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3150 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3151 rscratch1, stlrb);
3152 %}
3153
3154 enc_class aarch64_enc_stlrb0(memory mem) %{
3155 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3156 rscratch1, stlrb);
3157 %}
3158
3159 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3160 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3161 rscratch1, stlrh);
3162 %}
3163
3164 enc_class aarch64_enc_stlrh0(memory mem) %{
3165 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, stlrh);
3167 %}
3168
3169 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3170 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, stlrw);
3172 %}
3173
3174 enc_class aarch64_enc_stlrw0(memory mem) %{
3175 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3176 rscratch1, stlrw);
3177 %}
3178
3179 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3180 Register dst_reg = as_Register($dst$$reg);
3181 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3182 rscratch1, ldarb);
3183 __ sxtbw(dst_reg, dst_reg);
3184 %}
3185
3186 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3187 Register dst_reg = as_Register($dst$$reg);
3188 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, ldarb);
3190 __ sxtb(dst_reg, dst_reg);
3191 %}
3192
3193 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3194 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3195 rscratch1, ldarb);
3196 %}
3197
3198 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3199 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3200 rscratch1, ldarb);
3201 %}
3202
3203 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3204 Register dst_reg = as_Register($dst$$reg);
3205 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3206 rscratch1, ldarh);
3207 __ sxthw(dst_reg, dst_reg);
3208 %}
3209
3210 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3211 Register dst_reg = as_Register($dst$$reg);
3212 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3213 rscratch1, ldarh);
3214 __ sxth(dst_reg, dst_reg);
3215 %}
3216
3217 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3218 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3219 rscratch1, ldarh);
3220 %}
3221
3222 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3223 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3224 rscratch1, ldarh);
3225 %}
3226
3227 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3228 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3229 rscratch1, ldarw);
3230 %}
3231
3232 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3233 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3234 rscratch1, ldarw);
3235 %}
3236
3237 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3238 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3239 rscratch1, ldar);
3240 %}
3241
3242 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3243 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3244 rscratch1, ldarw);
3245 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3246 %}
3247
3248 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3249 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3250 rscratch1, ldar);
3251 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3252 %}
3253
3254 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3255 Register src_reg = as_Register($src$$reg);
3256 // we sometimes get asked to store the stack pointer into the
3257 // current thread -- we cannot do that directly on AArch64
3258 if (src_reg == r31_sp) {
3259 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3260 __ mov(rscratch2, sp);
3261 src_reg = rscratch2;
3262 }
3263 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3264 rscratch1, stlr);
3265 %}
3266
3267 enc_class aarch64_enc_stlr0(memory mem) %{
3268 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3269 rscratch1, stlr);
3270 %}
3271
3272 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3273 {
3274 FloatRegister src_reg = as_FloatRegister($src$$reg);
3275 __ fmovs(rscratch2, src_reg);
3276 }
3277 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3278 rscratch1, stlrw);
3279 %}
3280
3281 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3282 {
3283 FloatRegister src_reg = as_FloatRegister($src$$reg);
3284 __ fmovd(rscratch2, src_reg);
3285 }
3286 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3287 rscratch1, stlr);
3288 %}
3289
3290 // synchronized read/update encodings
3291
3292 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3293 Register dst_reg = as_Register($dst$$reg);
3294 Register base = as_Register($mem$$base);
3295 int index = $mem$$index;
3296 int scale = $mem$$scale;
3297 int disp = $mem$$disp;
3298 if (index == -1) {
3299 if (disp != 0) {
3300 __ lea(rscratch1, Address(base, disp));
3301 __ ldaxr(dst_reg, rscratch1);
3302 } else {
3303 // TODO
3304 // should we ever get anything other than this case?
3305 __ ldaxr(dst_reg, base);
3306 }
3307 } else {
3308 Register index_reg = as_Register(index);
3309 if (disp == 0) {
3310 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3311 __ ldaxr(dst_reg, rscratch1);
3312 } else {
3313 __ lea(rscratch1, Address(base, disp));
3314 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3315 __ ldaxr(dst_reg, rscratch1);
3316 }
3317 }
3318 %}
3319
3320 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3321 Register src_reg = as_Register($src$$reg);
3322 Register base = as_Register($mem$$base);
3323 int index = $mem$$index;
3324 int scale = $mem$$scale;
3325 int disp = $mem$$disp;
3326 if (index == -1) {
3327 if (disp != 0) {
3328 __ lea(rscratch2, Address(base, disp));
3329 __ stlxr(rscratch1, src_reg, rscratch2);
3330 } else {
3331 // TODO
3332 // should we ever get anything other than this case?
3333 __ stlxr(rscratch1, src_reg, base);
3334 }
3335 } else {
3336 Register index_reg = as_Register(index);
3337 if (disp == 0) {
3338 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3339 __ stlxr(rscratch1, src_reg, rscratch2);
3340 } else {
3341 __ lea(rscratch2, Address(base, disp));
3342 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3343 __ stlxr(rscratch1, src_reg, rscratch2);
3344 }
3345 }
3346 __ cmpw(rscratch1, zr);
3347 %}
3348
3349 // prefetch encodings
3350
3351 enc_class aarch64_enc_prefetchw(memory mem) %{
3352 Register base = as_Register($mem$$base);
3353 int index = $mem$$index;
3354 int scale = $mem$$scale;
3355 int disp = $mem$$disp;
3356 if (index == -1) {
3357 // Fix up any out-of-range offsets.
3358 assert_different_registers(rscratch1, base);
3359 Address addr = Address(base, disp);
3360 addr = __ legitimize_address(addr, 8, rscratch1);
3361 __ prfm(addr, PSTL1KEEP);
3362 } else {
3363 Register index_reg = as_Register(index);
3364 if (disp == 0) {
3365 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3366 } else {
3367 __ lea(rscratch1, Address(base, disp));
3368 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3369 }
3370 }
3371 %}
3372
3373 // mov encodings
3374
3375 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3376 uint32_t con = (uint32_t)$src$$constant;
3377 Register dst_reg = as_Register($dst$$reg);
3378 if (con == 0) {
3379 __ movw(dst_reg, zr);
3380 } else {
3381 __ movw(dst_reg, con);
3382 }
3383 %}
3384
3385 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3386 Register dst_reg = as_Register($dst$$reg);
3387 uint64_t con = (uint64_t)$src$$constant;
3388 if (con == 0) {
3389 __ mov(dst_reg, zr);
3390 } else {
3391 __ mov(dst_reg, con);
3392 }
3393 %}
3394
3395 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3396 Register dst_reg = as_Register($dst$$reg);
3397 address con = (address)$src$$constant;
3398 if (con == nullptr || con == (address)1) {
3399 ShouldNotReachHere();
3400 } else {
3401 relocInfo::relocType rtype = $src->constant_reloc();
3402 if (rtype == relocInfo::oop_type) {
3403 __ movoop(dst_reg, (jobject)con);
3404 } else if (rtype == relocInfo::metadata_type) {
3405 __ mov_metadata(dst_reg, (Metadata*)con);
3406 } else {
3407 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3408 // load fake address constants using a normal move
3409 if (! __ is_valid_AArch64_address(con) ||
3410 con < (address)(uintptr_t)os::vm_page_size() ||
3411 rtype == relocInfo::none) {
3412 __ mov(dst_reg, con);
3413 } else {
3414 // use shorter adrp/add sequence for external_word relocation
3415 uint64_t offset;
3416 __ adrp(dst_reg, Address(con, rtype), offset);
3417 __ add(dst_reg, dst_reg, offset);
3418 }
3419 }
3420 }
3421 %}
3422
3423 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3424 Register dst_reg = as_Register($dst$$reg);
3425 __ mov(dst_reg, zr);
3426 %}
3427
3428 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3429 Register dst_reg = as_Register($dst$$reg);
3430 __ mov(dst_reg, (uint64_t)1);
3431 %}
3432
3433 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3434 Register dst_reg = as_Register($dst$$reg);
3435 address con = (address)$src$$constant;
3436 if (con == nullptr) {
3437 ShouldNotReachHere();
3438 } else {
3439 relocInfo::relocType rtype = $src->constant_reloc();
3440 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3441 __ set_narrow_oop(dst_reg, (jobject)con);
3442 }
3443 %}
3444
3445 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3446 Register dst_reg = as_Register($dst$$reg);
3447 __ mov(dst_reg, zr);
3448 %}
3449
3450 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3451 Register dst_reg = as_Register($dst$$reg);
3452 address con = (address)$src$$constant;
3453 if (con == nullptr) {
3454 ShouldNotReachHere();
3455 } else {
3456 relocInfo::relocType rtype = $src->constant_reloc();
3457 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3458 __ set_narrow_klass(dst_reg, (Klass *)con);
3459 }
3460 %}
3461
3462 // arithmetic encodings
3463
3464 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3465 Register dst_reg = as_Register($dst$$reg);
3466 Register src_reg = as_Register($src1$$reg);
3467 int32_t con = (int32_t)$src2$$constant;
3468 // add has primary == 0, subtract has primary == 1
3469 if ($primary) { con = -con; }
3470 if (con < 0) {
3471 __ subw(dst_reg, src_reg, -con);
3472 } else {
3473 __ addw(dst_reg, src_reg, con);
3474 }
3475 %}
3476
3477 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3478 Register dst_reg = as_Register($dst$$reg);
3479 Register src_reg = as_Register($src1$$reg);
3480 int32_t con = (int32_t)$src2$$constant;
3481 // add has primary == 0, subtract has primary == 1
3482 if ($primary) { con = -con; }
3483 if (con < 0) {
3484 __ sub(dst_reg, src_reg, -con);
3485 } else {
3486 __ add(dst_reg, src_reg, con);
3487 }
3488 %}
3489
3490 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3491 Register dst_reg = as_Register($dst$$reg);
3492 Register src1_reg = as_Register($src1$$reg);
3493 Register src2_reg = as_Register($src2$$reg);
3494 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3495 %}
3496
3497 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3498 Register dst_reg = as_Register($dst$$reg);
3499 Register src1_reg = as_Register($src1$$reg);
3500 Register src2_reg = as_Register($src2$$reg);
3501 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3502 %}
3503
3504 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3505 Register dst_reg = as_Register($dst$$reg);
3506 Register src1_reg = as_Register($src1$$reg);
3507 Register src2_reg = as_Register($src2$$reg);
3508 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3509 %}
3510
3511 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3512 Register dst_reg = as_Register($dst$$reg);
3513 Register src1_reg = as_Register($src1$$reg);
3514 Register src2_reg = as_Register($src2$$reg);
3515 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3516 %}
3517
3518 // compare instruction encodings
3519
3520 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3521 Register reg1 = as_Register($src1$$reg);
3522 Register reg2 = as_Register($src2$$reg);
3523 __ cmpw(reg1, reg2);
3524 %}
3525
3526 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3527 Register reg = as_Register($src1$$reg);
3528 int32_t val = $src2$$constant;
3529 if (val >= 0) {
3530 __ subsw(zr, reg, val);
3531 } else {
3532 __ addsw(zr, reg, -val);
3533 }
3534 %}
3535
3536 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3537 Register reg1 = as_Register($src1$$reg);
3538 uint32_t val = (uint32_t)$src2$$constant;
3539 __ movw(rscratch1, val);
3540 __ cmpw(reg1, rscratch1);
3541 %}
3542
3543 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3544 Register reg1 = as_Register($src1$$reg);
3545 Register reg2 = as_Register($src2$$reg);
3546 __ cmp(reg1, reg2);
3547 %}
3548
3549 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3550 Register reg = as_Register($src1$$reg);
3551 int64_t val = $src2$$constant;
3552 if (val >= 0) {
3553 __ subs(zr, reg, val);
3554 } else if (val != -val) {
3555 __ adds(zr, reg, -val);
3556 } else {
3557 // aargh, Long.MIN_VALUE is a special case
3558 __ orr(rscratch1, zr, (uint64_t)val);
3559 __ subs(zr, reg, rscratch1);
3560 }
3561 %}
3562
3563 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3564 Register reg1 = as_Register($src1$$reg);
3565 uint64_t val = (uint64_t)$src2$$constant;
3566 __ mov(rscratch1, val);
3567 __ cmp(reg1, rscratch1);
3568 %}
3569
3570 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3571 Register reg1 = as_Register($src1$$reg);
3572 Register reg2 = as_Register($src2$$reg);
3573 __ cmp(reg1, reg2);
3574 %}
3575
3576 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3577 Register reg1 = as_Register($src1$$reg);
3578 Register reg2 = as_Register($src2$$reg);
3579 __ cmpw(reg1, reg2);
3580 %}
3581
3582 enc_class aarch64_enc_testp(iRegP src) %{
3583 Register reg = as_Register($src$$reg);
3584 __ cmp(reg, zr);
3585 %}
3586
3587 enc_class aarch64_enc_testn(iRegN src) %{
3588 Register reg = as_Register($src$$reg);
3589 __ cmpw(reg, zr);
3590 %}
3591
3592 enc_class aarch64_enc_b(label lbl) %{
3593 Label *L = $lbl$$label;
3594 __ b(*L);
3595 %}
3596
3597 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3598 Label *L = $lbl$$label;
3599 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3600 %}
3601
3602 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3603 Label *L = $lbl$$label;
3604 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3605 %}
3606
3607 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3608 %{
3609 Register sub_reg = as_Register($sub$$reg);
3610 Register super_reg = as_Register($super$$reg);
3611 Register temp_reg = as_Register($temp$$reg);
3612 Register result_reg = as_Register($result$$reg);
3613
3614 Label miss;
3615 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3616 nullptr, &miss,
3617 /*set_cond_codes:*/ true);
3618 if ($primary) {
3619 __ mov(result_reg, zr);
3620 }
3621 __ bind(miss);
3622 %}
3623
3624 enc_class aarch64_enc_java_static_call(method meth) %{
3625 address addr = (address)$meth$$method;
3626 address call;
3627 if (!_method) {
3628 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3629 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3630 if (call == nullptr) {
3631 ciEnv::current()->record_failure("CodeCache is full");
3632 return;
3633 }
3634 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3635 // The NOP here is purely to ensure that eliding a call to
3636 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3637 __ nop();
3638 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3639 } else {
3640 int method_index = resolved_method_index(masm);
3641 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3642 : static_call_Relocation::spec(method_index);
3643 call = __ trampoline_call(Address(addr, rspec));
3644 if (call == nullptr) {
3645 ciEnv::current()->record_failure("CodeCache is full");
3646 return;
3647 }
3648 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3649 // Calls of the same statically bound method can share
3650 // a stub to the interpreter.
3651 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3652 } else {
3653 // Emit stub for static call
3654 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3655 if (stub == nullptr) {
3656 ciEnv::current()->record_failure("CodeCache is full");
3657 return;
3658 }
3659 }
3660 }
3661
3662 __ post_call_nop();
3663
3664 // Only non uncommon_trap calls need to reinitialize ptrue.
3665 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3666 __ reinitialize_ptrue();
3667 }
3668 %}
3669
3670 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3671 int method_index = resolved_method_index(masm);
3672 address call = __ ic_call((address)$meth$$method, method_index);
3673 if (call == nullptr) {
3674 ciEnv::current()->record_failure("CodeCache is full");
3675 return;
3676 }
3677 __ post_call_nop();
3678 if (Compile::current()->max_vector_size() > 0) {
3679 __ reinitialize_ptrue();
3680 }
3681 %}
3682
3683 enc_class aarch64_enc_call_epilog() %{
3684 if (VerifyStackAtCalls) {
3685 // Check that stack depth is unchanged: find majik cookie on stack
3686 __ call_Unimplemented();
3687 }
3688 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
3689 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3690 // Search for the corresponding projection, get the register and emit code that initialized it.
3691 uint con = (tf()->range_cc()->cnt() - 1);
3692 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3693 ProjNode* proj = fast_out(i)->as_Proj();
3694 if (proj->_con == con) {
3695 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3696 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3697 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3698 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3699 __ cmp(r0, zr);
3700 __ cset(toReg, Assembler::NE);
3701 if (reg->is_stack()) {
3702 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3703 __ str(toReg, Address(sp, st_off));
3704 }
3705 break;
3706 }
3707 }
3708 if (return_value_is_used()) {
3709 // An inline type is returned as fields in multiple registers.
3710 // R0 either contains an oop if the inline type is buffered or a pointer
3711 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3712 // if the lowest bit is set to allow C2 to use the oop after null checking.
3713 // r0 &= (r0 & 1) - 1
3714 __ andr(rscratch1, r0, 0x1);
3715 __ sub(rscratch1, rscratch1, 0x1);
3716 __ andr(r0, r0, rscratch1);
3717 }
3718 }
3719 %}
3720
3721 enc_class aarch64_enc_java_to_runtime(method meth) %{
3722 // some calls to generated routines (arraycopy code) are scheduled
3723 // by C2 as runtime calls. if so we can call them using a br (they
3724 // will be in a reachable segment) otherwise we have to use a blr
3725 // which loads the absolute address into a register.
3726 address entry = (address)$meth$$method;
3727 CodeBlob *cb = CodeCache::find_blob(entry);
3728 if (cb) {
3729 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3730 if (call == nullptr) {
3731 ciEnv::current()->record_failure("CodeCache is full");
3732 return;
3733 }
3734 __ post_call_nop();
3735 } else {
3736 Label retaddr;
3737 // Make the anchor frame walkable
3738 __ adr(rscratch2, retaddr);
3739 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3740 __ lea(rscratch1, RuntimeAddress(entry));
3741 __ blr(rscratch1);
3742 __ bind(retaddr);
3743 __ post_call_nop();
3744 }
3745 if (Compile::current()->max_vector_size() > 0) {
3746 __ reinitialize_ptrue();
3747 }
3748 %}
3749
3750 enc_class aarch64_enc_rethrow() %{
3751 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3752 %}
3753
3754 enc_class aarch64_enc_ret() %{
3755 #ifdef ASSERT
3756 if (Compile::current()->max_vector_size() > 0) {
3757 __ verify_ptrue();
3758 }
3759 #endif
3760 __ ret(lr);
3761 %}
3762
3763 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3764 Register target_reg = as_Register($jump_target$$reg);
3765 __ br(target_reg);
3766 %}
3767
3768 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3769 Register target_reg = as_Register($jump_target$$reg);
3770 // exception oop should be in r0
3771 // ret addr has been popped into lr
3772 // callee expects it in r3
3773 __ mov(r3, lr);
3774 __ br(target_reg);
3775 %}
3776
3777 %}
3778
3779 //----------FRAME--------------------------------------------------------------
3780 // Definition of frame structure and management information.
3781 //
3782 // S T A C K L A Y O U T Allocators stack-slot number
3783 // | (to get allocators register number
3784 // G Owned by | | v add OptoReg::stack0())
3785 // r CALLER | |
3786 // o | +--------+ pad to even-align allocators stack-slot
3787 // w V | pad0 | numbers; owned by CALLER
3788 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3789 // h ^ | in | 5
3790 // | | args | 4 Holes in incoming args owned by SELF
3791 // | | | | 3
3792 // | | +--------+
3793 // V | | old out| Empty on Intel, window on Sparc
3794 // | old |preserve| Must be even aligned.
3795 // | SP-+--------+----> Matcher::_old_SP, even aligned
3796 // | | in | 3 area for Intel ret address
3797 // Owned by |preserve| Empty on Sparc.
3798 // SELF +--------+
3799 // | | pad2 | 2 pad to align old SP
3800 // | +--------+ 1
3801 // | | locks | 0
3802 // | +--------+----> OptoReg::stack0(), even aligned
3803 // | | pad1 | 11 pad to align new SP
3804 // | +--------+
3805 // | | | 10
3806 // | | spills | 9 spills
3807 // V | | 8 (pad0 slot for callee)
3808 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3809 // ^ | out | 7
3810 // | | args | 6 Holes in outgoing args owned by CALLEE
3811 // Owned by +--------+
3812 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3813 // | new |preserve| Must be even-aligned.
3814 // | SP-+--------+----> Matcher::_new_SP, even aligned
3815 // | | |
3816 //
3817 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3818 // known from SELF's arguments and the Java calling convention.
3819 // Region 6-7 is determined per call site.
3820 // Note 2: If the calling convention leaves holes in the incoming argument
3821 // area, those holes are owned by SELF. Holes in the outgoing area
3822 // are owned by the CALLEE. Holes should not be necessary in the
3823 // incoming area, as the Java calling convention is completely under
3824 // the control of the AD file. Doubles can be sorted and packed to
3825 // avoid holes. Holes in the outgoing arguments may be necessary for
3826 // varargs C calling conventions.
3827 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3828 // even aligned with pad0 as needed.
3829 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3830 // (the latter is true on Intel but is it false on AArch64?)
3831 // region 6-11 is even aligned; it may be padded out more so that
3832 // the region from SP to FP meets the minimum stack alignment.
3833 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3834 // alignment. Region 11, pad1, may be dynamically extended so that
3835 // SP meets the minimum alignment.
3836
3837 frame %{
3838 // These three registers define part of the calling convention
3839 // between compiled code and the interpreter.
3840
3841 // Inline Cache Register or Method for I2C.
3842 inline_cache_reg(R12);
3843
3844 // Number of stack slots consumed by locking an object
3845 sync_stack_slots(2);
3846
3847 // Compiled code's Frame Pointer
3848 frame_pointer(R31);
3849
3850 // Stack alignment requirement
3851 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3852
3853 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3854 // for calls to C. Supports the var-args backing area for register parms.
3855 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3856
3857 // The after-PROLOG location of the return address. Location of
3858 // return address specifies a type (REG or STACK) and a number
3859 // representing the register number (i.e. - use a register name) or
3860 // stack slot.
3861 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3862 // Otherwise, it is above the locks and verification slot and alignment word
3863 // TODO this may well be correct but need to check why that - 2 is there
3864 // ppc port uses 0 but we definitely need to allow for fixed_slots
3865 // which folds in the space used for monitors
3866 return_addr(STACK - 2 +
3867 align_up((Compile::current()->in_preserve_stack_slots() +
3868 Compile::current()->fixed_slots()),
3869 stack_alignment_in_slots()));
3870
3871 // Location of compiled Java return values. Same as C for now.
3872 return_value
3873 %{
3874 // TODO do we allow ideal_reg == Op_RegN???
3875 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3876 "only return normal values");
3877
3878 static const int lo[Op_RegL + 1] = { // enum name
3879 0, // Op_Node
3880 0, // Op_Set
3881 R0_num, // Op_RegN
3882 R0_num, // Op_RegI
3883 R0_num, // Op_RegP
3884 V0_num, // Op_RegF
3885 V0_num, // Op_RegD
3886 R0_num // Op_RegL
3887 };
3888
3889 static const int hi[Op_RegL + 1] = { // enum name
3890 0, // Op_Node
3891 0, // Op_Set
3892 OptoReg::Bad, // Op_RegN
3893 OptoReg::Bad, // Op_RegI
3894 R0_H_num, // Op_RegP
3895 OptoReg::Bad, // Op_RegF
3896 V0_H_num, // Op_RegD
3897 R0_H_num // Op_RegL
3898 };
3899
3900 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3901 %}
3902 %}
3903
3904 //----------ATTRIBUTES---------------------------------------------------------
3905 //----------Operand Attributes-------------------------------------------------
3906 op_attrib op_cost(1); // Required cost attribute
3907
3908 //----------Instruction Attributes---------------------------------------------
3909 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3910 ins_attrib ins_size(32); // Required size attribute (in bits)
3911 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3912 // a non-matching short branch variant
3913 // of some long branch?
3914 ins_attrib ins_alignment(4); // Required alignment attribute (must
3915 // be a power of 2) specifies the
3916 // alignment that some part of the
3917 // instruction (not necessarily the
3918 // start) requires. If > 1, a
3919 // compute_padding() function must be
3920 // provided for the instruction
3921
3922 // Whether this node is expanded during code emission into a sequence of
3923 // instructions and the first instruction can perform an implicit null check.
3924 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3925
3926 //----------OPERANDS-----------------------------------------------------------
3927 // Operand definitions must precede instruction definitions for correct parsing
3928 // in the ADLC because operands constitute user defined types which are used in
3929 // instruction definitions.
3930
3931 //----------Simple Operands----------------------------------------------------
3932
3933 // Integer operands 32 bit
3934 // 32 bit immediate
3935 operand immI()
3936 %{
3937 match(ConI);
3938
3939 op_cost(0);
3940 format %{ %}
3941 interface(CONST_INTER);
3942 %}
3943
3944 // 32 bit zero
3945 operand immI0()
3946 %{
3947 predicate(n->get_int() == 0);
3948 match(ConI);
3949
3950 op_cost(0);
3951 format %{ %}
3952 interface(CONST_INTER);
3953 %}
3954
3955 // 32 bit unit increment
3956 operand immI_1()
3957 %{
3958 predicate(n->get_int() == 1);
3959 match(ConI);
3960
3961 op_cost(0);
3962 format %{ %}
3963 interface(CONST_INTER);
3964 %}
3965
3966 // 32 bit unit decrement
3967 operand immI_M1()
3968 %{
3969 predicate(n->get_int() == -1);
3970 match(ConI);
3971
3972 op_cost(0);
3973 format %{ %}
3974 interface(CONST_INTER);
3975 %}
3976
3977 // Shift values for add/sub extension shift
3978 operand immIExt()
3979 %{
3980 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3981 match(ConI);
3982
3983 op_cost(0);
3984 format %{ %}
3985 interface(CONST_INTER);
3986 %}
3987
3988 operand immI_gt_1()
3989 %{
3990 predicate(n->get_int() > 1);
3991 match(ConI);
3992
3993 op_cost(0);
3994 format %{ %}
3995 interface(CONST_INTER);
3996 %}
3997
3998 operand immI_le_4()
3999 %{
4000 predicate(n->get_int() <= 4);
4001 match(ConI);
4002
4003 op_cost(0);
4004 format %{ %}
4005 interface(CONST_INTER);
4006 %}
4007
4008 operand immI_4()
4009 %{
4010 predicate(n->get_int() == 4);
4011 match(ConI);
4012
4013 op_cost(0);
4014 format %{ %}
4015 interface(CONST_INTER);
4016 %}
4017
4018 operand immI_16()
4019 %{
4020 predicate(n->get_int() == 16);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 operand immI_24()
4029 %{
4030 predicate(n->get_int() == 24);
4031 match(ConI);
4032
4033 op_cost(0);
4034 format %{ %}
4035 interface(CONST_INTER);
4036 %}
4037
4038 operand immI_32()
4039 %{
4040 predicate(n->get_int() == 32);
4041 match(ConI);
4042
4043 op_cost(0);
4044 format %{ %}
4045 interface(CONST_INTER);
4046 %}
4047
4048 operand immI_48()
4049 %{
4050 predicate(n->get_int() == 48);
4051 match(ConI);
4052
4053 op_cost(0);
4054 format %{ %}
4055 interface(CONST_INTER);
4056 %}
4057
4058 operand immI_56()
4059 %{
4060 predicate(n->get_int() == 56);
4061 match(ConI);
4062
4063 op_cost(0);
4064 format %{ %}
4065 interface(CONST_INTER);
4066 %}
4067
4068 operand immI_255()
4069 %{
4070 predicate(n->get_int() == 255);
4071 match(ConI);
4072
4073 op_cost(0);
4074 format %{ %}
4075 interface(CONST_INTER);
4076 %}
4077
4078 operand immI_65535()
4079 %{
4080 predicate(n->get_int() == 65535);
4081 match(ConI);
4082
4083 op_cost(0);
4084 format %{ %}
4085 interface(CONST_INTER);
4086 %}
4087
4088 operand immI_positive()
4089 %{
4090 predicate(n->get_int() > 0);
4091 match(ConI);
4092
4093 op_cost(0);
4094 format %{ %}
4095 interface(CONST_INTER);
4096 %}
4097
4098 // BoolTest condition for signed compare
4099 operand immI_cmp_cond()
4100 %{
4101 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4102 match(ConI);
4103
4104 op_cost(0);
4105 format %{ %}
4106 interface(CONST_INTER);
4107 %}
4108
4109 // BoolTest condition for unsigned compare
4110 operand immI_cmpU_cond()
4111 %{
4112 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4113 match(ConI);
4114
4115 op_cost(0);
4116 format %{ %}
4117 interface(CONST_INTER);
4118 %}
4119
4120 operand immL_255()
4121 %{
4122 predicate(n->get_long() == 255L);
4123 match(ConL);
4124
4125 op_cost(0);
4126 format %{ %}
4127 interface(CONST_INTER);
4128 %}
4129
4130 operand immL_65535()
4131 %{
4132 predicate(n->get_long() == 65535L);
4133 match(ConL);
4134
4135 op_cost(0);
4136 format %{ %}
4137 interface(CONST_INTER);
4138 %}
4139
4140 operand immL_4294967295()
4141 %{
4142 predicate(n->get_long() == 4294967295L);
4143 match(ConL);
4144
4145 op_cost(0);
4146 format %{ %}
4147 interface(CONST_INTER);
4148 %}
4149
4150 operand immL_bitmask()
4151 %{
4152 predicate((n->get_long() != 0)
4153 && ((n->get_long() & 0xc000000000000000l) == 0)
4154 && is_power_of_2(n->get_long() + 1));
4155 match(ConL);
4156
4157 op_cost(0);
4158 format %{ %}
4159 interface(CONST_INTER);
4160 %}
4161
4162 operand immI_bitmask()
4163 %{
4164 predicate((n->get_int() != 0)
4165 && ((n->get_int() & 0xc0000000) == 0)
4166 && is_power_of_2(n->get_int() + 1));
4167 match(ConI);
4168
4169 op_cost(0);
4170 format %{ %}
4171 interface(CONST_INTER);
4172 %}
4173
4174 operand immL_positive_bitmaskI()
4175 %{
4176 predicate((n->get_long() != 0)
4177 && ((julong)n->get_long() < 0x80000000ULL)
4178 && is_power_of_2(n->get_long() + 1));
4179 match(ConL);
4180
4181 op_cost(0);
4182 format %{ %}
4183 interface(CONST_INTER);
4184 %}
4185
4186 // Scale values for scaled offset addressing modes (up to long but not quad)
4187 operand immIScale()
4188 %{
4189 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4190 match(ConI);
4191
4192 op_cost(0);
4193 format %{ %}
4194 interface(CONST_INTER);
4195 %}
4196
4197 // 5 bit signed integer
4198 operand immI5()
4199 %{
4200 predicate(Assembler::is_simm(n->get_int(), 5));
4201 match(ConI);
4202
4203 op_cost(0);
4204 format %{ %}
4205 interface(CONST_INTER);
4206 %}
4207
4208 // 7 bit unsigned integer
4209 operand immIU7()
4210 %{
4211 predicate(Assembler::is_uimm(n->get_int(), 7));
4212 match(ConI);
4213
4214 op_cost(0);
4215 format %{ %}
4216 interface(CONST_INTER);
4217 %}
4218
4219 // Offset for scaled or unscaled immediate loads and stores
4220 operand immIOffset()
4221 %{
4222 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4223 match(ConI);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 operand immIOffset1()
4231 %{
4232 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4233 match(ConI);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immIOffset2()
4241 %{
4242 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4243 match(ConI);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immIOffset4()
4251 %{
4252 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4253 match(ConI);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immIOffset8()
4261 %{
4262 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4263 match(ConI);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immIOffset16()
4271 %{
4272 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4273 match(ConI);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immLOffset()
4281 %{
4282 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4283 match(ConL);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 operand immLoffset1()
4291 %{
4292 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4293 match(ConL);
4294
4295 op_cost(0);
4296 format %{ %}
4297 interface(CONST_INTER);
4298 %}
4299
4300 operand immLoffset2()
4301 %{
4302 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4303 match(ConL);
4304
4305 op_cost(0);
4306 format %{ %}
4307 interface(CONST_INTER);
4308 %}
4309
4310 operand immLoffset4()
4311 %{
4312 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4313 match(ConL);
4314
4315 op_cost(0);
4316 format %{ %}
4317 interface(CONST_INTER);
4318 %}
4319
4320 operand immLoffset8()
4321 %{
4322 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4323 match(ConL);
4324
4325 op_cost(0);
4326 format %{ %}
4327 interface(CONST_INTER);
4328 %}
4329
4330 operand immLoffset16()
4331 %{
4332 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4333 match(ConL);
4334
4335 op_cost(0);
4336 format %{ %}
4337 interface(CONST_INTER);
4338 %}
4339
4340 // 5 bit signed long integer
4341 operand immL5()
4342 %{
4343 predicate(Assembler::is_simm(n->get_long(), 5));
4344 match(ConL);
4345
4346 op_cost(0);
4347 format %{ %}
4348 interface(CONST_INTER);
4349 %}
4350
4351 // 7 bit unsigned long integer
4352 operand immLU7()
4353 %{
4354 predicate(Assembler::is_uimm(n->get_long(), 7));
4355 match(ConL);
4356
4357 op_cost(0);
4358 format %{ %}
4359 interface(CONST_INTER);
4360 %}
4361
4362 // 8 bit signed value.
4363 operand immI8()
4364 %{
4365 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4366 match(ConI);
4367
4368 op_cost(0);
4369 format %{ %}
4370 interface(CONST_INTER);
4371 %}
4372
4373 // 8 bit signed value (simm8), or #simm8 LSL 8.
4374 operand immIDupV()
4375 %{
4376 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4377 match(ConI);
4378
4379 op_cost(0);
4380 format %{ %}
4381 interface(CONST_INTER);
4382 %}
4383
4384 // 8 bit signed value (simm8), or #simm8 LSL 8.
4385 operand immLDupV()
4386 %{
4387 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4388 match(ConL);
4389
4390 op_cost(0);
4391 format %{ %}
4392 interface(CONST_INTER);
4393 %}
4394
4395 // 8 bit signed value (simm8), or #simm8 LSL 8.
4396 operand immHDupV()
4397 %{
4398 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4399 match(ConH);
4400
4401 op_cost(0);
4402 format %{ %}
4403 interface(CONST_INTER);
4404 %}
4405
4406 // 8 bit integer valid for vector add sub immediate
4407 operand immBAddSubV()
4408 %{
4409 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4410 match(ConI);
4411
4412 op_cost(0);
4413 format %{ %}
4414 interface(CONST_INTER);
4415 %}
4416
4417 // 32 bit integer valid for add sub immediate
4418 operand immIAddSub()
4419 %{
4420 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4421 match(ConI);
4422 op_cost(0);
4423 format %{ %}
4424 interface(CONST_INTER);
4425 %}
4426
4427 // 32 bit integer valid for vector add sub immediate
4428 operand immIAddSubV()
4429 %{
4430 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4431 match(ConI);
4432
4433 op_cost(0);
4434 format %{ %}
4435 interface(CONST_INTER);
4436 %}
4437
4438 // 32 bit unsigned integer valid for logical immediate
4439
4440 operand immBLog()
4441 %{
4442 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4443 match(ConI);
4444
4445 op_cost(0);
4446 format %{ %}
4447 interface(CONST_INTER);
4448 %}
4449
4450 operand immSLog()
4451 %{
4452 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4453 match(ConI);
4454
4455 op_cost(0);
4456 format %{ %}
4457 interface(CONST_INTER);
4458 %}
4459
4460 operand immILog()
4461 %{
4462 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4463 match(ConI);
4464
4465 op_cost(0);
4466 format %{ %}
4467 interface(CONST_INTER);
4468 %}
4469
4470 // Integer operands 64 bit
4471 // 64 bit immediate
4472 operand immL()
4473 %{
4474 match(ConL);
4475
4476 op_cost(0);
4477 format %{ %}
4478 interface(CONST_INTER);
4479 %}
4480
4481 // 64 bit zero
4482 operand immL0()
4483 %{
4484 predicate(n->get_long() == 0);
4485 match(ConL);
4486
4487 op_cost(0);
4488 format %{ %}
4489 interface(CONST_INTER);
4490 %}
4491
4492 // 64 bit unit decrement
4493 operand immL_M1()
4494 %{
4495 predicate(n->get_long() == -1);
4496 match(ConL);
4497
4498 op_cost(0);
4499 format %{ %}
4500 interface(CONST_INTER);
4501 %}
4502
4503 // 64 bit integer valid for add sub immediate
4504 operand immLAddSub()
4505 %{
4506 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4507 match(ConL);
4508 op_cost(0);
4509 format %{ %}
4510 interface(CONST_INTER);
4511 %}
4512
4513 // 64 bit integer valid for addv subv immediate
4514 operand immLAddSubV()
4515 %{
4516 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4517 match(ConL);
4518
4519 op_cost(0);
4520 format %{ %}
4521 interface(CONST_INTER);
4522 %}
4523
4524 // 64 bit integer valid for logical immediate
4525 operand immLLog()
4526 %{
4527 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4528 match(ConL);
4529 op_cost(0);
4530 format %{ %}
4531 interface(CONST_INTER);
4532 %}
4533
4534 // Long Immediate: low 32-bit mask
4535 operand immL_32bits()
4536 %{
4537 predicate(n->get_long() == 0xFFFFFFFFL);
4538 match(ConL);
4539 op_cost(0);
4540 format %{ %}
4541 interface(CONST_INTER);
4542 %}
4543
4544 // Pointer operands
4545 // Pointer Immediate
4546 operand immP()
4547 %{
4548 match(ConP);
4549
4550 op_cost(0);
4551 format %{ %}
4552 interface(CONST_INTER);
4553 %}
4554
4555 // nullptr Pointer Immediate
4556 operand immP0()
4557 %{
4558 predicate(n->get_ptr() == 0);
4559 match(ConP);
4560
4561 op_cost(0);
4562 format %{ %}
4563 interface(CONST_INTER);
4564 %}
4565
4566 // Pointer Immediate One
4567 // this is used in object initialization (initial object header)
4568 operand immP_1()
4569 %{
4570 predicate(n->get_ptr() == 1);
4571 match(ConP);
4572
4573 op_cost(0);
4574 format %{ %}
4575 interface(CONST_INTER);
4576 %}
4577
4578 // AOT Runtime Constants Address
4579 operand immAOTRuntimeConstantsAddress()
4580 %{
4581 // Check if the address is in the range of AOT Runtime Constants
4582 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4583 match(ConP);
4584
4585 op_cost(0);
4586 format %{ %}
4587 interface(CONST_INTER);
4588 %}
4589
4590 // Float and Double operands
4591 // Double Immediate
4592 operand immD()
4593 %{
4594 match(ConD);
4595 op_cost(0);
4596 format %{ %}
4597 interface(CONST_INTER);
4598 %}
4599
4600 // Double Immediate: +0.0d
4601 operand immD0()
4602 %{
4603 predicate(jlong_cast(n->getd()) == 0);
4604 match(ConD);
4605
4606 op_cost(0);
4607 format %{ %}
4608 interface(CONST_INTER);
4609 %}
4610
4611 // constant 'double +0.0'.
4612 operand immDPacked()
4613 %{
4614 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4615 match(ConD);
4616 op_cost(0);
4617 format %{ %}
4618 interface(CONST_INTER);
4619 %}
4620
4621 // Float Immediate
4622 operand immF()
4623 %{
4624 match(ConF);
4625 op_cost(0);
4626 format %{ %}
4627 interface(CONST_INTER);
4628 %}
4629
4630 // Float Immediate: +0.0f.
4631 operand immF0()
4632 %{
4633 predicate(jint_cast(n->getf()) == 0);
4634 match(ConF);
4635
4636 op_cost(0);
4637 format %{ %}
4638 interface(CONST_INTER);
4639 %}
4640
4641 // Half Float (FP16) Immediate
4642 operand immH()
4643 %{
4644 match(ConH);
4645 op_cost(0);
4646 format %{ %}
4647 interface(CONST_INTER);
4648 %}
4649
4650 //
4651 operand immFPacked()
4652 %{
4653 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4654 match(ConF);
4655 op_cost(0);
4656 format %{ %}
4657 interface(CONST_INTER);
4658 %}
4659
4660 // Narrow pointer operands
4661 // Narrow Pointer Immediate
4662 operand immN()
4663 %{
4664 match(ConN);
4665
4666 op_cost(0);
4667 format %{ %}
4668 interface(CONST_INTER);
4669 %}
4670
4671 // Narrow nullptr Pointer Immediate
4672 operand immN0()
4673 %{
4674 predicate(n->get_narrowcon() == 0);
4675 match(ConN);
4676
4677 op_cost(0);
4678 format %{ %}
4679 interface(CONST_INTER);
4680 %}
4681
4682 operand immNKlass()
4683 %{
4684 match(ConNKlass);
4685
4686 op_cost(0);
4687 format %{ %}
4688 interface(CONST_INTER);
4689 %}
4690
4691 // Integer 32 bit Register Operands
4692 // Integer 32 bitRegister (excludes SP)
4693 operand iRegI()
4694 %{
4695 constraint(ALLOC_IN_RC(any_reg32));
4696 match(RegI);
4697 match(iRegINoSp);
4698 op_cost(0);
4699 format %{ %}
4700 interface(REG_INTER);
4701 %}
4702
4703 // Integer 32 bit Register not Special
4704 operand iRegINoSp()
4705 %{
4706 constraint(ALLOC_IN_RC(no_special_reg32));
4707 match(RegI);
4708 op_cost(0);
4709 format %{ %}
4710 interface(REG_INTER);
4711 %}
4712
4713 // Integer 64 bit Register Operands
4714 // Integer 64 bit Register (includes SP)
4715 operand iRegL()
4716 %{
4717 constraint(ALLOC_IN_RC(any_reg));
4718 match(RegL);
4719 match(iRegLNoSp);
4720 op_cost(0);
4721 format %{ %}
4722 interface(REG_INTER);
4723 %}
4724
4725 // Integer 64 bit Register not Special
4726 operand iRegLNoSp()
4727 %{
4728 constraint(ALLOC_IN_RC(no_special_reg));
4729 match(RegL);
4730 match(iRegL_R0);
4731 format %{ %}
4732 interface(REG_INTER);
4733 %}
4734
4735 // Pointer Register Operands
4736 // Pointer Register
4737 operand iRegP()
4738 %{
4739 constraint(ALLOC_IN_RC(ptr_reg));
4740 match(RegP);
4741 match(iRegPNoSp);
4742 match(iRegP_R0);
4743 //match(iRegP_R2);
4744 //match(iRegP_R4);
4745 match(iRegP_R5);
4746 match(thread_RegP);
4747 op_cost(0);
4748 format %{ %}
4749 interface(REG_INTER);
4750 %}
4751
4752 // Pointer 64 bit Register not Special
4753 operand iRegPNoSp()
4754 %{
4755 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4756 match(RegP);
4757 // match(iRegP);
4758 // match(iRegP_R0);
4759 // match(iRegP_R2);
4760 // match(iRegP_R4);
4761 // match(iRegP_R5);
4762 // match(thread_RegP);
4763 op_cost(0);
4764 format %{ %}
4765 interface(REG_INTER);
4766 %}
4767
4768 // This operand is not allowed to use rfp even if
4769 // rfp is not used to hold the frame pointer.
4770 operand iRegPNoSpNoRfp()
4771 %{
4772 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4773 match(RegP);
4774 match(iRegPNoSp);
4775 op_cost(0);
4776 format %{ %}
4777 interface(REG_INTER);
4778 %}
4779
4780 // Pointer 64 bit Register R0 only
4781 operand iRegP_R0()
4782 %{
4783 constraint(ALLOC_IN_RC(r0_reg));
4784 match(RegP);
4785 // match(iRegP);
4786 match(iRegPNoSp);
4787 op_cost(0);
4788 format %{ %}
4789 interface(REG_INTER);
4790 %}
4791
4792 // Pointer 64 bit Register R1 only
4793 operand iRegP_R1()
4794 %{
4795 constraint(ALLOC_IN_RC(r1_reg));
4796 match(RegP);
4797 // match(iRegP);
4798 match(iRegPNoSp);
4799 op_cost(0);
4800 format %{ %}
4801 interface(REG_INTER);
4802 %}
4803
4804 // Pointer 64 bit Register R2 only
4805 operand iRegP_R2()
4806 %{
4807 constraint(ALLOC_IN_RC(r2_reg));
4808 match(RegP);
4809 // match(iRegP);
4810 match(iRegPNoSp);
4811 op_cost(0);
4812 format %{ %}
4813 interface(REG_INTER);
4814 %}
4815
4816 // Pointer 64 bit Register R3 only
4817 operand iRegP_R3()
4818 %{
4819 constraint(ALLOC_IN_RC(r3_reg));
4820 match(RegP);
4821 // match(iRegP);
4822 match(iRegPNoSp);
4823 op_cost(0);
4824 format %{ %}
4825 interface(REG_INTER);
4826 %}
4827
4828 // Pointer 64 bit Register R4 only
4829 operand iRegP_R4()
4830 %{
4831 constraint(ALLOC_IN_RC(r4_reg));
4832 match(RegP);
4833 // match(iRegP);
4834 match(iRegPNoSp);
4835 op_cost(0);
4836 format %{ %}
4837 interface(REG_INTER);
4838 %}
4839
4840 // Pointer 64 bit Register R5 only
4841 operand iRegP_R5()
4842 %{
4843 constraint(ALLOC_IN_RC(r5_reg));
4844 match(RegP);
4845 // match(iRegP);
4846 match(iRegPNoSp);
4847 op_cost(0);
4848 format %{ %}
4849 interface(REG_INTER);
4850 %}
4851
4852 // Pointer 64 bit Register R10 only
4853 operand iRegP_R10()
4854 %{
4855 constraint(ALLOC_IN_RC(r10_reg));
4856 match(RegP);
4857 // match(iRegP);
4858 match(iRegPNoSp);
4859 op_cost(0);
4860 format %{ %}
4861 interface(REG_INTER);
4862 %}
4863
4864 // Long 64 bit Register R0 only
4865 operand iRegL_R0()
4866 %{
4867 constraint(ALLOC_IN_RC(r0_reg));
4868 match(RegL);
4869 match(iRegLNoSp);
4870 op_cost(0);
4871 format %{ %}
4872 interface(REG_INTER);
4873 %}
4874
4875 // Long 64 bit Register R11 only
4876 operand iRegL_R11()
4877 %{
4878 constraint(ALLOC_IN_RC(r11_reg));
4879 match(RegL);
4880 match(iRegLNoSp);
4881 op_cost(0);
4882 format %{ %}
4883 interface(REG_INTER);
4884 %}
4885
4886 // Register R0 only
4887 operand iRegI_R0()
4888 %{
4889 constraint(ALLOC_IN_RC(int_r0_reg));
4890 match(RegI);
4891 match(iRegINoSp);
4892 op_cost(0);
4893 format %{ %}
4894 interface(REG_INTER);
4895 %}
4896
4897 // Register R2 only
4898 operand iRegI_R2()
4899 %{
4900 constraint(ALLOC_IN_RC(int_r2_reg));
4901 match(RegI);
4902 match(iRegINoSp);
4903 op_cost(0);
4904 format %{ %}
4905 interface(REG_INTER);
4906 %}
4907
4908 // Register R3 only
4909 operand iRegI_R3()
4910 %{
4911 constraint(ALLOC_IN_RC(int_r3_reg));
4912 match(RegI);
4913 match(iRegINoSp);
4914 op_cost(0);
4915 format %{ %}
4916 interface(REG_INTER);
4917 %}
4918
4919
4920 // Register R4 only
4921 operand iRegI_R4()
4922 %{
4923 constraint(ALLOC_IN_RC(int_r4_reg));
4924 match(RegI);
4925 match(iRegINoSp);
4926 op_cost(0);
4927 format %{ %}
4928 interface(REG_INTER);
4929 %}
4930
4931
4932 // Pointer Register Operands
4933 // Narrow Pointer Register
4934 operand iRegN()
4935 %{
4936 constraint(ALLOC_IN_RC(any_reg32));
4937 match(RegN);
4938 match(iRegNNoSp);
4939 op_cost(0);
4940 format %{ %}
4941 interface(REG_INTER);
4942 %}
4943
4944 // Integer 64 bit Register not Special
4945 operand iRegNNoSp()
4946 %{
4947 constraint(ALLOC_IN_RC(no_special_reg32));
4948 match(RegN);
4949 op_cost(0);
4950 format %{ %}
4951 interface(REG_INTER);
4952 %}
4953
4954 // Float Register
4955 // Float register operands
4956 operand vRegF()
4957 %{
4958 constraint(ALLOC_IN_RC(float_reg));
4959 match(RegF);
4960
4961 op_cost(0);
4962 format %{ %}
4963 interface(REG_INTER);
4964 %}
4965
4966 // Double Register
4967 // Double register operands
4968 operand vRegD()
4969 %{
4970 constraint(ALLOC_IN_RC(double_reg));
4971 match(RegD);
4972
4973 op_cost(0);
4974 format %{ %}
4975 interface(REG_INTER);
4976 %}
4977
4978 // Generic vector class. This will be used for
4979 // all vector operands, including NEON and SVE.
4980 operand vReg()
4981 %{
4982 constraint(ALLOC_IN_RC(dynamic));
4983 match(VecA);
4984 match(VecD);
4985 match(VecX);
4986
4987 op_cost(0);
4988 format %{ %}
4989 interface(REG_INTER);
4990 %}
4991
4992 operand vReg_V10()
4993 %{
4994 constraint(ALLOC_IN_RC(v10_veca_reg));
4995 match(vReg);
4996
4997 op_cost(0);
4998 format %{ %}
4999 interface(REG_INTER);
5000 %}
5001
5002 operand vReg_V11()
5003 %{
5004 constraint(ALLOC_IN_RC(v11_veca_reg));
5005 match(vReg);
5006
5007 op_cost(0);
5008 format %{ %}
5009 interface(REG_INTER);
5010 %}
5011
5012 operand vReg_V12()
5013 %{
5014 constraint(ALLOC_IN_RC(v12_veca_reg));
5015 match(vReg);
5016
5017 op_cost(0);
5018 format %{ %}
5019 interface(REG_INTER);
5020 %}
5021
5022 operand vReg_V13()
5023 %{
5024 constraint(ALLOC_IN_RC(v13_veca_reg));
5025 match(vReg);
5026
5027 op_cost(0);
5028 format %{ %}
5029 interface(REG_INTER);
5030 %}
5031
5032 operand vReg_V17()
5033 %{
5034 constraint(ALLOC_IN_RC(v17_veca_reg));
5035 match(vReg);
5036
5037 op_cost(0);
5038 format %{ %}
5039 interface(REG_INTER);
5040 %}
5041
5042 operand vReg_V18()
5043 %{
5044 constraint(ALLOC_IN_RC(v18_veca_reg));
5045 match(vReg);
5046
5047 op_cost(0);
5048 format %{ %}
5049 interface(REG_INTER);
5050 %}
5051
5052 operand vReg_V23()
5053 %{
5054 constraint(ALLOC_IN_RC(v23_veca_reg));
5055 match(vReg);
5056
5057 op_cost(0);
5058 format %{ %}
5059 interface(REG_INTER);
5060 %}
5061
5062 operand vReg_V24()
5063 %{
5064 constraint(ALLOC_IN_RC(v24_veca_reg));
5065 match(vReg);
5066
5067 op_cost(0);
5068 format %{ %}
5069 interface(REG_INTER);
5070 %}
5071
5072 operand vecA()
5073 %{
5074 constraint(ALLOC_IN_RC(vectora_reg));
5075 match(VecA);
5076
5077 op_cost(0);
5078 format %{ %}
5079 interface(REG_INTER);
5080 %}
5081
5082 operand vecD()
5083 %{
5084 constraint(ALLOC_IN_RC(vectord_reg));
5085 match(VecD);
5086
5087 op_cost(0);
5088 format %{ %}
5089 interface(REG_INTER);
5090 %}
5091
5092 operand vecX()
5093 %{
5094 constraint(ALLOC_IN_RC(vectorx_reg));
5095 match(VecX);
5096
5097 op_cost(0);
5098 format %{ %}
5099 interface(REG_INTER);
5100 %}
5101
5102 operand vRegD_V0()
5103 %{
5104 constraint(ALLOC_IN_RC(v0_reg));
5105 match(RegD);
5106 op_cost(0);
5107 format %{ %}
5108 interface(REG_INTER);
5109 %}
5110
5111 operand vRegD_V1()
5112 %{
5113 constraint(ALLOC_IN_RC(v1_reg));
5114 match(RegD);
5115 op_cost(0);
5116 format %{ %}
5117 interface(REG_INTER);
5118 %}
5119
5120 operand vRegD_V2()
5121 %{
5122 constraint(ALLOC_IN_RC(v2_reg));
5123 match(RegD);
5124 op_cost(0);
5125 format %{ %}
5126 interface(REG_INTER);
5127 %}
5128
5129 operand vRegD_V3()
5130 %{
5131 constraint(ALLOC_IN_RC(v3_reg));
5132 match(RegD);
5133 op_cost(0);
5134 format %{ %}
5135 interface(REG_INTER);
5136 %}
5137
5138 operand vRegD_V4()
5139 %{
5140 constraint(ALLOC_IN_RC(v4_reg));
5141 match(RegD);
5142 op_cost(0);
5143 format %{ %}
5144 interface(REG_INTER);
5145 %}
5146
5147 operand vRegD_V5()
5148 %{
5149 constraint(ALLOC_IN_RC(v5_reg));
5150 match(RegD);
5151 op_cost(0);
5152 format %{ %}
5153 interface(REG_INTER);
5154 %}
5155
5156 operand vRegD_V6()
5157 %{
5158 constraint(ALLOC_IN_RC(v6_reg));
5159 match(RegD);
5160 op_cost(0);
5161 format %{ %}
5162 interface(REG_INTER);
5163 %}
5164
5165 operand vRegD_V7()
5166 %{
5167 constraint(ALLOC_IN_RC(v7_reg));
5168 match(RegD);
5169 op_cost(0);
5170 format %{ %}
5171 interface(REG_INTER);
5172 %}
5173
5174 operand vRegD_V12()
5175 %{
5176 constraint(ALLOC_IN_RC(v12_reg));
5177 match(RegD);
5178 op_cost(0);
5179 format %{ %}
5180 interface(REG_INTER);
5181 %}
5182
5183 operand vRegD_V13()
5184 %{
5185 constraint(ALLOC_IN_RC(v13_reg));
5186 match(RegD);
5187 op_cost(0);
5188 format %{ %}
5189 interface(REG_INTER);
5190 %}
5191
5192 operand pReg()
5193 %{
5194 constraint(ALLOC_IN_RC(pr_reg));
5195 match(RegVectMask);
5196 match(pRegGov);
5197 op_cost(0);
5198 format %{ %}
5199 interface(REG_INTER);
5200 %}
5201
5202 operand pRegGov()
5203 %{
5204 constraint(ALLOC_IN_RC(gov_pr));
5205 match(RegVectMask);
5206 match(pReg);
5207 op_cost(0);
5208 format %{ %}
5209 interface(REG_INTER);
5210 %}
5211
5212 operand pRegGov_P0()
5213 %{
5214 constraint(ALLOC_IN_RC(p0_reg));
5215 match(RegVectMask);
5216 op_cost(0);
5217 format %{ %}
5218 interface(REG_INTER);
5219 %}
5220
5221 operand pRegGov_P1()
5222 %{
5223 constraint(ALLOC_IN_RC(p1_reg));
5224 match(RegVectMask);
5225 op_cost(0);
5226 format %{ %}
5227 interface(REG_INTER);
5228 %}
5229
5230 // Flags register, used as output of signed compare instructions
5231
5232 // note that on AArch64 we also use this register as the output for
5233 // for floating point compare instructions (CmpF CmpD). this ensures
5234 // that ordered inequality tests use GT, GE, LT or LE none of which
5235 // pass through cases where the result is unordered i.e. one or both
5236 // inputs to the compare is a NaN. this means that the ideal code can
5237 // replace e.g. a GT with an LE and not end up capturing the NaN case
5238 // (where the comparison should always fail). EQ and NE tests are
5239 // always generated in ideal code so that unordered folds into the NE
5240 // case, matching the behaviour of AArch64 NE.
5241 //
5242 // This differs from x86 where the outputs of FP compares use a
5243 // special FP flags registers and where compares based on this
5244 // register are distinguished into ordered inequalities (cmpOpUCF) and
5245 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5246 // to explicitly handle the unordered case in branches. x86 also has
5247 // to include extra CMoveX rules to accept a cmpOpUCF input.
5248
5249 operand rFlagsReg()
5250 %{
5251 constraint(ALLOC_IN_RC(int_flags));
5252 match(RegFlags);
5253
5254 op_cost(0);
5255 format %{ "RFLAGS" %}
5256 interface(REG_INTER);
5257 %}
5258
5259 // Flags register, used as output of unsigned compare instructions
5260 operand rFlagsRegU()
5261 %{
5262 constraint(ALLOC_IN_RC(int_flags));
5263 match(RegFlags);
5264
5265 op_cost(0);
5266 format %{ "RFLAGSU" %}
5267 interface(REG_INTER);
5268 %}
5269
5270 // Special Registers
5271
5272 // Method Register
5273 operand inline_cache_RegP(iRegP reg)
5274 %{
5275 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5276 match(reg);
5277 match(iRegPNoSp);
5278 op_cost(0);
5279 format %{ %}
5280 interface(REG_INTER);
5281 %}
5282
5283 // Thread Register
5284 operand thread_RegP(iRegP reg)
5285 %{
5286 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5287 match(reg);
5288 op_cost(0);
5289 format %{ %}
5290 interface(REG_INTER);
5291 %}
5292
5293 //----------Memory Operands----------------------------------------------------
5294
5295 operand indirect(iRegP reg)
5296 %{
5297 constraint(ALLOC_IN_RC(ptr_reg));
5298 match(reg);
5299 op_cost(0);
5300 format %{ "[$reg]" %}
5301 interface(MEMORY_INTER) %{
5302 base($reg);
5303 index(0xffffffff);
5304 scale(0x0);
5305 disp(0x0);
5306 %}
5307 %}
5308
5309 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5310 %{
5311 constraint(ALLOC_IN_RC(ptr_reg));
5312 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5313 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5314 op_cost(0);
5315 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5316 interface(MEMORY_INTER) %{
5317 base($reg);
5318 index($ireg);
5319 scale($scale);
5320 disp(0x0);
5321 %}
5322 %}
5323
5324 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5325 %{
5326 constraint(ALLOC_IN_RC(ptr_reg));
5327 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5328 match(AddP reg (LShiftL lreg scale));
5329 op_cost(0);
5330 format %{ "$reg, $lreg lsl($scale)" %}
5331 interface(MEMORY_INTER) %{
5332 base($reg);
5333 index($lreg);
5334 scale($scale);
5335 disp(0x0);
5336 %}
5337 %}
5338
5339 operand indIndexI2L(iRegP reg, iRegI ireg)
5340 %{
5341 constraint(ALLOC_IN_RC(ptr_reg));
5342 match(AddP reg (ConvI2L ireg));
5343 op_cost(0);
5344 format %{ "$reg, $ireg, 0, I2L" %}
5345 interface(MEMORY_INTER) %{
5346 base($reg);
5347 index($ireg);
5348 scale(0x0);
5349 disp(0x0);
5350 %}
5351 %}
5352
5353 operand indIndex(iRegP reg, iRegL lreg)
5354 %{
5355 constraint(ALLOC_IN_RC(ptr_reg));
5356 match(AddP reg lreg);
5357 op_cost(0);
5358 format %{ "$reg, $lreg" %}
5359 interface(MEMORY_INTER) %{
5360 base($reg);
5361 index($lreg);
5362 scale(0x0);
5363 disp(0x0);
5364 %}
5365 %}
5366
5367 operand indOffI1(iRegP reg, immIOffset1 off)
5368 %{
5369 constraint(ALLOC_IN_RC(ptr_reg));
5370 match(AddP reg off);
5371 op_cost(0);
5372 format %{ "[$reg, $off]" %}
5373 interface(MEMORY_INTER) %{
5374 base($reg);
5375 index(0xffffffff);
5376 scale(0x0);
5377 disp($off);
5378 %}
5379 %}
5380
5381 operand indOffI2(iRegP reg, immIOffset2 off)
5382 %{
5383 constraint(ALLOC_IN_RC(ptr_reg));
5384 match(AddP reg off);
5385 op_cost(0);
5386 format %{ "[$reg, $off]" %}
5387 interface(MEMORY_INTER) %{
5388 base($reg);
5389 index(0xffffffff);
5390 scale(0x0);
5391 disp($off);
5392 %}
5393 %}
5394
5395 operand indOffI4(iRegP reg, immIOffset4 off)
5396 %{
5397 constraint(ALLOC_IN_RC(ptr_reg));
5398 match(AddP reg off);
5399 op_cost(0);
5400 format %{ "[$reg, $off]" %}
5401 interface(MEMORY_INTER) %{
5402 base($reg);
5403 index(0xffffffff);
5404 scale(0x0);
5405 disp($off);
5406 %}
5407 %}
5408
5409 operand indOffI8(iRegP reg, immIOffset8 off)
5410 %{
5411 constraint(ALLOC_IN_RC(ptr_reg));
5412 match(AddP reg off);
5413 op_cost(0);
5414 format %{ "[$reg, $off]" %}
5415 interface(MEMORY_INTER) %{
5416 base($reg);
5417 index(0xffffffff);
5418 scale(0x0);
5419 disp($off);
5420 %}
5421 %}
5422
5423 operand indOffI16(iRegP reg, immIOffset16 off)
5424 %{
5425 constraint(ALLOC_IN_RC(ptr_reg));
5426 match(AddP reg off);
5427 op_cost(0);
5428 format %{ "[$reg, $off]" %}
5429 interface(MEMORY_INTER) %{
5430 base($reg);
5431 index(0xffffffff);
5432 scale(0x0);
5433 disp($off);
5434 %}
5435 %}
5436
5437 operand indOffL1(iRegP reg, immLoffset1 off)
5438 %{
5439 constraint(ALLOC_IN_RC(ptr_reg));
5440 match(AddP reg off);
5441 op_cost(0);
5442 format %{ "[$reg, $off]" %}
5443 interface(MEMORY_INTER) %{
5444 base($reg);
5445 index(0xffffffff);
5446 scale(0x0);
5447 disp($off);
5448 %}
5449 %}
5450
5451 operand indOffL2(iRegP reg, immLoffset2 off)
5452 %{
5453 constraint(ALLOC_IN_RC(ptr_reg));
5454 match(AddP reg off);
5455 op_cost(0);
5456 format %{ "[$reg, $off]" %}
5457 interface(MEMORY_INTER) %{
5458 base($reg);
5459 index(0xffffffff);
5460 scale(0x0);
5461 disp($off);
5462 %}
5463 %}
5464
5465 operand indOffL4(iRegP reg, immLoffset4 off)
5466 %{
5467 constraint(ALLOC_IN_RC(ptr_reg));
5468 match(AddP reg off);
5469 op_cost(0);
5470 format %{ "[$reg, $off]" %}
5471 interface(MEMORY_INTER) %{
5472 base($reg);
5473 index(0xffffffff);
5474 scale(0x0);
5475 disp($off);
5476 %}
5477 %}
5478
5479 operand indOffL8(iRegP reg, immLoffset8 off)
5480 %{
5481 constraint(ALLOC_IN_RC(ptr_reg));
5482 match(AddP reg off);
5483 op_cost(0);
5484 format %{ "[$reg, $off]" %}
5485 interface(MEMORY_INTER) %{
5486 base($reg);
5487 index(0xffffffff);
5488 scale(0x0);
5489 disp($off);
5490 %}
5491 %}
5492
5493 operand indOffL16(iRegP reg, immLoffset16 off)
5494 %{
5495 constraint(ALLOC_IN_RC(ptr_reg));
5496 match(AddP reg off);
5497 op_cost(0);
5498 format %{ "[$reg, $off]" %}
5499 interface(MEMORY_INTER) %{
5500 base($reg);
5501 index(0xffffffff);
5502 scale(0x0);
5503 disp($off);
5504 %}
5505 %}
5506
5507 operand indirectX2P(iRegL reg)
5508 %{
5509 constraint(ALLOC_IN_RC(ptr_reg));
5510 match(CastX2P reg);
5511 op_cost(0);
5512 format %{ "[$reg]\t# long -> ptr" %}
5513 interface(MEMORY_INTER) %{
5514 base($reg);
5515 index(0xffffffff);
5516 scale(0x0);
5517 disp(0x0);
5518 %}
5519 %}
5520
5521 operand indOffX2P(iRegL reg, immLOffset off)
5522 %{
5523 constraint(ALLOC_IN_RC(ptr_reg));
5524 match(AddP (CastX2P reg) off);
5525 op_cost(0);
5526 format %{ "[$reg, $off]\t# long -> ptr" %}
5527 interface(MEMORY_INTER) %{
5528 base($reg);
5529 index(0xffffffff);
5530 scale(0x0);
5531 disp($off);
5532 %}
5533 %}
5534
5535 operand indirectN(iRegN reg)
5536 %{
5537 predicate(CompressedOops::shift() == 0);
5538 constraint(ALLOC_IN_RC(ptr_reg));
5539 match(DecodeN reg);
5540 op_cost(0);
5541 format %{ "[$reg]\t# narrow" %}
5542 interface(MEMORY_INTER) %{
5543 base($reg);
5544 index(0xffffffff);
5545 scale(0x0);
5546 disp(0x0);
5547 %}
5548 %}
5549
5550 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5551 %{
5552 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5553 constraint(ALLOC_IN_RC(ptr_reg));
5554 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5555 op_cost(0);
5556 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5557 interface(MEMORY_INTER) %{
5558 base($reg);
5559 index($ireg);
5560 scale($scale);
5561 disp(0x0);
5562 %}
5563 %}
5564
5565 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5566 %{
5567 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5568 constraint(ALLOC_IN_RC(ptr_reg));
5569 match(AddP (DecodeN reg) (LShiftL lreg scale));
5570 op_cost(0);
5571 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5572 interface(MEMORY_INTER) %{
5573 base($reg);
5574 index($lreg);
5575 scale($scale);
5576 disp(0x0);
5577 %}
5578 %}
5579
5580 operand indIndexI2LN(iRegN reg, iRegI ireg)
5581 %{
5582 predicate(CompressedOops::shift() == 0);
5583 constraint(ALLOC_IN_RC(ptr_reg));
5584 match(AddP (DecodeN reg) (ConvI2L ireg));
5585 op_cost(0);
5586 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5587 interface(MEMORY_INTER) %{
5588 base($reg);
5589 index($ireg);
5590 scale(0x0);
5591 disp(0x0);
5592 %}
5593 %}
5594
5595 operand indIndexN(iRegN reg, iRegL lreg)
5596 %{
5597 predicate(CompressedOops::shift() == 0);
5598 constraint(ALLOC_IN_RC(ptr_reg));
5599 match(AddP (DecodeN reg) lreg);
5600 op_cost(0);
5601 format %{ "$reg, $lreg\t# narrow" %}
5602 interface(MEMORY_INTER) %{
5603 base($reg);
5604 index($lreg);
5605 scale(0x0);
5606 disp(0x0);
5607 %}
5608 %}
5609
5610 operand indOffIN(iRegN reg, immIOffset off)
5611 %{
5612 predicate(CompressedOops::shift() == 0);
5613 constraint(ALLOC_IN_RC(ptr_reg));
5614 match(AddP (DecodeN reg) off);
5615 op_cost(0);
5616 format %{ "[$reg, $off]\t# narrow" %}
5617 interface(MEMORY_INTER) %{
5618 base($reg);
5619 index(0xffffffff);
5620 scale(0x0);
5621 disp($off);
5622 %}
5623 %}
5624
5625 operand indOffLN(iRegN reg, immLOffset off)
5626 %{
5627 predicate(CompressedOops::shift() == 0);
5628 constraint(ALLOC_IN_RC(ptr_reg));
5629 match(AddP (DecodeN reg) off);
5630 op_cost(0);
5631 format %{ "[$reg, $off]\t# narrow" %}
5632 interface(MEMORY_INTER) %{
5633 base($reg);
5634 index(0xffffffff);
5635 scale(0x0);
5636 disp($off);
5637 %}
5638 %}
5639
5640
5641 //----------Special Memory Operands--------------------------------------------
5642 // Stack Slot Operand - This operand is used for loading and storing temporary
5643 // values on the stack where a match requires a value to
5644 // flow through memory.
5645 operand stackSlotP(sRegP reg)
5646 %{
5647 constraint(ALLOC_IN_RC(stack_slots));
5648 op_cost(100);
5649 // No match rule because this operand is only generated in matching
5650 // match(RegP);
5651 format %{ "[$reg]" %}
5652 interface(MEMORY_INTER) %{
5653 base(0x1e); // RSP
5654 index(0x0); // No Index
5655 scale(0x0); // No Scale
5656 disp($reg); // Stack Offset
5657 %}
5658 %}
5659
5660 operand stackSlotI(sRegI reg)
5661 %{
5662 constraint(ALLOC_IN_RC(stack_slots));
5663 // No match rule because this operand is only generated in matching
5664 // match(RegI);
5665 format %{ "[$reg]" %}
5666 interface(MEMORY_INTER) %{
5667 base(0x1e); // RSP
5668 index(0x0); // No Index
5669 scale(0x0); // No Scale
5670 disp($reg); // Stack Offset
5671 %}
5672 %}
5673
5674 operand stackSlotF(sRegF reg)
5675 %{
5676 constraint(ALLOC_IN_RC(stack_slots));
5677 // No match rule because this operand is only generated in matching
5678 // match(RegF);
5679 format %{ "[$reg]" %}
5680 interface(MEMORY_INTER) %{
5681 base(0x1e); // RSP
5682 index(0x0); // No Index
5683 scale(0x0); // No Scale
5684 disp($reg); // Stack Offset
5685 %}
5686 %}
5687
5688 operand stackSlotD(sRegD reg)
5689 %{
5690 constraint(ALLOC_IN_RC(stack_slots));
5691 // No match rule because this operand is only generated in matching
5692 // match(RegD);
5693 format %{ "[$reg]" %}
5694 interface(MEMORY_INTER) %{
5695 base(0x1e); // RSP
5696 index(0x0); // No Index
5697 scale(0x0); // No Scale
5698 disp($reg); // Stack Offset
5699 %}
5700 %}
5701
5702 operand stackSlotL(sRegL reg)
5703 %{
5704 constraint(ALLOC_IN_RC(stack_slots));
5705 // No match rule because this operand is only generated in matching
5706 // match(RegL);
5707 format %{ "[$reg]" %}
5708 interface(MEMORY_INTER) %{
5709 base(0x1e); // RSP
5710 index(0x0); // No Index
5711 scale(0x0); // No Scale
5712 disp($reg); // Stack Offset
5713 %}
5714 %}
5715
5716 // Operands for expressing Control Flow
5717 // NOTE: Label is a predefined operand which should not be redefined in
5718 // the AD file. It is generically handled within the ADLC.
5719
5720 //----------Conditional Branch Operands----------------------------------------
5721 // Comparison Op - This is the operation of the comparison, and is limited to
5722 // the following set of codes:
5723 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5724 //
5725 // Other attributes of the comparison, such as unsignedness, are specified
5726 // by the comparison instruction that sets a condition code flags register.
5727 // That result is represented by a flags operand whose subtype is appropriate
5728 // to the unsignedness (etc.) of the comparison.
5729 //
5730 // Later, the instruction which matches both the Comparison Op (a Bool) and
5731 // the flags (produced by the Cmp) specifies the coding of the comparison op
5732 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5733
5734 // used for signed integral comparisons and fp comparisons
5735
5736 operand cmpOp()
5737 %{
5738 match(Bool);
5739
5740 format %{ "" %}
5741 interface(COND_INTER) %{
5742 equal(0x0, "eq");
5743 not_equal(0x1, "ne");
5744 less(0xb, "lt");
5745 greater_equal(0xa, "ge");
5746 less_equal(0xd, "le");
5747 greater(0xc, "gt");
5748 overflow(0x6, "vs");
5749 no_overflow(0x7, "vc");
5750 %}
5751 %}
5752
5753 // used for unsigned integral comparisons
5754
5755 operand cmpOpU()
5756 %{
5757 match(Bool);
5758
5759 format %{ "" %}
5760 interface(COND_INTER) %{
5761 equal(0x0, "eq");
5762 not_equal(0x1, "ne");
5763 less(0x3, "lo");
5764 greater_equal(0x2, "hs");
5765 less_equal(0x9, "ls");
5766 greater(0x8, "hi");
5767 overflow(0x6, "vs");
5768 no_overflow(0x7, "vc");
5769 %}
5770 %}
5771
5772 // used for certain integral comparisons which can be
5773 // converted to cbxx or tbxx instructions
5774
5775 operand cmpOpEqNe()
5776 %{
5777 match(Bool);
5778 op_cost(0);
5779 predicate(n->as_Bool()->_test._test == BoolTest::ne
5780 || n->as_Bool()->_test._test == BoolTest::eq);
5781
5782 format %{ "" %}
5783 interface(COND_INTER) %{
5784 equal(0x0, "eq");
5785 not_equal(0x1, "ne");
5786 less(0xb, "lt");
5787 greater_equal(0xa, "ge");
5788 less_equal(0xd, "le");
5789 greater(0xc, "gt");
5790 overflow(0x6, "vs");
5791 no_overflow(0x7, "vc");
5792 %}
5793 %}
5794
5795 // used for certain integral comparisons which can be
5796 // converted to cbxx or tbxx instructions
5797
5798 operand cmpOpLtGe()
5799 %{
5800 match(Bool);
5801 op_cost(0);
5802
5803 predicate(n->as_Bool()->_test._test == BoolTest::lt
5804 || n->as_Bool()->_test._test == BoolTest::ge);
5805
5806 format %{ "" %}
5807 interface(COND_INTER) %{
5808 equal(0x0, "eq");
5809 not_equal(0x1, "ne");
5810 less(0xb, "lt");
5811 greater_equal(0xa, "ge");
5812 less_equal(0xd, "le");
5813 greater(0xc, "gt");
5814 overflow(0x6, "vs");
5815 no_overflow(0x7, "vc");
5816 %}
5817 %}
5818
5819 // used for certain unsigned integral comparisons which can be
5820 // converted to cbxx or tbxx instructions
5821
5822 operand cmpOpUEqNeLeGt()
5823 %{
5824 match(Bool);
5825 op_cost(0);
5826
5827 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5828 n->as_Bool()->_test._test == BoolTest::ne ||
5829 n->as_Bool()->_test._test == BoolTest::le ||
5830 n->as_Bool()->_test._test == BoolTest::gt);
5831
5832 format %{ "" %}
5833 interface(COND_INTER) %{
5834 equal(0x0, "eq");
5835 not_equal(0x1, "ne");
5836 less(0x3, "lo");
5837 greater_equal(0x2, "hs");
5838 less_equal(0x9, "ls");
5839 greater(0x8, "hi");
5840 overflow(0x6, "vs");
5841 no_overflow(0x7, "vc");
5842 %}
5843 %}
5844
5845 // Special operand allowing long args to int ops to be truncated for free
5846
5847 operand iRegL2I(iRegL reg) %{
5848
5849 op_cost(0);
5850
5851 match(ConvL2I reg);
5852
5853 format %{ "l2i($reg)" %}
5854
5855 interface(REG_INTER)
5856 %}
5857
5858 operand iRegL2P(iRegL reg) %{
5859
5860 op_cost(0);
5861
5862 match(CastX2P reg);
5863
5864 format %{ "l2p($reg)" %}
5865
5866 interface(REG_INTER)
5867 %}
5868
5869 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5870 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5871 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5872 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5873
5874 //----------OPERAND CLASSES----------------------------------------------------
5875 // Operand Classes are groups of operands that are used as to simplify
5876 // instruction definitions by not requiring the AD writer to specify
5877 // separate instructions for every form of operand when the
5878 // instruction accepts multiple operand types with the same basic
5879 // encoding and format. The classic case of this is memory operands.
5880
5881 // memory is used to define read/write location for load/store
5882 // instruction defs. we can turn a memory op into an Address
5883
5884 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5885 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5886
5887 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5888 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5889
5890 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5891 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5892
5893 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5894 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5895
5896 // All of the memory operands. For the pipeline description.
5897 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5898 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5899 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5900
5901
5902 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5903 // operations. it allows the src to be either an iRegI or a (ConvL2I
5904 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5905 // can be elided because the 32-bit instruction will just employ the
5906 // lower 32 bits anyway.
5907 //
5908 // n.b. this does not elide all L2I conversions. if the truncated
5909 // value is consumed by more than one operation then the ConvL2I
5910 // cannot be bundled into the consuming nodes so an l2i gets planted
5911 // (actually a movw $dst $src) and the downstream instructions consume
5912 // the result of the l2i as an iRegI input. That's a shame since the
5913 // movw is actually redundant but its not too costly.
5914
5915 opclass iRegIorL2I(iRegI, iRegL2I);
5916 opclass iRegPorL2P(iRegP, iRegL2P);
5917
5918 //----------PIPELINE-----------------------------------------------------------
5919 // Rules which define the behavior of the target architectures pipeline.
5920
5921 // For specific pipelines, eg A53, define the stages of that pipeline
5922 //pipe_desc(ISS, EX1, EX2, WR);
5923 #define ISS S0
5924 #define EX1 S1
5925 #define EX2 S2
5926 #define WR S3
5927
5928 // Integer ALU reg operation
5929 pipeline %{
5930
5931 attributes %{
5932 // ARM instructions are of fixed length
5933 fixed_size_instructions; // Fixed size instructions TODO does
5934 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5935 // ARM instructions come in 32-bit word units
5936 instruction_unit_size = 4; // An instruction is 4 bytes long
5937 instruction_fetch_unit_size = 64; // The processor fetches one line
5938 instruction_fetch_units = 1; // of 64 bytes
5939 %}
5940
5941 // We don't use an actual pipeline model so don't care about resources
5942 // or description. we do use pipeline classes to introduce fixed
5943 // latencies
5944
5945 //----------RESOURCES----------------------------------------------------------
5946 // Resources are the functional units available to the machine
5947
5948 resources( INS0, INS1, INS01 = INS0 | INS1,
5949 ALU0, ALU1, ALU = ALU0 | ALU1,
5950 MAC,
5951 DIV,
5952 BRANCH,
5953 LDST,
5954 NEON_FP);
5955
5956 //----------PIPELINE DESCRIPTION-----------------------------------------------
5957 // Pipeline Description specifies the stages in the machine's pipeline
5958
5959 // Define the pipeline as a generic 6 stage pipeline
5960 pipe_desc(S0, S1, S2, S3, S4, S5);
5961
5962 //----------PIPELINE CLASSES---------------------------------------------------
5963 // Pipeline Classes describe the stages in which input and output are
5964 // referenced by the hardware pipeline.
5965
5966 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5967 %{
5968 single_instruction;
5969 src1 : S1(read);
5970 src2 : S2(read);
5971 dst : S5(write);
5972 INS01 : ISS;
5973 NEON_FP : S5;
5974 %}
5975
5976 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5977 %{
5978 single_instruction;
5979 src1 : S1(read);
5980 src2 : S2(read);
5981 dst : S5(write);
5982 INS01 : ISS;
5983 NEON_FP : S5;
5984 %}
5985
5986 pipe_class fp_uop_s(vRegF dst, vRegF src)
5987 %{
5988 single_instruction;
5989 src : S1(read);
5990 dst : S5(write);
5991 INS01 : ISS;
5992 NEON_FP : S5;
5993 %}
5994
5995 pipe_class fp_uop_d(vRegD dst, vRegD src)
5996 %{
5997 single_instruction;
5998 src : S1(read);
5999 dst : S5(write);
6000 INS01 : ISS;
6001 NEON_FP : S5;
6002 %}
6003
6004 pipe_class fp_d2f(vRegF dst, vRegD src)
6005 %{
6006 single_instruction;
6007 src : S1(read);
6008 dst : S5(write);
6009 INS01 : ISS;
6010 NEON_FP : S5;
6011 %}
6012
6013 pipe_class fp_f2d(vRegD dst, vRegF src)
6014 %{
6015 single_instruction;
6016 src : S1(read);
6017 dst : S5(write);
6018 INS01 : ISS;
6019 NEON_FP : S5;
6020 %}
6021
6022 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6023 %{
6024 single_instruction;
6025 src : S1(read);
6026 dst : S5(write);
6027 INS01 : ISS;
6028 NEON_FP : S5;
6029 %}
6030
6031 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6032 %{
6033 single_instruction;
6034 src : S1(read);
6035 dst : S5(write);
6036 INS01 : ISS;
6037 NEON_FP : S5;
6038 %}
6039
6040 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6041 %{
6042 single_instruction;
6043 src : S1(read);
6044 dst : S5(write);
6045 INS01 : ISS;
6046 NEON_FP : S5;
6047 %}
6048
6049 pipe_class fp_l2f(vRegF dst, iRegL src)
6050 %{
6051 single_instruction;
6052 src : S1(read);
6053 dst : S5(write);
6054 INS01 : ISS;
6055 NEON_FP : S5;
6056 %}
6057
6058 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6059 %{
6060 single_instruction;
6061 src : S1(read);
6062 dst : S5(write);
6063 INS01 : ISS;
6064 NEON_FP : S5;
6065 %}
6066
6067 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6068 %{
6069 single_instruction;
6070 src : S1(read);
6071 dst : S5(write);
6072 INS01 : ISS;
6073 NEON_FP : S5;
6074 %}
6075
6076 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6077 %{
6078 single_instruction;
6079 src : S1(read);
6080 dst : S5(write);
6081 INS01 : ISS;
6082 NEON_FP : S5;
6083 %}
6084
6085 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6086 %{
6087 single_instruction;
6088 src : S1(read);
6089 dst : S5(write);
6090 INS01 : ISS;
6091 NEON_FP : S5;
6092 %}
6093
6094 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6095 %{
6096 single_instruction;
6097 src1 : S1(read);
6098 src2 : S2(read);
6099 dst : S5(write);
6100 INS0 : ISS;
6101 NEON_FP : S5;
6102 %}
6103
6104 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6105 %{
6106 single_instruction;
6107 src1 : S1(read);
6108 src2 : S2(read);
6109 dst : S5(write);
6110 INS0 : ISS;
6111 NEON_FP : S5;
6112 %}
6113
6114 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6115 %{
6116 single_instruction;
6117 cr : S1(read);
6118 src1 : S1(read);
6119 src2 : S1(read);
6120 dst : S3(write);
6121 INS01 : ISS;
6122 NEON_FP : S3;
6123 %}
6124
6125 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6126 %{
6127 single_instruction;
6128 cr : S1(read);
6129 src1 : S1(read);
6130 src2 : S1(read);
6131 dst : S3(write);
6132 INS01 : ISS;
6133 NEON_FP : S3;
6134 %}
6135
6136 pipe_class fp_imm_s(vRegF dst)
6137 %{
6138 single_instruction;
6139 dst : S3(write);
6140 INS01 : ISS;
6141 NEON_FP : S3;
6142 %}
6143
6144 pipe_class fp_imm_d(vRegD dst)
6145 %{
6146 single_instruction;
6147 dst : S3(write);
6148 INS01 : ISS;
6149 NEON_FP : S3;
6150 %}
6151
6152 pipe_class fp_load_constant_s(vRegF dst)
6153 %{
6154 single_instruction;
6155 dst : S4(write);
6156 INS01 : ISS;
6157 NEON_FP : S4;
6158 %}
6159
6160 pipe_class fp_load_constant_d(vRegD dst)
6161 %{
6162 single_instruction;
6163 dst : S4(write);
6164 INS01 : ISS;
6165 NEON_FP : S4;
6166 %}
6167
6168 //------- Integer ALU operations --------------------------
6169
6170 // Integer ALU reg-reg operation
6171 // Operands needed in EX1, result generated in EX2
6172 // Eg. ADD x0, x1, x2
6173 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6174 %{
6175 single_instruction;
6176 dst : EX2(write);
6177 src1 : EX1(read);
6178 src2 : EX1(read);
6179 INS01 : ISS; // Dual issue as instruction 0 or 1
6180 ALU : EX2;
6181 %}
6182
6183 // Integer ALU reg-reg operation with constant shift
6184 // Shifted register must be available in LATE_ISS instead of EX1
6185 // Eg. ADD x0, x1, x2, LSL #2
6186 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6187 %{
6188 single_instruction;
6189 dst : EX2(write);
6190 src1 : EX1(read);
6191 src2 : ISS(read);
6192 INS01 : ISS;
6193 ALU : EX2;
6194 %}
6195
6196 // Integer ALU reg operation with constant shift
6197 // Eg. LSL x0, x1, #shift
6198 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6199 %{
6200 single_instruction;
6201 dst : EX2(write);
6202 src1 : ISS(read);
6203 INS01 : ISS;
6204 ALU : EX2;
6205 %}
6206
6207 // Integer ALU reg-reg operation with variable shift
6208 // Both operands must be available in LATE_ISS instead of EX1
6209 // Result is available in EX1 instead of EX2
6210 // Eg. LSLV x0, x1, x2
6211 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6212 %{
6213 single_instruction;
6214 dst : EX1(write);
6215 src1 : ISS(read);
6216 src2 : ISS(read);
6217 INS01 : ISS;
6218 ALU : EX1;
6219 %}
6220
6221 // Integer ALU reg-reg operation with extract
6222 // As for _vshift above, but result generated in EX2
6223 // Eg. EXTR x0, x1, x2, #N
6224 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6225 %{
6226 single_instruction;
6227 dst : EX2(write);
6228 src1 : ISS(read);
6229 src2 : ISS(read);
6230 INS1 : ISS; // Can only dual issue as Instruction 1
6231 ALU : EX1;
6232 %}
6233
6234 // Integer ALU reg operation
6235 // Eg. NEG x0, x1
6236 pipe_class ialu_reg(iRegI dst, iRegI src)
6237 %{
6238 single_instruction;
6239 dst : EX2(write);
6240 src : EX1(read);
6241 INS01 : ISS;
6242 ALU : EX2;
6243 %}
6244
6245 // Integer ALU reg mmediate operation
6246 // Eg. ADD x0, x1, #N
6247 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6248 %{
6249 single_instruction;
6250 dst : EX2(write);
6251 src1 : EX1(read);
6252 INS01 : ISS;
6253 ALU : EX2;
6254 %}
6255
6256 // Integer ALU immediate operation (no source operands)
6257 // Eg. MOV x0, #N
6258 pipe_class ialu_imm(iRegI dst)
6259 %{
6260 single_instruction;
6261 dst : EX1(write);
6262 INS01 : ISS;
6263 ALU : EX1;
6264 %}
6265
6266 //------- Compare operation -------------------------------
6267
6268 // Compare reg-reg
6269 // Eg. CMP x0, x1
6270 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6271 %{
6272 single_instruction;
6273 // fixed_latency(16);
6274 cr : EX2(write);
6275 op1 : EX1(read);
6276 op2 : EX1(read);
6277 INS01 : ISS;
6278 ALU : EX2;
6279 %}
6280
6281 // Compare reg-reg
6282 // Eg. CMP x0, #N
6283 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6284 %{
6285 single_instruction;
6286 // fixed_latency(16);
6287 cr : EX2(write);
6288 op1 : EX1(read);
6289 INS01 : ISS;
6290 ALU : EX2;
6291 %}
6292
6293 //------- Conditional instructions ------------------------
6294
6295 // Conditional no operands
6296 // Eg. CSINC x0, zr, zr, <cond>
6297 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6298 %{
6299 single_instruction;
6300 cr : EX1(read);
6301 dst : EX2(write);
6302 INS01 : ISS;
6303 ALU : EX2;
6304 %}
6305
6306 // Conditional 2 operand
6307 // EG. CSEL X0, X1, X2, <cond>
6308 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6309 %{
6310 single_instruction;
6311 cr : EX1(read);
6312 src1 : EX1(read);
6313 src2 : EX1(read);
6314 dst : EX2(write);
6315 INS01 : ISS;
6316 ALU : EX2;
6317 %}
6318
6319 // Conditional 2 operand
6320 // EG. CSEL X0, X1, X2, <cond>
6321 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6322 %{
6323 single_instruction;
6324 cr : EX1(read);
6325 src : EX1(read);
6326 dst : EX2(write);
6327 INS01 : ISS;
6328 ALU : EX2;
6329 %}
6330
6331 //------- Multiply pipeline operations --------------------
6332
6333 // Multiply reg-reg
6334 // Eg. MUL w0, w1, w2
6335 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6336 %{
6337 single_instruction;
6338 dst : WR(write);
6339 src1 : ISS(read);
6340 src2 : ISS(read);
6341 INS01 : ISS;
6342 MAC : WR;
6343 %}
6344
6345 // Multiply accumulate
6346 // Eg. MADD w0, w1, w2, w3
6347 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6348 %{
6349 single_instruction;
6350 dst : WR(write);
6351 src1 : ISS(read);
6352 src2 : ISS(read);
6353 src3 : ISS(read);
6354 INS01 : ISS;
6355 MAC : WR;
6356 %}
6357
6358 // Eg. MUL w0, w1, w2
6359 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6360 %{
6361 single_instruction;
6362 fixed_latency(3); // Maximum latency for 64 bit mul
6363 dst : WR(write);
6364 src1 : ISS(read);
6365 src2 : ISS(read);
6366 INS01 : ISS;
6367 MAC : WR;
6368 %}
6369
6370 // Multiply accumulate
6371 // Eg. MADD w0, w1, w2, w3
6372 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6373 %{
6374 single_instruction;
6375 fixed_latency(3); // Maximum latency for 64 bit mul
6376 dst : WR(write);
6377 src1 : ISS(read);
6378 src2 : ISS(read);
6379 src3 : ISS(read);
6380 INS01 : ISS;
6381 MAC : WR;
6382 %}
6383
6384 //------- Divide pipeline operations --------------------
6385
6386 // Eg. SDIV w0, w1, w2
6387 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6388 %{
6389 single_instruction;
6390 fixed_latency(8); // Maximum latency for 32 bit divide
6391 dst : WR(write);
6392 src1 : ISS(read);
6393 src2 : ISS(read);
6394 INS0 : ISS; // Can only dual issue as instruction 0
6395 DIV : WR;
6396 %}
6397
6398 // Eg. SDIV x0, x1, x2
6399 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6400 %{
6401 single_instruction;
6402 fixed_latency(16); // Maximum latency for 64 bit divide
6403 dst : WR(write);
6404 src1 : ISS(read);
6405 src2 : ISS(read);
6406 INS0 : ISS; // Can only dual issue as instruction 0
6407 DIV : WR;
6408 %}
6409
6410 //------- Load pipeline operations ------------------------
6411
6412 // Load - prefetch
6413 // Eg. PFRM <mem>
6414 pipe_class iload_prefetch(memory mem)
6415 %{
6416 single_instruction;
6417 mem : ISS(read);
6418 INS01 : ISS;
6419 LDST : WR;
6420 %}
6421
6422 // Load - reg, mem
6423 // Eg. LDR x0, <mem>
6424 pipe_class iload_reg_mem(iRegI dst, memory mem)
6425 %{
6426 single_instruction;
6427 dst : WR(write);
6428 mem : ISS(read);
6429 INS01 : ISS;
6430 LDST : WR;
6431 %}
6432
6433 // Load - reg, reg
6434 // Eg. LDR x0, [sp, x1]
6435 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6436 %{
6437 single_instruction;
6438 dst : WR(write);
6439 src : ISS(read);
6440 INS01 : ISS;
6441 LDST : WR;
6442 %}
6443
6444 //------- Store pipeline operations -----------------------
6445
6446 // Store - zr, mem
6447 // Eg. STR zr, <mem>
6448 pipe_class istore_mem(memory mem)
6449 %{
6450 single_instruction;
6451 mem : ISS(read);
6452 INS01 : ISS;
6453 LDST : WR;
6454 %}
6455
6456 // Store - reg, mem
6457 // Eg. STR x0, <mem>
6458 pipe_class istore_reg_mem(iRegI src, memory mem)
6459 %{
6460 single_instruction;
6461 mem : ISS(read);
6462 src : EX2(read);
6463 INS01 : ISS;
6464 LDST : WR;
6465 %}
6466
6467 // Store - reg, reg
6468 // Eg. STR x0, [sp, x1]
6469 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6470 %{
6471 single_instruction;
6472 dst : ISS(read);
6473 src : EX2(read);
6474 INS01 : ISS;
6475 LDST : WR;
6476 %}
6477
6478 //------- Store pipeline operations -----------------------
6479
6480 // Branch
6481 pipe_class pipe_branch()
6482 %{
6483 single_instruction;
6484 INS01 : ISS;
6485 BRANCH : EX1;
6486 %}
6487
6488 // Conditional branch
6489 pipe_class pipe_branch_cond(rFlagsReg cr)
6490 %{
6491 single_instruction;
6492 cr : EX1(read);
6493 INS01 : ISS;
6494 BRANCH : EX1;
6495 %}
6496
6497 // Compare & Branch
6498 // EG. CBZ/CBNZ
6499 pipe_class pipe_cmp_branch(iRegI op1)
6500 %{
6501 single_instruction;
6502 op1 : EX1(read);
6503 INS01 : ISS;
6504 BRANCH : EX1;
6505 %}
6506
6507 //------- Synchronisation operations ----------------------
6508
6509 // Any operation requiring serialization.
6510 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6511 pipe_class pipe_serial()
6512 %{
6513 single_instruction;
6514 force_serialization;
6515 fixed_latency(16);
6516 INS01 : ISS(2); // Cannot dual issue with any other instruction
6517 LDST : WR;
6518 %}
6519
6520 // Generic big/slow expanded idiom - also serialized
6521 pipe_class pipe_slow()
6522 %{
6523 instruction_count(10);
6524 multiple_bundles;
6525 force_serialization;
6526 fixed_latency(16);
6527 INS01 : ISS(2); // Cannot dual issue with any other instruction
6528 LDST : WR;
6529 %}
6530
6531 // Empty pipeline class
6532 pipe_class pipe_class_empty()
6533 %{
6534 single_instruction;
6535 fixed_latency(0);
6536 %}
6537
6538 // Default pipeline class.
6539 pipe_class pipe_class_default()
6540 %{
6541 single_instruction;
6542 fixed_latency(2);
6543 %}
6544
6545 // Pipeline class for compares.
6546 pipe_class pipe_class_compare()
6547 %{
6548 single_instruction;
6549 fixed_latency(16);
6550 %}
6551
6552 // Pipeline class for memory operations.
6553 pipe_class pipe_class_memory()
6554 %{
6555 single_instruction;
6556 fixed_latency(16);
6557 %}
6558
6559 // Pipeline class for call.
6560 pipe_class pipe_class_call()
6561 %{
6562 single_instruction;
6563 fixed_latency(100);
6564 %}
6565
6566 // Define the class for the Nop node.
6567 define %{
6568 MachNop = pipe_class_empty;
6569 %}
6570
6571 %}
6572 //----------INSTRUCTIONS-------------------------------------------------------
6573 //
6574 // match -- States which machine-independent subtree may be replaced
6575 // by this instruction.
6576 // ins_cost -- The estimated cost of this instruction is used by instruction
6577 // selection to identify a minimum cost tree of machine
6578 // instructions that matches a tree of machine-independent
6579 // instructions.
6580 // format -- A string providing the disassembly for this instruction.
6581 // The value of an instruction's operand may be inserted
6582 // by referring to it with a '$' prefix.
6583 // opcode -- Three instruction opcodes may be provided. These are referred
6584 // to within an encode class as $primary, $secondary, and $tertiary
6585 // rrspectively. The primary opcode is commonly used to
6586 // indicate the type of machine instruction, while secondary
6587 // and tertiary are often used for prefix options or addressing
6588 // modes.
6589 // ins_encode -- A list of encode classes with parameters. The encode class
6590 // name must have been defined in an 'enc_class' specification
6591 // in the encode section of the architecture description.
6592
6593 // ============================================================================
6594 // Memory (Load/Store) Instructions
6595
6596 // Load Instructions
6597
6598 // Load Byte (8 bit signed)
6599 instruct loadB(iRegINoSp dst, memory1 mem)
6600 %{
6601 match(Set dst (LoadB mem));
6602 predicate(!needs_acquiring_load(n));
6603
6604 ins_cost(4 * INSN_COST);
6605 format %{ "ldrsbw $dst, $mem\t# byte" %}
6606
6607 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6608
6609 ins_pipe(iload_reg_mem);
6610 %}
6611
6612 // Load Byte (8 bit signed) into long
6613 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6614 %{
6615 match(Set dst (ConvI2L (LoadB mem)));
6616 predicate(!needs_acquiring_load(n->in(1)));
6617
6618 ins_cost(4 * INSN_COST);
6619 format %{ "ldrsb $dst, $mem\t# byte" %}
6620
6621 ins_encode(aarch64_enc_ldrsb(dst, mem));
6622
6623 ins_pipe(iload_reg_mem);
6624 %}
6625
6626 // Load Byte (8 bit unsigned)
6627 instruct loadUB(iRegINoSp dst, memory1 mem)
6628 %{
6629 match(Set dst (LoadUB mem));
6630 predicate(!needs_acquiring_load(n));
6631
6632 ins_cost(4 * INSN_COST);
6633 format %{ "ldrbw $dst, $mem\t# byte" %}
6634
6635 ins_encode(aarch64_enc_ldrb(dst, mem));
6636
6637 ins_pipe(iload_reg_mem);
6638 %}
6639
6640 // Load Byte (8 bit unsigned) into long
6641 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6642 %{
6643 match(Set dst (ConvI2L (LoadUB mem)));
6644 predicate(!needs_acquiring_load(n->in(1)));
6645
6646 ins_cost(4 * INSN_COST);
6647 format %{ "ldrb $dst, $mem\t# byte" %}
6648
6649 ins_encode(aarch64_enc_ldrb(dst, mem));
6650
6651 ins_pipe(iload_reg_mem);
6652 %}
6653
6654 // Load Short (16 bit signed)
6655 instruct loadS(iRegINoSp dst, memory2 mem)
6656 %{
6657 match(Set dst (LoadS mem));
6658 predicate(!needs_acquiring_load(n));
6659
6660 ins_cost(4 * INSN_COST);
6661 format %{ "ldrshw $dst, $mem\t# short" %}
6662
6663 ins_encode(aarch64_enc_ldrshw(dst, mem));
6664
6665 ins_pipe(iload_reg_mem);
6666 %}
6667
6668 // Load Short (16 bit signed) into long
6669 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6670 %{
6671 match(Set dst (ConvI2L (LoadS mem)));
6672 predicate(!needs_acquiring_load(n->in(1)));
6673
6674 ins_cost(4 * INSN_COST);
6675 format %{ "ldrsh $dst, $mem\t# short" %}
6676
6677 ins_encode(aarch64_enc_ldrsh(dst, mem));
6678
6679 ins_pipe(iload_reg_mem);
6680 %}
6681
6682 // Load Char (16 bit unsigned)
6683 instruct loadUS(iRegINoSp dst, memory2 mem)
6684 %{
6685 match(Set dst (LoadUS mem));
6686 predicate(!needs_acquiring_load(n));
6687
6688 ins_cost(4 * INSN_COST);
6689 format %{ "ldrh $dst, $mem\t# short" %}
6690
6691 ins_encode(aarch64_enc_ldrh(dst, mem));
6692
6693 ins_pipe(iload_reg_mem);
6694 %}
6695
6696 // Load Short/Char (16 bit unsigned) into long
6697 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6698 %{
6699 match(Set dst (ConvI2L (LoadUS mem)));
6700 predicate(!needs_acquiring_load(n->in(1)));
6701
6702 ins_cost(4 * INSN_COST);
6703 format %{ "ldrh $dst, $mem\t# short" %}
6704
6705 ins_encode(aarch64_enc_ldrh(dst, mem));
6706
6707 ins_pipe(iload_reg_mem);
6708 %}
6709
6710 // Load Integer (32 bit signed)
6711 instruct loadI(iRegINoSp dst, memory4 mem)
6712 %{
6713 match(Set dst (LoadI mem));
6714 predicate(!needs_acquiring_load(n));
6715
6716 ins_cost(4 * INSN_COST);
6717 format %{ "ldrw $dst, $mem\t# int" %}
6718
6719 ins_encode(aarch64_enc_ldrw(dst, mem));
6720
6721 ins_pipe(iload_reg_mem);
6722 %}
6723
6724 // Load Integer (32 bit signed) into long
6725 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6726 %{
6727 match(Set dst (ConvI2L (LoadI mem)));
6728 predicate(!needs_acquiring_load(n->in(1)));
6729
6730 ins_cost(4 * INSN_COST);
6731 format %{ "ldrsw $dst, $mem\t# int" %}
6732
6733 ins_encode(aarch64_enc_ldrsw(dst, mem));
6734
6735 ins_pipe(iload_reg_mem);
6736 %}
6737
6738 // Load Integer (32 bit unsigned) into long
6739 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6740 %{
6741 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6742 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6743
6744 ins_cost(4 * INSN_COST);
6745 format %{ "ldrw $dst, $mem\t# int" %}
6746
6747 ins_encode(aarch64_enc_ldrw(dst, mem));
6748
6749 ins_pipe(iload_reg_mem);
6750 %}
6751
6752 // Load Long (64 bit signed)
6753 instruct loadL(iRegLNoSp dst, memory8 mem)
6754 %{
6755 match(Set dst (LoadL mem));
6756 predicate(!needs_acquiring_load(n));
6757
6758 ins_cost(4 * INSN_COST);
6759 format %{ "ldr $dst, $mem\t# int" %}
6760
6761 ins_encode(aarch64_enc_ldr(dst, mem));
6762
6763 ins_pipe(iload_reg_mem);
6764 %}
6765
6766 // Load Range
6767 instruct loadRange(iRegINoSp dst, memory4 mem)
6768 %{
6769 match(Set dst (LoadRange mem));
6770
6771 ins_cost(4 * INSN_COST);
6772 format %{ "ldrw $dst, $mem\t# range" %}
6773
6774 ins_encode(aarch64_enc_ldrw(dst, mem));
6775
6776 ins_pipe(iload_reg_mem);
6777 %}
6778
6779 // Load Pointer
6780 instruct loadP(iRegPNoSp dst, memory8 mem)
6781 %{
6782 match(Set dst (LoadP mem));
6783 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6784
6785 ins_cost(4 * INSN_COST);
6786 format %{ "ldr $dst, $mem\t# ptr" %}
6787
6788 ins_encode(aarch64_enc_ldr(dst, mem));
6789
6790 ins_pipe(iload_reg_mem);
6791 %}
6792
6793 // Load Compressed Pointer
6794 instruct loadN(iRegNNoSp dst, memory4 mem)
6795 %{
6796 match(Set dst (LoadN mem));
6797 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6798
6799 ins_cost(4 * INSN_COST);
6800 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6801
6802 ins_encode(aarch64_enc_ldrw(dst, mem));
6803
6804 ins_pipe(iload_reg_mem);
6805 %}
6806
6807 // Load Klass Pointer
6808 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6809 %{
6810 match(Set dst (LoadKlass mem));
6811 predicate(!needs_acquiring_load(n));
6812
6813 ins_cost(4 * INSN_COST);
6814 format %{ "ldr $dst, $mem\t# class" %}
6815
6816 ins_encode(aarch64_enc_ldr(dst, mem));
6817
6818 ins_pipe(iload_reg_mem);
6819 %}
6820
6821 // Load Narrow Klass Pointer
6822 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6823 %{
6824 match(Set dst (LoadNKlass mem));
6825 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6826
6827 ins_cost(4 * INSN_COST);
6828 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6829
6830 ins_encode(aarch64_enc_ldrw(dst, mem));
6831
6832 ins_pipe(iload_reg_mem);
6833 %}
6834
6835 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6836 %{
6837 match(Set dst (LoadNKlass mem));
6838 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6839
6840 ins_cost(4 * INSN_COST);
6841 format %{
6842 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6843 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6844 %}
6845 ins_encode %{
6846 // inlined aarch64_enc_ldrw
6847 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6848 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6849 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6850 %}
6851 ins_pipe(iload_reg_mem);
6852 %}
6853
6854 // Load Float
6855 instruct loadF(vRegF dst, memory4 mem)
6856 %{
6857 match(Set dst (LoadF mem));
6858 predicate(!needs_acquiring_load(n));
6859
6860 ins_cost(4 * INSN_COST);
6861 format %{ "ldrs $dst, $mem\t# float" %}
6862
6863 ins_encode( aarch64_enc_ldrs(dst, mem) );
6864
6865 ins_pipe(pipe_class_memory);
6866 %}
6867
6868 // Load Double
6869 instruct loadD(vRegD dst, memory8 mem)
6870 %{
6871 match(Set dst (LoadD mem));
6872 predicate(!needs_acquiring_load(n));
6873
6874 ins_cost(4 * INSN_COST);
6875 format %{ "ldrd $dst, $mem\t# double" %}
6876
6877 ins_encode( aarch64_enc_ldrd(dst, mem) );
6878
6879 ins_pipe(pipe_class_memory);
6880 %}
6881
6882
6883 // Load Int Constant
6884 instruct loadConI(iRegINoSp dst, immI src)
6885 %{
6886 match(Set dst src);
6887
6888 ins_cost(INSN_COST);
6889 format %{ "mov $dst, $src\t# int" %}
6890
6891 ins_encode( aarch64_enc_movw_imm(dst, src) );
6892
6893 ins_pipe(ialu_imm);
6894 %}
6895
6896 // Load Long Constant
6897 instruct loadConL(iRegLNoSp dst, immL src)
6898 %{
6899 match(Set dst src);
6900
6901 ins_cost(INSN_COST);
6902 format %{ "mov $dst, $src\t# long" %}
6903
6904 ins_encode( aarch64_enc_mov_imm(dst, src) );
6905
6906 ins_pipe(ialu_imm);
6907 %}
6908
6909 // Load Pointer Constant
6910
6911 instruct loadConP(iRegPNoSp dst, immP con)
6912 %{
6913 match(Set dst con);
6914
6915 ins_cost(INSN_COST * 4);
6916 format %{
6917 "mov $dst, $con\t# ptr\n\t"
6918 %}
6919
6920 ins_encode(aarch64_enc_mov_p(dst, con));
6921
6922 ins_pipe(ialu_imm);
6923 %}
6924
6925 // Load Null Pointer Constant
6926
6927 instruct loadConP0(iRegPNoSp dst, immP0 con)
6928 %{
6929 match(Set dst con);
6930
6931 ins_cost(INSN_COST);
6932 format %{ "mov $dst, $con\t# nullptr ptr" %}
6933
6934 ins_encode(aarch64_enc_mov_p0(dst, con));
6935
6936 ins_pipe(ialu_imm);
6937 %}
6938
6939 // Load Pointer Constant One
6940
6941 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6942 %{
6943 match(Set dst con);
6944
6945 ins_cost(INSN_COST);
6946 format %{ "mov $dst, $con\t# nullptr ptr" %}
6947
6948 ins_encode(aarch64_enc_mov_p1(dst, con));
6949
6950 ins_pipe(ialu_imm);
6951 %}
6952
6953 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6954 %{
6955 match(Set dst con);
6956
6957 ins_cost(INSN_COST);
6958 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6959
6960 ins_encode %{
6961 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6962 %}
6963
6964 ins_pipe(ialu_imm);
6965 %}
6966
6967 // Load Narrow Pointer Constant
6968
6969 instruct loadConN(iRegNNoSp dst, immN con)
6970 %{
6971 match(Set dst con);
6972
6973 ins_cost(INSN_COST * 4);
6974 format %{ "mov $dst, $con\t# compressed ptr" %}
6975
6976 ins_encode(aarch64_enc_mov_n(dst, con));
6977
6978 ins_pipe(ialu_imm);
6979 %}
6980
6981 // Load Narrow Null Pointer Constant
6982
6983 instruct loadConN0(iRegNNoSp dst, immN0 con)
6984 %{
6985 match(Set dst con);
6986
6987 ins_cost(INSN_COST);
6988 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6989
6990 ins_encode(aarch64_enc_mov_n0(dst, con));
6991
6992 ins_pipe(ialu_imm);
6993 %}
6994
6995 // Load Narrow Klass Constant
6996
6997 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6998 %{
6999 match(Set dst con);
7000
7001 ins_cost(INSN_COST);
7002 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7003
7004 ins_encode(aarch64_enc_mov_nk(dst, con));
7005
7006 ins_pipe(ialu_imm);
7007 %}
7008
7009 // Load Packed Float Constant
7010
7011 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7012 match(Set dst con);
7013 ins_cost(INSN_COST * 4);
7014 format %{ "fmovs $dst, $con"%}
7015 ins_encode %{
7016 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7017 %}
7018
7019 ins_pipe(fp_imm_s);
7020 %}
7021
7022 // Load Float Constant
7023
7024 instruct loadConF(vRegF dst, immF con) %{
7025 match(Set dst con);
7026
7027 ins_cost(INSN_COST * 4);
7028
7029 format %{
7030 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7031 %}
7032
7033 ins_encode %{
7034 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7035 %}
7036
7037 ins_pipe(fp_load_constant_s);
7038 %}
7039
7040 // Load Packed Double Constant
7041
7042 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7043 match(Set dst con);
7044 ins_cost(INSN_COST);
7045 format %{ "fmovd $dst, $con"%}
7046 ins_encode %{
7047 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7048 %}
7049
7050 ins_pipe(fp_imm_d);
7051 %}
7052
7053 // Load Double Constant
7054
7055 instruct loadConD(vRegD dst, immD con) %{
7056 match(Set dst con);
7057
7058 ins_cost(INSN_COST * 5);
7059 format %{
7060 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7061 %}
7062
7063 ins_encode %{
7064 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7065 %}
7066
7067 ins_pipe(fp_load_constant_d);
7068 %}
7069
7070 // Load Half Float Constant
7071 instruct loadConH(vRegF dst, immH con) %{
7072 match(Set dst con);
7073 format %{ "mov rscratch1, $con\n\t"
7074 "fmov $dst, rscratch1"
7075 %}
7076 ins_encode %{
7077 __ movw(rscratch1, (uint32_t)$con$$constant);
7078 __ fmovs($dst$$FloatRegister, rscratch1);
7079 %}
7080 ins_pipe(pipe_class_default);
7081 %}
7082
7083 // Store Instructions
7084
7085 // Store Byte
7086 instruct storeB(iRegIorL2I src, memory1 mem)
7087 %{
7088 match(Set mem (StoreB mem src));
7089 predicate(!needs_releasing_store(n));
7090
7091 ins_cost(INSN_COST);
7092 format %{ "strb $src, $mem\t# byte" %}
7093
7094 ins_encode(aarch64_enc_strb(src, mem));
7095
7096 ins_pipe(istore_reg_mem);
7097 %}
7098
7099
7100 instruct storeimmB0(immI0 zero, memory1 mem)
7101 %{
7102 match(Set mem (StoreB mem zero));
7103 predicate(!needs_releasing_store(n));
7104
7105 ins_cost(INSN_COST);
7106 format %{ "strb rscractch2, $mem\t# byte" %}
7107
7108 ins_encode(aarch64_enc_strb0(mem));
7109
7110 ins_pipe(istore_mem);
7111 %}
7112
7113 // Store Char/Short
7114 instruct storeC(iRegIorL2I src, memory2 mem)
7115 %{
7116 match(Set mem (StoreC mem src));
7117 predicate(!needs_releasing_store(n));
7118
7119 ins_cost(INSN_COST);
7120 format %{ "strh $src, $mem\t# short" %}
7121
7122 ins_encode(aarch64_enc_strh(src, mem));
7123
7124 ins_pipe(istore_reg_mem);
7125 %}
7126
7127 instruct storeimmC0(immI0 zero, memory2 mem)
7128 %{
7129 match(Set mem (StoreC mem zero));
7130 predicate(!needs_releasing_store(n));
7131
7132 ins_cost(INSN_COST);
7133 format %{ "strh zr, $mem\t# short" %}
7134
7135 ins_encode(aarch64_enc_strh0(mem));
7136
7137 ins_pipe(istore_mem);
7138 %}
7139
7140 // Store Integer
7141
7142 instruct storeI(iRegIorL2I src, memory4 mem)
7143 %{
7144 match(Set mem(StoreI mem src));
7145 predicate(!needs_releasing_store(n));
7146
7147 ins_cost(INSN_COST);
7148 format %{ "strw $src, $mem\t# int" %}
7149
7150 ins_encode(aarch64_enc_strw(src, mem));
7151
7152 ins_pipe(istore_reg_mem);
7153 %}
7154
7155 instruct storeimmI0(immI0 zero, memory4 mem)
7156 %{
7157 match(Set mem(StoreI mem zero));
7158 predicate(!needs_releasing_store(n));
7159
7160 ins_cost(INSN_COST);
7161 format %{ "strw zr, $mem\t# int" %}
7162
7163 ins_encode(aarch64_enc_strw0(mem));
7164
7165 ins_pipe(istore_mem);
7166 %}
7167
7168 // Store Long (64 bit signed)
7169 instruct storeL(iRegL src, memory8 mem)
7170 %{
7171 match(Set mem (StoreL mem src));
7172 predicate(!needs_releasing_store(n));
7173
7174 ins_cost(INSN_COST);
7175 format %{ "str $src, $mem\t# int" %}
7176
7177 ins_encode(aarch64_enc_str(src, mem));
7178
7179 ins_pipe(istore_reg_mem);
7180 %}
7181
7182 // Store Long (64 bit signed)
7183 instruct storeimmL0(immL0 zero, memory8 mem)
7184 %{
7185 match(Set mem (StoreL mem zero));
7186 predicate(!needs_releasing_store(n));
7187
7188 ins_cost(INSN_COST);
7189 format %{ "str zr, $mem\t# int" %}
7190
7191 ins_encode(aarch64_enc_str0(mem));
7192
7193 ins_pipe(istore_mem);
7194 %}
7195
7196 // Store Pointer
7197 instruct storeP(iRegP src, memory8 mem)
7198 %{
7199 match(Set mem (StoreP mem src));
7200 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7201
7202 ins_cost(INSN_COST);
7203 format %{ "str $src, $mem\t# ptr" %}
7204
7205 ins_encode(aarch64_enc_str(src, mem));
7206
7207 ins_pipe(istore_reg_mem);
7208 %}
7209
7210 // Store Pointer
7211 instruct storeimmP0(immP0 zero, memory8 mem)
7212 %{
7213 match(Set mem (StoreP mem zero));
7214 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7215
7216 ins_cost(INSN_COST);
7217 format %{ "str zr, $mem\t# ptr" %}
7218
7219 ins_encode(aarch64_enc_str0(mem));
7220
7221 ins_pipe(istore_mem);
7222 %}
7223
7224 // Store Compressed Pointer
7225 instruct storeN(iRegN src, memory4 mem)
7226 %{
7227 match(Set mem (StoreN mem src));
7228 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7229
7230 ins_cost(INSN_COST);
7231 format %{ "strw $src, $mem\t# compressed ptr" %}
7232
7233 ins_encode(aarch64_enc_strw(src, mem));
7234
7235 ins_pipe(istore_reg_mem);
7236 %}
7237
7238 instruct storeImmN0(immN0 zero, memory4 mem)
7239 %{
7240 match(Set mem (StoreN mem zero));
7241 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7242
7243 ins_cost(INSN_COST);
7244 format %{ "strw zr, $mem\t# compressed ptr" %}
7245
7246 ins_encode(aarch64_enc_strw0(mem));
7247
7248 ins_pipe(istore_mem);
7249 %}
7250
7251 // Store Float
7252 instruct storeF(vRegF src, memory4 mem)
7253 %{
7254 match(Set mem (StoreF mem src));
7255 predicate(!needs_releasing_store(n));
7256
7257 ins_cost(INSN_COST);
7258 format %{ "strs $src, $mem\t# float" %}
7259
7260 ins_encode( aarch64_enc_strs(src, mem) );
7261
7262 ins_pipe(pipe_class_memory);
7263 %}
7264
7265 // TODO
7266 // implement storeImmF0 and storeFImmPacked
7267
7268 // Store Double
7269 instruct storeD(vRegD src, memory8 mem)
7270 %{
7271 match(Set mem (StoreD mem src));
7272 predicate(!needs_releasing_store(n));
7273
7274 ins_cost(INSN_COST);
7275 format %{ "strd $src, $mem\t# double" %}
7276
7277 ins_encode( aarch64_enc_strd(src, mem) );
7278
7279 ins_pipe(pipe_class_memory);
7280 %}
7281
7282 // Store Compressed Klass Pointer
7283 instruct storeNKlass(iRegN src, memory4 mem)
7284 %{
7285 predicate(!needs_releasing_store(n));
7286 match(Set mem (StoreNKlass mem src));
7287
7288 ins_cost(INSN_COST);
7289 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7290
7291 ins_encode(aarch64_enc_strw(src, mem));
7292
7293 ins_pipe(istore_reg_mem);
7294 %}
7295
7296 // TODO
7297 // implement storeImmD0 and storeDImmPacked
7298
7299 // prefetch instructions
7300 // Must be safe to execute with invalid address (cannot fault).
7301
7302 instruct prefetchalloc( memory8 mem ) %{
7303 match(PrefetchAllocation mem);
7304
7305 ins_cost(INSN_COST);
7306 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7307
7308 ins_encode( aarch64_enc_prefetchw(mem) );
7309
7310 ins_pipe(iload_prefetch);
7311 %}
7312
7313 // ---------------- volatile loads and stores ----------------
7314
7315 // Load Byte (8 bit signed)
7316 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7317 %{
7318 match(Set dst (LoadB mem));
7319
7320 ins_cost(VOLATILE_REF_COST);
7321 format %{ "ldarsb $dst, $mem\t# byte" %}
7322
7323 ins_encode(aarch64_enc_ldarsb(dst, mem));
7324
7325 ins_pipe(pipe_serial);
7326 %}
7327
7328 // Load Byte (8 bit signed) into long
7329 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7330 %{
7331 match(Set dst (ConvI2L (LoadB mem)));
7332
7333 ins_cost(VOLATILE_REF_COST);
7334 format %{ "ldarsb $dst, $mem\t# byte" %}
7335
7336 ins_encode(aarch64_enc_ldarsb(dst, mem));
7337
7338 ins_pipe(pipe_serial);
7339 %}
7340
7341 // Load Byte (8 bit unsigned)
7342 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7343 %{
7344 match(Set dst (LoadUB mem));
7345
7346 ins_cost(VOLATILE_REF_COST);
7347 format %{ "ldarb $dst, $mem\t# byte" %}
7348
7349 ins_encode(aarch64_enc_ldarb(dst, mem));
7350
7351 ins_pipe(pipe_serial);
7352 %}
7353
7354 // Load Byte (8 bit unsigned) into long
7355 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7356 %{
7357 match(Set dst (ConvI2L (LoadUB mem)));
7358
7359 ins_cost(VOLATILE_REF_COST);
7360 format %{ "ldarb $dst, $mem\t# byte" %}
7361
7362 ins_encode(aarch64_enc_ldarb(dst, mem));
7363
7364 ins_pipe(pipe_serial);
7365 %}
7366
7367 // Load Short (16 bit signed)
7368 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7369 %{
7370 match(Set dst (LoadS mem));
7371
7372 ins_cost(VOLATILE_REF_COST);
7373 format %{ "ldarshw $dst, $mem\t# short" %}
7374
7375 ins_encode(aarch64_enc_ldarshw(dst, mem));
7376
7377 ins_pipe(pipe_serial);
7378 %}
7379
7380 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7381 %{
7382 match(Set dst (LoadUS mem));
7383
7384 ins_cost(VOLATILE_REF_COST);
7385 format %{ "ldarhw $dst, $mem\t# short" %}
7386
7387 ins_encode(aarch64_enc_ldarhw(dst, mem));
7388
7389 ins_pipe(pipe_serial);
7390 %}
7391
7392 // Load Short/Char (16 bit unsigned) into long
7393 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7394 %{
7395 match(Set dst (ConvI2L (LoadUS mem)));
7396
7397 ins_cost(VOLATILE_REF_COST);
7398 format %{ "ldarh $dst, $mem\t# short" %}
7399
7400 ins_encode(aarch64_enc_ldarh(dst, mem));
7401
7402 ins_pipe(pipe_serial);
7403 %}
7404
7405 // Load Short/Char (16 bit signed) into long
7406 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7407 %{
7408 match(Set dst (ConvI2L (LoadS mem)));
7409
7410 ins_cost(VOLATILE_REF_COST);
7411 format %{ "ldarh $dst, $mem\t# short" %}
7412
7413 ins_encode(aarch64_enc_ldarsh(dst, mem));
7414
7415 ins_pipe(pipe_serial);
7416 %}
7417
7418 // Load Integer (32 bit signed)
7419 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7420 %{
7421 match(Set dst (LoadI mem));
7422
7423 ins_cost(VOLATILE_REF_COST);
7424 format %{ "ldarw $dst, $mem\t# int" %}
7425
7426 ins_encode(aarch64_enc_ldarw(dst, mem));
7427
7428 ins_pipe(pipe_serial);
7429 %}
7430
7431 // Load Integer (32 bit unsigned) into long
7432 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7433 %{
7434 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7435
7436 ins_cost(VOLATILE_REF_COST);
7437 format %{ "ldarw $dst, $mem\t# int" %}
7438
7439 ins_encode(aarch64_enc_ldarw(dst, mem));
7440
7441 ins_pipe(pipe_serial);
7442 %}
7443
7444 // Load Long (64 bit signed)
7445 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7446 %{
7447 match(Set dst (LoadL mem));
7448
7449 ins_cost(VOLATILE_REF_COST);
7450 format %{ "ldar $dst, $mem\t# int" %}
7451
7452 ins_encode(aarch64_enc_ldar(dst, mem));
7453
7454 ins_pipe(pipe_serial);
7455 %}
7456
7457 // Load Pointer
7458 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7459 %{
7460 match(Set dst (LoadP mem));
7461 predicate(n->as_Load()->barrier_data() == 0);
7462
7463 ins_cost(VOLATILE_REF_COST);
7464 format %{ "ldar $dst, $mem\t# ptr" %}
7465
7466 ins_encode(aarch64_enc_ldar(dst, mem));
7467
7468 ins_pipe(pipe_serial);
7469 %}
7470
7471 // Load Compressed Pointer
7472 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7473 %{
7474 match(Set dst (LoadN mem));
7475 predicate(n->as_Load()->barrier_data() == 0);
7476
7477 ins_cost(VOLATILE_REF_COST);
7478 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7479
7480 ins_encode(aarch64_enc_ldarw(dst, mem));
7481
7482 ins_pipe(pipe_serial);
7483 %}
7484
7485 // Load Float
7486 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7487 %{
7488 match(Set dst (LoadF mem));
7489
7490 ins_cost(VOLATILE_REF_COST);
7491 format %{ "ldars $dst, $mem\t# float" %}
7492
7493 ins_encode( aarch64_enc_fldars(dst, mem) );
7494
7495 ins_pipe(pipe_serial);
7496 %}
7497
7498 // Load Double
7499 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7500 %{
7501 match(Set dst (LoadD mem));
7502
7503 ins_cost(VOLATILE_REF_COST);
7504 format %{ "ldard $dst, $mem\t# double" %}
7505
7506 ins_encode( aarch64_enc_fldard(dst, mem) );
7507
7508 ins_pipe(pipe_serial);
7509 %}
7510
7511 // Store Byte
7512 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7513 %{
7514 match(Set mem (StoreB mem src));
7515
7516 ins_cost(VOLATILE_REF_COST);
7517 format %{ "stlrb $src, $mem\t# byte" %}
7518
7519 ins_encode(aarch64_enc_stlrb(src, mem));
7520
7521 ins_pipe(pipe_class_memory);
7522 %}
7523
7524 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7525 %{
7526 match(Set mem (StoreB mem zero));
7527
7528 ins_cost(VOLATILE_REF_COST);
7529 format %{ "stlrb zr, $mem\t# byte" %}
7530
7531 ins_encode(aarch64_enc_stlrb0(mem));
7532
7533 ins_pipe(pipe_class_memory);
7534 %}
7535
7536 // Store Char/Short
7537 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7538 %{
7539 match(Set mem (StoreC mem src));
7540
7541 ins_cost(VOLATILE_REF_COST);
7542 format %{ "stlrh $src, $mem\t# short" %}
7543
7544 ins_encode(aarch64_enc_stlrh(src, mem));
7545
7546 ins_pipe(pipe_class_memory);
7547 %}
7548
7549 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7550 %{
7551 match(Set mem (StoreC mem zero));
7552
7553 ins_cost(VOLATILE_REF_COST);
7554 format %{ "stlrh zr, $mem\t# short" %}
7555
7556 ins_encode(aarch64_enc_stlrh0(mem));
7557
7558 ins_pipe(pipe_class_memory);
7559 %}
7560
7561 // Store Integer
7562
7563 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7564 %{
7565 match(Set mem(StoreI mem src));
7566
7567 ins_cost(VOLATILE_REF_COST);
7568 format %{ "stlrw $src, $mem\t# int" %}
7569
7570 ins_encode(aarch64_enc_stlrw(src, mem));
7571
7572 ins_pipe(pipe_class_memory);
7573 %}
7574
7575 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7576 %{
7577 match(Set mem(StoreI mem zero));
7578
7579 ins_cost(VOLATILE_REF_COST);
7580 format %{ "stlrw zr, $mem\t# int" %}
7581
7582 ins_encode(aarch64_enc_stlrw0(mem));
7583
7584 ins_pipe(pipe_class_memory);
7585 %}
7586
7587 // Store Long (64 bit signed)
7588 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7589 %{
7590 match(Set mem (StoreL mem src));
7591
7592 ins_cost(VOLATILE_REF_COST);
7593 format %{ "stlr $src, $mem\t# int" %}
7594
7595 ins_encode(aarch64_enc_stlr(src, mem));
7596
7597 ins_pipe(pipe_class_memory);
7598 %}
7599
7600 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7601 %{
7602 match(Set mem (StoreL mem zero));
7603
7604 ins_cost(VOLATILE_REF_COST);
7605 format %{ "stlr zr, $mem\t# int" %}
7606
7607 ins_encode(aarch64_enc_stlr0(mem));
7608
7609 ins_pipe(pipe_class_memory);
7610 %}
7611
7612 // Store Pointer
7613 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7614 %{
7615 match(Set mem (StoreP mem src));
7616 predicate(n->as_Store()->barrier_data() == 0);
7617
7618 ins_cost(VOLATILE_REF_COST);
7619 format %{ "stlr $src, $mem\t# ptr" %}
7620
7621 ins_encode(aarch64_enc_stlr(src, mem));
7622
7623 ins_pipe(pipe_class_memory);
7624 %}
7625
7626 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7627 %{
7628 match(Set mem (StoreP mem zero));
7629 predicate(n->as_Store()->barrier_data() == 0);
7630
7631 ins_cost(VOLATILE_REF_COST);
7632 format %{ "stlr zr, $mem\t# ptr" %}
7633
7634 ins_encode(aarch64_enc_stlr0(mem));
7635
7636 ins_pipe(pipe_class_memory);
7637 %}
7638
7639 // Store Compressed Pointer
7640 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7641 %{
7642 match(Set mem (StoreN mem src));
7643 predicate(n->as_Store()->barrier_data() == 0);
7644
7645 ins_cost(VOLATILE_REF_COST);
7646 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7647
7648 ins_encode(aarch64_enc_stlrw(src, mem));
7649
7650 ins_pipe(pipe_class_memory);
7651 %}
7652
7653 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7654 %{
7655 match(Set mem (StoreN mem zero));
7656 predicate(n->as_Store()->barrier_data() == 0);
7657
7658 ins_cost(VOLATILE_REF_COST);
7659 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7660
7661 ins_encode(aarch64_enc_stlrw0(mem));
7662
7663 ins_pipe(pipe_class_memory);
7664 %}
7665
7666 // Store Float
7667 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7668 %{
7669 match(Set mem (StoreF mem src));
7670
7671 ins_cost(VOLATILE_REF_COST);
7672 format %{ "stlrs $src, $mem\t# float" %}
7673
7674 ins_encode( aarch64_enc_fstlrs(src, mem) );
7675
7676 ins_pipe(pipe_class_memory);
7677 %}
7678
7679 // TODO
7680 // implement storeImmF0 and storeFImmPacked
7681
7682 // Store Double
7683 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7684 %{
7685 match(Set mem (StoreD mem src));
7686
7687 ins_cost(VOLATILE_REF_COST);
7688 format %{ "stlrd $src, $mem\t# double" %}
7689
7690 ins_encode( aarch64_enc_fstlrd(src, mem) );
7691
7692 ins_pipe(pipe_class_memory);
7693 %}
7694
7695 // ---------------- end of volatile loads and stores ----------------
7696
7697 instruct cacheWB(indirect addr)
7698 %{
7699 predicate(VM_Version::supports_data_cache_line_flush());
7700 match(CacheWB addr);
7701
7702 ins_cost(100);
7703 format %{"cache wb $addr" %}
7704 ins_encode %{
7705 assert($addr->index_position() < 0, "should be");
7706 assert($addr$$disp == 0, "should be");
7707 __ cache_wb(Address($addr$$base$$Register, 0));
7708 %}
7709 ins_pipe(pipe_slow); // XXX
7710 %}
7711
7712 instruct cacheWBPreSync()
7713 %{
7714 predicate(VM_Version::supports_data_cache_line_flush());
7715 match(CacheWBPreSync);
7716
7717 ins_cost(100);
7718 format %{"cache wb presync" %}
7719 ins_encode %{
7720 __ cache_wbsync(true);
7721 %}
7722 ins_pipe(pipe_slow); // XXX
7723 %}
7724
7725 instruct cacheWBPostSync()
7726 %{
7727 predicate(VM_Version::supports_data_cache_line_flush());
7728 match(CacheWBPostSync);
7729
7730 ins_cost(100);
7731 format %{"cache wb postsync" %}
7732 ins_encode %{
7733 __ cache_wbsync(false);
7734 %}
7735 ins_pipe(pipe_slow); // XXX
7736 %}
7737
7738 // ============================================================================
7739 // BSWAP Instructions
7740
7741 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7742 match(Set dst (ReverseBytesI src));
7743
7744 ins_cost(INSN_COST);
7745 format %{ "revw $dst, $src" %}
7746
7747 ins_encode %{
7748 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7749 %}
7750
7751 ins_pipe(ialu_reg);
7752 %}
7753
7754 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7755 match(Set dst (ReverseBytesL src));
7756
7757 ins_cost(INSN_COST);
7758 format %{ "rev $dst, $src" %}
7759
7760 ins_encode %{
7761 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7762 %}
7763
7764 ins_pipe(ialu_reg);
7765 %}
7766
7767 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7768 match(Set dst (ReverseBytesUS src));
7769
7770 ins_cost(INSN_COST);
7771 format %{ "rev16w $dst, $src\t# $dst -> unsigned short" %}
7772
7773 ins_encode %{
7774 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7775 __ narrow_subword_type(as_Register($dst$$reg), T_CHAR);
7776 %}
7777
7778 ins_pipe(ialu_reg);
7779 %}
7780
7781 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7782 match(Set dst (ReverseBytesS src));
7783
7784 ins_cost(INSN_COST);
7785 format %{ "rev16w $dst, $src\n\t"
7786 "sbfmw $dst, $dst, #0, #15" %}
7787
7788 ins_encode %{
7789 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7790 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7791 %}
7792
7793 ins_pipe(ialu_reg);
7794 %}
7795
7796 // ============================================================================
7797 // Zero Count Instructions
7798
7799 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7800 match(Set dst (CountLeadingZerosI src));
7801
7802 ins_cost(INSN_COST);
7803 format %{ "clzw $dst, $src" %}
7804 ins_encode %{
7805 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7806 %}
7807
7808 ins_pipe(ialu_reg);
7809 %}
7810
7811 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7812 match(Set dst (CountLeadingZerosL src));
7813
7814 ins_cost(INSN_COST);
7815 format %{ "clz $dst, $src" %}
7816 ins_encode %{
7817 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7818 %}
7819
7820 ins_pipe(ialu_reg);
7821 %}
7822
7823 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7824 match(Set dst (CountTrailingZerosI src));
7825
7826 ins_cost(INSN_COST * 2);
7827 format %{ "rbitw $dst, $src\n\t"
7828 "clzw $dst, $dst" %}
7829 ins_encode %{
7830 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7831 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7832 %}
7833
7834 ins_pipe(ialu_reg);
7835 %}
7836
7837 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7838 match(Set dst (CountTrailingZerosL src));
7839
7840 ins_cost(INSN_COST * 2);
7841 format %{ "rbit $dst, $src\n\t"
7842 "clz $dst, $dst" %}
7843 ins_encode %{
7844 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7845 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7846 %}
7847
7848 ins_pipe(ialu_reg);
7849 %}
7850
7851 //---------- Population Count Instructions -------------------------------------
7852 //
7853
7854 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7855 match(Set dst (PopCountI src));
7856 effect(TEMP tmp);
7857 ins_cost(INSN_COST * 13);
7858
7859 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7860 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7861 "addv $tmp, $tmp\t# vector (8B)\n\t"
7862 "mov $dst, $tmp\t# vector (1D)" %}
7863 ins_encode %{
7864 __ fmovs($tmp$$FloatRegister, $src$$Register);
7865 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7866 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7867 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7868 %}
7869
7870 ins_pipe(pipe_class_default);
7871 %}
7872
7873 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7874 match(Set dst (PopCountI (LoadI mem)));
7875 effect(TEMP tmp);
7876 ins_cost(INSN_COST * 13);
7877
7878 format %{ "ldrs $tmp, $mem\n\t"
7879 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7880 "addv $tmp, $tmp\t# vector (8B)\n\t"
7881 "mov $dst, $tmp\t# vector (1D)" %}
7882 ins_encode %{
7883 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7884 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7885 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7886 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7887 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7888 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7889 %}
7890
7891 ins_pipe(pipe_class_default);
7892 %}
7893
7894 // Note: Long.bitCount(long) returns an int.
7895 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7896 match(Set dst (PopCountL src));
7897 effect(TEMP tmp);
7898 ins_cost(INSN_COST * 13);
7899
7900 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7901 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7902 "addv $tmp, $tmp\t# vector (8B)\n\t"
7903 "mov $dst, $tmp\t# vector (1D)" %}
7904 ins_encode %{
7905 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7906 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7907 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7908 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7909 %}
7910
7911 ins_pipe(pipe_class_default);
7912 %}
7913
7914 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7915 match(Set dst (PopCountL (LoadL mem)));
7916 effect(TEMP tmp);
7917 ins_cost(INSN_COST * 13);
7918
7919 format %{ "ldrd $tmp, $mem\n\t"
7920 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7921 "addv $tmp, $tmp\t# vector (8B)\n\t"
7922 "mov $dst, $tmp\t# vector (1D)" %}
7923 ins_encode %{
7924 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7925 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7926 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7927 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7928 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7929 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7930 %}
7931
7932 ins_pipe(pipe_class_default);
7933 %}
7934
7935 // ============================================================================
7936 // VerifyVectorAlignment Instruction
7937
7938 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7939 match(Set addr (VerifyVectorAlignment addr mask));
7940 effect(KILL cr);
7941 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7942 ins_encode %{
7943 Label Lskip;
7944 // check if masked bits of addr are zero
7945 __ tst($addr$$Register, $mask$$constant);
7946 __ br(Assembler::EQ, Lskip);
7947 __ stop("verify_vector_alignment found a misaligned vector memory access");
7948 __ bind(Lskip);
7949 %}
7950 ins_pipe(pipe_slow);
7951 %}
7952
7953 // ============================================================================
7954 // MemBar Instruction
7955
7956 instruct load_fence() %{
7957 match(LoadFence);
7958 ins_cost(VOLATILE_REF_COST);
7959
7960 format %{ "load_fence" %}
7961
7962 ins_encode %{
7963 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7964 %}
7965 ins_pipe(pipe_serial);
7966 %}
7967
7968 instruct unnecessary_membar_acquire() %{
7969 predicate(unnecessary_acquire(n));
7970 match(MemBarAcquire);
7971 ins_cost(0);
7972
7973 format %{ "membar_acquire (elided)" %}
7974
7975 ins_encode %{
7976 __ block_comment("membar_acquire (elided)");
7977 %}
7978
7979 ins_pipe(pipe_class_empty);
7980 %}
7981
7982 instruct membar_acquire() %{
7983 match(MemBarAcquire);
7984 ins_cost(VOLATILE_REF_COST);
7985
7986 format %{ "membar_acquire\n\t"
7987 "dmb ishld" %}
7988
7989 ins_encode %{
7990 __ block_comment("membar_acquire");
7991 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7992 %}
7993
7994 ins_pipe(pipe_serial);
7995 %}
7996
7997
7998 instruct membar_acquire_lock() %{
7999 match(MemBarAcquireLock);
8000 ins_cost(VOLATILE_REF_COST);
8001
8002 format %{ "membar_acquire_lock (elided)" %}
8003
8004 ins_encode %{
8005 __ block_comment("membar_acquire_lock (elided)");
8006 %}
8007
8008 ins_pipe(pipe_serial);
8009 %}
8010
8011 instruct store_fence() %{
8012 match(StoreFence);
8013 ins_cost(VOLATILE_REF_COST);
8014
8015 format %{ "store_fence" %}
8016
8017 ins_encode %{
8018 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8019 %}
8020 ins_pipe(pipe_serial);
8021 %}
8022
8023 instruct unnecessary_membar_release() %{
8024 predicate(unnecessary_release(n));
8025 match(MemBarRelease);
8026 ins_cost(0);
8027
8028 format %{ "membar_release (elided)" %}
8029
8030 ins_encode %{
8031 __ block_comment("membar_release (elided)");
8032 %}
8033 ins_pipe(pipe_serial);
8034 %}
8035
8036 instruct membar_release() %{
8037 match(MemBarRelease);
8038 ins_cost(VOLATILE_REF_COST);
8039
8040 format %{ "membar_release\n\t"
8041 "dmb ishst\n\tdmb ishld" %}
8042
8043 ins_encode %{
8044 __ block_comment("membar_release");
8045 // These will be merged if AlwaysMergeDMB is enabled.
8046 __ membar(Assembler::StoreStore);
8047 __ membar(Assembler::LoadStore);
8048 %}
8049 ins_pipe(pipe_serial);
8050 %}
8051
8052 instruct membar_storestore() %{
8053 match(MemBarStoreStore);
8054 match(StoreStoreFence);
8055 ins_cost(VOLATILE_REF_COST);
8056
8057 format %{ "MEMBAR-store-store" %}
8058
8059 ins_encode %{
8060 __ membar(Assembler::StoreStore);
8061 %}
8062 ins_pipe(pipe_serial);
8063 %}
8064
8065 instruct membar_release_lock() %{
8066 match(MemBarReleaseLock);
8067 ins_cost(VOLATILE_REF_COST);
8068
8069 format %{ "membar_release_lock (elided)" %}
8070
8071 ins_encode %{
8072 __ block_comment("membar_release_lock (elided)");
8073 %}
8074
8075 ins_pipe(pipe_serial);
8076 %}
8077
8078 instruct membar_storeload() %{
8079 match(MemBarStoreLoad);
8080 ins_cost(VOLATILE_REF_COST*100);
8081
8082 format %{ "MEMBAR-store-load\n\t"
8083 "dmb ish" %}
8084
8085 ins_encode %{
8086 __ block_comment("membar_storeload");
8087 __ membar(Assembler::StoreLoad);
8088 %}
8089
8090 ins_pipe(pipe_serial);
8091 %}
8092
8093 instruct unnecessary_membar_volatile() %{
8094 predicate(unnecessary_volatile(n));
8095 match(MemBarVolatile);
8096 ins_cost(0);
8097
8098 format %{ "membar_volatile (elided)" %}
8099
8100 ins_encode %{
8101 __ block_comment("membar_volatile (elided)");
8102 %}
8103
8104 ins_pipe(pipe_serial);
8105 %}
8106
8107 instruct membar_volatile() %{
8108 match(MemBarVolatile);
8109 ins_cost(VOLATILE_REF_COST*100);
8110
8111 format %{ "membar_volatile\n\t"
8112 "dmb ish"%}
8113
8114 ins_encode %{
8115 __ block_comment("membar_volatile");
8116 __ membar(Assembler::StoreLoad);
8117 %}
8118
8119 ins_pipe(pipe_serial);
8120 %}
8121
8122 instruct membar_full() %{
8123 match(MemBarFull);
8124 ins_cost(VOLATILE_REF_COST*100);
8125
8126 format %{ "membar_full\n\t"
8127 "dmb ish" %}
8128 ins_encode %{
8129 __ block_comment("membar_full");
8130 __ membar(Assembler::AnyAny);
8131 %}
8132
8133 ins_pipe(pipe_serial);
8134 %}
8135
8136 // ============================================================================
8137 // Cast/Convert Instructions
8138
8139 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8140 match(Set dst (CastX2P src));
8141
8142 ins_cost(INSN_COST);
8143 format %{ "mov $dst, $src\t# long -> ptr" %}
8144
8145 ins_encode %{
8146 if ($dst$$reg != $src$$reg) {
8147 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8148 }
8149 %}
8150
8151 ins_pipe(ialu_reg);
8152 %}
8153
8154 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8155 match(Set dst (CastI2N src));
8156
8157 ins_cost(INSN_COST);
8158 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8159
8160 ins_encode %{
8161 if ($dst$$reg != $src$$reg) {
8162 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8163 }
8164 %}
8165
8166 ins_pipe(ialu_reg);
8167 %}
8168
8169 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8170 match(Set dst (CastP2X src));
8171
8172 ins_cost(INSN_COST);
8173 format %{ "mov $dst, $src\t# ptr -> long" %}
8174
8175 ins_encode %{
8176 if ($dst$$reg != $src$$reg) {
8177 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8178 }
8179 %}
8180
8181 ins_pipe(ialu_reg);
8182 %}
8183
8184 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8185 match(Set dst (CastP2X src));
8186
8187 ins_cost(INSN_COST);
8188 format %{ "mov $dst, $src\t# ptr -> long" %}
8189
8190 ins_encode %{
8191 if ($dst$$reg != $src$$reg) {
8192 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8193 }
8194 %}
8195
8196 ins_pipe(ialu_reg);
8197 %}
8198
8199 // Convert oop into int for vectors alignment masking
8200 instruct convP2I(iRegINoSp dst, iRegP src) %{
8201 match(Set dst (ConvL2I (CastP2X src)));
8202
8203 ins_cost(INSN_COST);
8204 format %{ "movw $dst, $src\t# ptr -> int" %}
8205 ins_encode %{
8206 __ movw($dst$$Register, $src$$Register);
8207 %}
8208
8209 ins_pipe(ialu_reg);
8210 %}
8211
8212 // Convert compressed oop into int for vectors alignment masking
8213 // in case of 32bit oops (heap < 4Gb).
8214 instruct convN2I(iRegINoSp dst, iRegN src)
8215 %{
8216 predicate(CompressedOops::shift() == 0);
8217 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8218
8219 ins_cost(INSN_COST);
8220 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8221 ins_encode %{
8222 __ movw($dst$$Register, $src$$Register);
8223 %}
8224
8225 ins_pipe(ialu_reg);
8226 %}
8227
8228
8229 // Convert oop pointer into compressed form
8230 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8231 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8232 match(Set dst (EncodeP src));
8233 effect(KILL cr);
8234 ins_cost(INSN_COST * 3);
8235 format %{ "encode_heap_oop $dst, $src" %}
8236 ins_encode %{
8237 Register s = $src$$Register;
8238 Register d = $dst$$Register;
8239 __ encode_heap_oop(d, s);
8240 %}
8241 ins_pipe(ialu_reg);
8242 %}
8243
8244 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8245 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8246 match(Set dst (EncodeP src));
8247 ins_cost(INSN_COST * 3);
8248 format %{ "encode_heap_oop_not_null $dst, $src" %}
8249 ins_encode %{
8250 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8251 %}
8252 ins_pipe(ialu_reg);
8253 %}
8254
8255 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8256 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8257 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8258 match(Set dst (DecodeN src));
8259 ins_cost(INSN_COST * 3);
8260 format %{ "decode_heap_oop $dst, $src" %}
8261 ins_encode %{
8262 Register s = $src$$Register;
8263 Register d = $dst$$Register;
8264 __ decode_heap_oop(d, s);
8265 %}
8266 ins_pipe(ialu_reg);
8267 %}
8268
8269 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8270 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8271 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8272 match(Set dst (DecodeN src));
8273 ins_cost(INSN_COST * 3);
8274 format %{ "decode_heap_oop_not_null $dst, $src" %}
8275 ins_encode %{
8276 Register s = $src$$Register;
8277 Register d = $dst$$Register;
8278 __ decode_heap_oop_not_null(d, s);
8279 %}
8280 ins_pipe(ialu_reg);
8281 %}
8282
8283 // n.b. AArch64 implementations of encode_klass_not_null and
8284 // decode_klass_not_null do not modify the flags register so, unlike
8285 // Intel, we don't kill CR as a side effect here
8286
8287 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8288 match(Set dst (EncodePKlass src));
8289
8290 ins_cost(INSN_COST * 3);
8291 format %{ "encode_klass_not_null $dst,$src" %}
8292
8293 ins_encode %{
8294 Register src_reg = as_Register($src$$reg);
8295 Register dst_reg = as_Register($dst$$reg);
8296 __ encode_klass_not_null(dst_reg, src_reg);
8297 %}
8298
8299 ins_pipe(ialu_reg);
8300 %}
8301
8302 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8303 match(Set dst (DecodeNKlass src));
8304
8305 ins_cost(INSN_COST * 3);
8306 format %{ "decode_klass_not_null $dst,$src" %}
8307
8308 ins_encode %{
8309 Register src_reg = as_Register($src$$reg);
8310 Register dst_reg = as_Register($dst$$reg);
8311 if (dst_reg != src_reg) {
8312 __ decode_klass_not_null(dst_reg, src_reg);
8313 } else {
8314 __ decode_klass_not_null(dst_reg);
8315 }
8316 %}
8317
8318 ins_pipe(ialu_reg);
8319 %}
8320
8321 instruct checkCastPP(iRegPNoSp dst)
8322 %{
8323 match(Set dst (CheckCastPP dst));
8324
8325 size(0);
8326 format %{ "# checkcastPP of $dst" %}
8327 ins_encode(/* empty encoding */);
8328 ins_pipe(pipe_class_empty);
8329 %}
8330
8331 instruct castPP(iRegPNoSp dst)
8332 %{
8333 match(Set dst (CastPP dst));
8334
8335 size(0);
8336 format %{ "# castPP of $dst" %}
8337 ins_encode(/* empty encoding */);
8338 ins_pipe(pipe_class_empty);
8339 %}
8340
8341 instruct castII(iRegI dst)
8342 %{
8343 predicate(VerifyConstraintCasts == 0);
8344 match(Set dst (CastII dst));
8345
8346 size(0);
8347 format %{ "# castII of $dst" %}
8348 ins_encode(/* empty encoding */);
8349 ins_cost(0);
8350 ins_pipe(pipe_class_empty);
8351 %}
8352
8353 instruct castII_checked(iRegI dst, rFlagsReg cr)
8354 %{
8355 predicate(VerifyConstraintCasts > 0);
8356 match(Set dst (CastII dst));
8357 effect(KILL cr);
8358
8359 format %{ "# castII_checked of $dst" %}
8360 ins_encode %{
8361 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8362 %}
8363 ins_pipe(pipe_slow);
8364 %}
8365
8366 instruct castLL(iRegL dst)
8367 %{
8368 predicate(VerifyConstraintCasts == 0);
8369 match(Set dst (CastLL dst));
8370
8371 size(0);
8372 format %{ "# castLL of $dst" %}
8373 ins_encode(/* empty encoding */);
8374 ins_cost(0);
8375 ins_pipe(pipe_class_empty);
8376 %}
8377
8378 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8379 %{
8380 predicate(VerifyConstraintCasts > 0);
8381 match(Set dst (CastLL dst));
8382 effect(KILL cr);
8383
8384 format %{ "# castLL_checked of $dst" %}
8385 ins_encode %{
8386 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8387 %}
8388 ins_pipe(pipe_slow);
8389 %}
8390
8391 instruct castHH(vRegF dst)
8392 %{
8393 match(Set dst (CastHH dst));
8394 size(0);
8395 format %{ "# castHH of $dst" %}
8396 ins_encode(/* empty encoding */);
8397 ins_cost(0);
8398 ins_pipe(pipe_class_empty);
8399 %}
8400
8401 instruct castFF(vRegF dst)
8402 %{
8403 match(Set dst (CastFF dst));
8404
8405 size(0);
8406 format %{ "# castFF of $dst" %}
8407 ins_encode(/* empty encoding */);
8408 ins_cost(0);
8409 ins_pipe(pipe_class_empty);
8410 %}
8411
8412 instruct castDD(vRegD dst)
8413 %{
8414 match(Set dst (CastDD dst));
8415
8416 size(0);
8417 format %{ "# castDD of $dst" %}
8418 ins_encode(/* empty encoding */);
8419 ins_cost(0);
8420 ins_pipe(pipe_class_empty);
8421 %}
8422
8423 instruct castVV(vReg dst)
8424 %{
8425 match(Set dst (CastVV dst));
8426
8427 size(0);
8428 format %{ "# castVV of $dst" %}
8429 ins_encode(/* empty encoding */);
8430 ins_cost(0);
8431 ins_pipe(pipe_class_empty);
8432 %}
8433
8434 instruct castVVMask(pRegGov dst)
8435 %{
8436 match(Set dst (CastVV dst));
8437
8438 size(0);
8439 format %{ "# castVV of $dst" %}
8440 ins_encode(/* empty encoding */);
8441 ins_cost(0);
8442 ins_pipe(pipe_class_empty);
8443 %}
8444
8445 // Manifest a CmpU result in an integer register.
8446 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8447 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
8448 %{
8449 match(Set dst (CmpU3 src1 src2));
8450 effect(KILL flags);
8451
8452 ins_cost(INSN_COST * 3);
8453 format %{
8454 "cmpw $src1, $src2\n\t"
8455 "csetw $dst, ne\n\t"
8456 "cnegw $dst, lo\t# CmpU3(reg)"
8457 %}
8458 ins_encode %{
8459 __ cmpw($src1$$Register, $src2$$Register);
8460 __ csetw($dst$$Register, Assembler::NE);
8461 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8462 %}
8463
8464 ins_pipe(pipe_class_default);
8465 %}
8466
8467 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
8468 %{
8469 match(Set dst (CmpU3 src1 src2));
8470 effect(KILL flags);
8471
8472 ins_cost(INSN_COST * 3);
8473 format %{
8474 "subsw zr, $src1, $src2\n\t"
8475 "csetw $dst, ne\n\t"
8476 "cnegw $dst, lo\t# CmpU3(imm)"
8477 %}
8478 ins_encode %{
8479 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
8480 __ csetw($dst$$Register, Assembler::NE);
8481 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8482 %}
8483
8484 ins_pipe(pipe_class_default);
8485 %}
8486
8487 // Manifest a CmpUL result in an integer register.
8488 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8489 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8490 %{
8491 match(Set dst (CmpUL3 src1 src2));
8492 effect(KILL flags);
8493
8494 ins_cost(INSN_COST * 3);
8495 format %{
8496 "cmp $src1, $src2\n\t"
8497 "csetw $dst, ne\n\t"
8498 "cnegw $dst, lo\t# CmpUL3(reg)"
8499 %}
8500 ins_encode %{
8501 __ cmp($src1$$Register, $src2$$Register);
8502 __ csetw($dst$$Register, Assembler::NE);
8503 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8504 %}
8505
8506 ins_pipe(pipe_class_default);
8507 %}
8508
8509 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8510 %{
8511 match(Set dst (CmpUL3 src1 src2));
8512 effect(KILL flags);
8513
8514 ins_cost(INSN_COST * 3);
8515 format %{
8516 "subs zr, $src1, $src2\n\t"
8517 "csetw $dst, ne\n\t"
8518 "cnegw $dst, lo\t# CmpUL3(imm)"
8519 %}
8520 ins_encode %{
8521 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8522 __ csetw($dst$$Register, Assembler::NE);
8523 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
8524 %}
8525
8526 ins_pipe(pipe_class_default);
8527 %}
8528
8529 // Manifest a CmpL result in an integer register.
8530 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
8531 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
8532 %{
8533 match(Set dst (CmpL3 src1 src2));
8534 effect(KILL flags);
8535
8536 ins_cost(INSN_COST * 3);
8537 format %{
8538 "cmp $src1, $src2\n\t"
8539 "csetw $dst, ne\n\t"
8540 "cnegw $dst, lt\t# CmpL3(reg)"
8541 %}
8542 ins_encode %{
8543 __ cmp($src1$$Register, $src2$$Register);
8544 __ csetw($dst$$Register, Assembler::NE);
8545 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8546 %}
8547
8548 ins_pipe(pipe_class_default);
8549 %}
8550
8551 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
8552 %{
8553 match(Set dst (CmpL3 src1 src2));
8554 effect(KILL flags);
8555
8556 ins_cost(INSN_COST * 3);
8557 format %{
8558 "subs zr, $src1, $src2\n\t"
8559 "csetw $dst, ne\n\t"
8560 "cnegw $dst, lt\t# CmpL3(imm)"
8561 %}
8562 ins_encode %{
8563 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
8564 __ csetw($dst$$Register, Assembler::NE);
8565 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
8566 %}
8567
8568 ins_pipe(pipe_class_default);
8569 %}
8570
8571 // ============================================================================
8572 // Conditional Move Instructions
8573
8574 // n.b. we have identical rules for both a signed compare op (cmpOp)
8575 // and an unsigned compare op (cmpOpU). it would be nice if we could
8576 // define an op class which merged both inputs and use it to type the
8577 // argument to a single rule. unfortunatelyt his fails because the
8578 // opclass does not live up to the COND_INTER interface of its
8579 // component operands. When the generic code tries to negate the
8580 // operand it ends up running the generci Machoper::negate method
8581 // which throws a ShouldNotHappen. So, we have to provide two flavours
8582 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
8583
8584 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8585 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8586
8587 ins_cost(INSN_COST * 2);
8588 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
8589
8590 ins_encode %{
8591 __ cselw(as_Register($dst$$reg),
8592 as_Register($src2$$reg),
8593 as_Register($src1$$reg),
8594 (Assembler::Condition)$cmp$$cmpcode);
8595 %}
8596
8597 ins_pipe(icond_reg_reg);
8598 %}
8599
8600 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
8601 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
8602
8603 ins_cost(INSN_COST * 2);
8604 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
8605
8606 ins_encode %{
8607 __ cselw(as_Register($dst$$reg),
8608 as_Register($src2$$reg),
8609 as_Register($src1$$reg),
8610 (Assembler::Condition)$cmp$$cmpcode);
8611 %}
8612
8613 ins_pipe(icond_reg_reg);
8614 %}
8615
8616 // special cases where one arg is zero
8617
8618 // n.b. this is selected in preference to the rule above because it
8619 // avoids loading constant 0 into a source register
8620
8621 // TODO
8622 // we ought only to be able to cull one of these variants as the ideal
8623 // transforms ought always to order the zero consistently (to left/right?)
8624
8625 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8626 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8627
8628 ins_cost(INSN_COST * 2);
8629 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
8630
8631 ins_encode %{
8632 __ cselw(as_Register($dst$$reg),
8633 as_Register($src$$reg),
8634 zr,
8635 (Assembler::Condition)$cmp$$cmpcode);
8636 %}
8637
8638 ins_pipe(icond_reg);
8639 %}
8640
8641 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
8642 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
8643
8644 ins_cost(INSN_COST * 2);
8645 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
8646
8647 ins_encode %{
8648 __ cselw(as_Register($dst$$reg),
8649 as_Register($src$$reg),
8650 zr,
8651 (Assembler::Condition)$cmp$$cmpcode);
8652 %}
8653
8654 ins_pipe(icond_reg);
8655 %}
8656
8657 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8658 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8659
8660 ins_cost(INSN_COST * 2);
8661 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
8662
8663 ins_encode %{
8664 __ cselw(as_Register($dst$$reg),
8665 zr,
8666 as_Register($src$$reg),
8667 (Assembler::Condition)$cmp$$cmpcode);
8668 %}
8669
8670 ins_pipe(icond_reg);
8671 %}
8672
8673 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
8674 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
8675
8676 ins_cost(INSN_COST * 2);
8677 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
8678
8679 ins_encode %{
8680 __ cselw(as_Register($dst$$reg),
8681 zr,
8682 as_Register($src$$reg),
8683 (Assembler::Condition)$cmp$$cmpcode);
8684 %}
8685
8686 ins_pipe(icond_reg);
8687 %}
8688
8689 // special case for creating a boolean 0 or 1
8690
8691 // n.b. this is selected in preference to the rule above because it
8692 // avoids loading constants 0 and 1 into a source register
8693
8694 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8695 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8696
8697 ins_cost(INSN_COST * 2);
8698 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
8699
8700 ins_encode %{
8701 // equivalently
8702 // cset(as_Register($dst$$reg),
8703 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8704 __ csincw(as_Register($dst$$reg),
8705 zr,
8706 zr,
8707 (Assembler::Condition)$cmp$$cmpcode);
8708 %}
8709
8710 ins_pipe(icond_none);
8711 %}
8712
8713 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
8714 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
8715
8716 ins_cost(INSN_COST * 2);
8717 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
8718
8719 ins_encode %{
8720 // equivalently
8721 // cset(as_Register($dst$$reg),
8722 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
8723 __ csincw(as_Register($dst$$reg),
8724 zr,
8725 zr,
8726 (Assembler::Condition)$cmp$$cmpcode);
8727 %}
8728
8729 ins_pipe(icond_none);
8730 %}
8731
8732 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8733 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8734
8735 ins_cost(INSN_COST * 2);
8736 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
8737
8738 ins_encode %{
8739 __ csel(as_Register($dst$$reg),
8740 as_Register($src2$$reg),
8741 as_Register($src1$$reg),
8742 (Assembler::Condition)$cmp$$cmpcode);
8743 %}
8744
8745 ins_pipe(icond_reg_reg);
8746 %}
8747
8748 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
8749 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
8750
8751 ins_cost(INSN_COST * 2);
8752 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
8753
8754 ins_encode %{
8755 __ csel(as_Register($dst$$reg),
8756 as_Register($src2$$reg),
8757 as_Register($src1$$reg),
8758 (Assembler::Condition)$cmp$$cmpcode);
8759 %}
8760
8761 ins_pipe(icond_reg_reg);
8762 %}
8763
8764 // special cases where one arg is zero
8765
8766 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8767 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8768
8769 ins_cost(INSN_COST * 2);
8770 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
8771
8772 ins_encode %{
8773 __ csel(as_Register($dst$$reg),
8774 zr,
8775 as_Register($src$$reg),
8776 (Assembler::Condition)$cmp$$cmpcode);
8777 %}
8778
8779 ins_pipe(icond_reg);
8780 %}
8781
8782 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
8783 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
8784
8785 ins_cost(INSN_COST * 2);
8786 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
8787
8788 ins_encode %{
8789 __ csel(as_Register($dst$$reg),
8790 zr,
8791 as_Register($src$$reg),
8792 (Assembler::Condition)$cmp$$cmpcode);
8793 %}
8794
8795 ins_pipe(icond_reg);
8796 %}
8797
8798 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8799 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8800
8801 ins_cost(INSN_COST * 2);
8802 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
8803
8804 ins_encode %{
8805 __ csel(as_Register($dst$$reg),
8806 as_Register($src$$reg),
8807 zr,
8808 (Assembler::Condition)$cmp$$cmpcode);
8809 %}
8810
8811 ins_pipe(icond_reg);
8812 %}
8813
8814 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
8815 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
8816
8817 ins_cost(INSN_COST * 2);
8818 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
8819
8820 ins_encode %{
8821 __ csel(as_Register($dst$$reg),
8822 as_Register($src$$reg),
8823 zr,
8824 (Assembler::Condition)$cmp$$cmpcode);
8825 %}
8826
8827 ins_pipe(icond_reg);
8828 %}
8829
8830 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8831 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8832
8833 ins_cost(INSN_COST * 2);
8834 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
8835
8836 ins_encode %{
8837 __ csel(as_Register($dst$$reg),
8838 as_Register($src2$$reg),
8839 as_Register($src1$$reg),
8840 (Assembler::Condition)$cmp$$cmpcode);
8841 %}
8842
8843 ins_pipe(icond_reg_reg);
8844 %}
8845
8846 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
8847 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
8848
8849 ins_cost(INSN_COST * 2);
8850 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
8851
8852 ins_encode %{
8853 __ csel(as_Register($dst$$reg),
8854 as_Register($src2$$reg),
8855 as_Register($src1$$reg),
8856 (Assembler::Condition)$cmp$$cmpcode);
8857 %}
8858
8859 ins_pipe(icond_reg_reg);
8860 %}
8861
8862 // special cases where one arg is zero
8863
8864 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8865 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8866
8867 ins_cost(INSN_COST * 2);
8868 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
8869
8870 ins_encode %{
8871 __ csel(as_Register($dst$$reg),
8872 zr,
8873 as_Register($src$$reg),
8874 (Assembler::Condition)$cmp$$cmpcode);
8875 %}
8876
8877 ins_pipe(icond_reg);
8878 %}
8879
8880 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
8881 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
8882
8883 ins_cost(INSN_COST * 2);
8884 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
8885
8886 ins_encode %{
8887 __ csel(as_Register($dst$$reg),
8888 zr,
8889 as_Register($src$$reg),
8890 (Assembler::Condition)$cmp$$cmpcode);
8891 %}
8892
8893 ins_pipe(icond_reg);
8894 %}
8895
8896 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8897 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8898
8899 ins_cost(INSN_COST * 2);
8900 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
8901
8902 ins_encode %{
8903 __ csel(as_Register($dst$$reg),
8904 as_Register($src$$reg),
8905 zr,
8906 (Assembler::Condition)$cmp$$cmpcode);
8907 %}
8908
8909 ins_pipe(icond_reg);
8910 %}
8911
8912 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
8913 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
8914
8915 ins_cost(INSN_COST * 2);
8916 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
8917
8918 ins_encode %{
8919 __ csel(as_Register($dst$$reg),
8920 as_Register($src$$reg),
8921 zr,
8922 (Assembler::Condition)$cmp$$cmpcode);
8923 %}
8924
8925 ins_pipe(icond_reg);
8926 %}
8927
8928 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8929 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8930
8931 ins_cost(INSN_COST * 2);
8932 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8933
8934 ins_encode %{
8935 __ cselw(as_Register($dst$$reg),
8936 as_Register($src2$$reg),
8937 as_Register($src1$$reg),
8938 (Assembler::Condition)$cmp$$cmpcode);
8939 %}
8940
8941 ins_pipe(icond_reg_reg);
8942 %}
8943
8944 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
8945 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
8946
8947 ins_cost(INSN_COST * 2);
8948 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
8949
8950 ins_encode %{
8951 __ cselw(as_Register($dst$$reg),
8952 as_Register($src2$$reg),
8953 as_Register($src1$$reg),
8954 (Assembler::Condition)$cmp$$cmpcode);
8955 %}
8956
8957 ins_pipe(icond_reg_reg);
8958 %}
8959
8960 // special cases where one arg is zero
8961
8962 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8963 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8964
8965 ins_cost(INSN_COST * 2);
8966 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
8967
8968 ins_encode %{
8969 __ cselw(as_Register($dst$$reg),
8970 zr,
8971 as_Register($src$$reg),
8972 (Assembler::Condition)$cmp$$cmpcode);
8973 %}
8974
8975 ins_pipe(icond_reg);
8976 %}
8977
8978 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
8979 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
8980
8981 ins_cost(INSN_COST * 2);
8982 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
8983
8984 ins_encode %{
8985 __ cselw(as_Register($dst$$reg),
8986 zr,
8987 as_Register($src$$reg),
8988 (Assembler::Condition)$cmp$$cmpcode);
8989 %}
8990
8991 ins_pipe(icond_reg);
8992 %}
8993
8994 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
8995 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
8996
8997 ins_cost(INSN_COST * 2);
8998 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
8999
9000 ins_encode %{
9001 __ cselw(as_Register($dst$$reg),
9002 as_Register($src$$reg),
9003 zr,
9004 (Assembler::Condition)$cmp$$cmpcode);
9005 %}
9006
9007 ins_pipe(icond_reg);
9008 %}
9009
9010 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9011 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9012
9013 ins_cost(INSN_COST * 2);
9014 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9015
9016 ins_encode %{
9017 __ cselw(as_Register($dst$$reg),
9018 as_Register($src$$reg),
9019 zr,
9020 (Assembler::Condition)$cmp$$cmpcode);
9021 %}
9022
9023 ins_pipe(icond_reg);
9024 %}
9025
9026 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9027 %{
9028 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9029
9030 ins_cost(INSN_COST * 3);
9031
9032 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9033 ins_encode %{
9034 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9035 __ fcsels(as_FloatRegister($dst$$reg),
9036 as_FloatRegister($src2$$reg),
9037 as_FloatRegister($src1$$reg),
9038 cond);
9039 %}
9040
9041 ins_pipe(fp_cond_reg_reg_s);
9042 %}
9043
9044 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9045 %{
9046 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9047
9048 ins_cost(INSN_COST * 3);
9049
9050 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9051 ins_encode %{
9052 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9053 __ fcsels(as_FloatRegister($dst$$reg),
9054 as_FloatRegister($src2$$reg),
9055 as_FloatRegister($src1$$reg),
9056 cond);
9057 %}
9058
9059 ins_pipe(fp_cond_reg_reg_s);
9060 %}
9061
9062 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9063 %{
9064 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9065
9066 ins_cost(INSN_COST * 3);
9067
9068 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9069 ins_encode %{
9070 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9071 __ fcseld(as_FloatRegister($dst$$reg),
9072 as_FloatRegister($src2$$reg),
9073 as_FloatRegister($src1$$reg),
9074 cond);
9075 %}
9076
9077 ins_pipe(fp_cond_reg_reg_d);
9078 %}
9079
9080 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9081 %{
9082 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9083
9084 ins_cost(INSN_COST * 3);
9085
9086 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9087 ins_encode %{
9088 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9089 __ fcseld(as_FloatRegister($dst$$reg),
9090 as_FloatRegister($src2$$reg),
9091 as_FloatRegister($src1$$reg),
9092 cond);
9093 %}
9094
9095 ins_pipe(fp_cond_reg_reg_d);
9096 %}
9097
9098 // ============================================================================
9099 // Arithmetic Instructions
9100 //
9101
9102 // Integer Addition
9103
9104 // TODO
9105 // these currently employ operations which do not set CR and hence are
9106 // not flagged as killing CR but we would like to isolate the cases
9107 // where we want to set flags from those where we don't. need to work
9108 // out how to do that.
9109
9110 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9111 match(Set dst (AddI src1 src2));
9112
9113 ins_cost(INSN_COST);
9114 format %{ "addw $dst, $src1, $src2" %}
9115
9116 ins_encode %{
9117 __ addw(as_Register($dst$$reg),
9118 as_Register($src1$$reg),
9119 as_Register($src2$$reg));
9120 %}
9121
9122 ins_pipe(ialu_reg_reg);
9123 %}
9124
9125 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9126 match(Set dst (AddI src1 src2));
9127
9128 ins_cost(INSN_COST);
9129 format %{ "addw $dst, $src1, $src2" %}
9130
9131 // use opcode to indicate that this is an add not a sub
9132 opcode(0x0);
9133
9134 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9135
9136 ins_pipe(ialu_reg_imm);
9137 %}
9138
9139 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9140 match(Set dst (AddI (ConvL2I src1) src2));
9141
9142 ins_cost(INSN_COST);
9143 format %{ "addw $dst, $src1, $src2" %}
9144
9145 // use opcode to indicate that this is an add not a sub
9146 opcode(0x0);
9147
9148 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9149
9150 ins_pipe(ialu_reg_imm);
9151 %}
9152
9153 // Pointer Addition
9154 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9155 match(Set dst (AddP src1 src2));
9156
9157 ins_cost(INSN_COST);
9158 format %{ "add $dst, $src1, $src2\t# ptr" %}
9159
9160 ins_encode %{
9161 __ add(as_Register($dst$$reg),
9162 as_Register($src1$$reg),
9163 as_Register($src2$$reg));
9164 %}
9165
9166 ins_pipe(ialu_reg_reg);
9167 %}
9168
9169 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9170 match(Set dst (AddP src1 (ConvI2L src2)));
9171
9172 ins_cost(1.9 * INSN_COST);
9173 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9174
9175 ins_encode %{
9176 __ add(as_Register($dst$$reg),
9177 as_Register($src1$$reg),
9178 as_Register($src2$$reg), ext::sxtw);
9179 %}
9180
9181 ins_pipe(ialu_reg_reg);
9182 %}
9183
9184 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9185 match(Set dst (AddP src1 (LShiftL src2 scale)));
9186
9187 ins_cost(1.9 * INSN_COST);
9188 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9189
9190 ins_encode %{
9191 __ lea(as_Register($dst$$reg),
9192 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9193 Address::lsl($scale$$constant)));
9194 %}
9195
9196 ins_pipe(ialu_reg_reg_shift);
9197 %}
9198
9199 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9200 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9201
9202 ins_cost(1.9 * INSN_COST);
9203 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9204
9205 ins_encode %{
9206 __ lea(as_Register($dst$$reg),
9207 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9208 Address::sxtw($scale$$constant)));
9209 %}
9210
9211 ins_pipe(ialu_reg_reg_shift);
9212 %}
9213
9214 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9215 match(Set dst (LShiftL (ConvI2L src) scale));
9216
9217 ins_cost(INSN_COST);
9218 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9219
9220 ins_encode %{
9221 __ sbfiz(as_Register($dst$$reg),
9222 as_Register($src$$reg),
9223 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9224 %}
9225
9226 ins_pipe(ialu_reg_shift);
9227 %}
9228
9229 // Pointer Immediate Addition
9230 // n.b. this needs to be more expensive than using an indirect memory
9231 // operand
9232 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9233 match(Set dst (AddP src1 src2));
9234
9235 ins_cost(INSN_COST);
9236 format %{ "add $dst, $src1, $src2\t# ptr" %}
9237
9238 // use opcode to indicate that this is an add not a sub
9239 opcode(0x0);
9240
9241 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9242
9243 ins_pipe(ialu_reg_imm);
9244 %}
9245
9246 // Long Addition
9247 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9248
9249 match(Set dst (AddL src1 src2));
9250
9251 ins_cost(INSN_COST);
9252 format %{ "add $dst, $src1, $src2" %}
9253
9254 ins_encode %{
9255 __ add(as_Register($dst$$reg),
9256 as_Register($src1$$reg),
9257 as_Register($src2$$reg));
9258 %}
9259
9260 ins_pipe(ialu_reg_reg);
9261 %}
9262
9263 // No constant pool entries requiredLong Immediate Addition.
9264 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9265 match(Set dst (AddL src1 src2));
9266
9267 ins_cost(INSN_COST);
9268 format %{ "add $dst, $src1, $src2" %}
9269
9270 // use opcode to indicate that this is an add not a sub
9271 opcode(0x0);
9272
9273 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9274
9275 ins_pipe(ialu_reg_imm);
9276 %}
9277
9278 // Integer Subtraction
9279 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9280 match(Set dst (SubI src1 src2));
9281
9282 ins_cost(INSN_COST);
9283 format %{ "subw $dst, $src1, $src2" %}
9284
9285 ins_encode %{
9286 __ subw(as_Register($dst$$reg),
9287 as_Register($src1$$reg),
9288 as_Register($src2$$reg));
9289 %}
9290
9291 ins_pipe(ialu_reg_reg);
9292 %}
9293
9294 // Immediate Subtraction
9295 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9296 match(Set dst (SubI src1 src2));
9297
9298 ins_cost(INSN_COST);
9299 format %{ "subw $dst, $src1, $src2" %}
9300
9301 // use opcode to indicate that this is a sub not an add
9302 opcode(0x1);
9303
9304 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9305
9306 ins_pipe(ialu_reg_imm);
9307 %}
9308
9309 // Long Subtraction
9310 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9311
9312 match(Set dst (SubL src1 src2));
9313
9314 ins_cost(INSN_COST);
9315 format %{ "sub $dst, $src1, $src2" %}
9316
9317 ins_encode %{
9318 __ sub(as_Register($dst$$reg),
9319 as_Register($src1$$reg),
9320 as_Register($src2$$reg));
9321 %}
9322
9323 ins_pipe(ialu_reg_reg);
9324 %}
9325
9326 // No constant pool entries requiredLong Immediate Subtraction.
9327 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9328 match(Set dst (SubL src1 src2));
9329
9330 ins_cost(INSN_COST);
9331 format %{ "sub$dst, $src1, $src2" %}
9332
9333 // use opcode to indicate that this is a sub not an add
9334 opcode(0x1);
9335
9336 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9337
9338 ins_pipe(ialu_reg_imm);
9339 %}
9340
9341 // Integer Negation (special case for sub)
9342
9343 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
9344 match(Set dst (SubI zero src));
9345
9346 ins_cost(INSN_COST);
9347 format %{ "negw $dst, $src\t# int" %}
9348
9349 ins_encode %{
9350 __ negw(as_Register($dst$$reg),
9351 as_Register($src$$reg));
9352 %}
9353
9354 ins_pipe(ialu_reg);
9355 %}
9356
9357 // Long Negation
9358
9359 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
9360 match(Set dst (SubL zero src));
9361
9362 ins_cost(INSN_COST);
9363 format %{ "neg $dst, $src\t# long" %}
9364
9365 ins_encode %{
9366 __ neg(as_Register($dst$$reg),
9367 as_Register($src$$reg));
9368 %}
9369
9370 ins_pipe(ialu_reg);
9371 %}
9372
9373 // Integer Multiply
9374
9375 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9376 match(Set dst (MulI src1 src2));
9377
9378 ins_cost(INSN_COST * 3);
9379 format %{ "mulw $dst, $src1, $src2" %}
9380
9381 ins_encode %{
9382 __ mulw(as_Register($dst$$reg),
9383 as_Register($src1$$reg),
9384 as_Register($src2$$reg));
9385 %}
9386
9387 ins_pipe(imul_reg_reg);
9388 %}
9389
9390 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9391 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
9392
9393 ins_cost(INSN_COST * 3);
9394 format %{ "smull $dst, $src1, $src2" %}
9395
9396 ins_encode %{
9397 __ smull(as_Register($dst$$reg),
9398 as_Register($src1$$reg),
9399 as_Register($src2$$reg));
9400 %}
9401
9402 ins_pipe(imul_reg_reg);
9403 %}
9404
9405 // Long Multiply
9406
9407 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9408 match(Set dst (MulL src1 src2));
9409
9410 ins_cost(INSN_COST * 5);
9411 format %{ "mul $dst, $src1, $src2" %}
9412
9413 ins_encode %{
9414 __ mul(as_Register($dst$$reg),
9415 as_Register($src1$$reg),
9416 as_Register($src2$$reg));
9417 %}
9418
9419 ins_pipe(lmul_reg_reg);
9420 %}
9421
9422 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9423 %{
9424 match(Set dst (MulHiL src1 src2));
9425
9426 ins_cost(INSN_COST * 7);
9427 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
9428
9429 ins_encode %{
9430 __ smulh(as_Register($dst$$reg),
9431 as_Register($src1$$reg),
9432 as_Register($src2$$reg));
9433 %}
9434
9435 ins_pipe(lmul_reg_reg);
9436 %}
9437
9438 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
9439 %{
9440 match(Set dst (UMulHiL src1 src2));
9441
9442 ins_cost(INSN_COST * 7);
9443 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
9444
9445 ins_encode %{
9446 __ umulh(as_Register($dst$$reg),
9447 as_Register($src1$$reg),
9448 as_Register($src2$$reg));
9449 %}
9450
9451 ins_pipe(lmul_reg_reg);
9452 %}
9453
9454 // Combined Integer Multiply & Add/Sub
9455
9456 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9457 match(Set dst (AddI src3 (MulI src1 src2)));
9458
9459 ins_cost(INSN_COST * 3);
9460 format %{ "madd $dst, $src1, $src2, $src3" %}
9461
9462 ins_encode %{
9463 __ maddw(as_Register($dst$$reg),
9464 as_Register($src1$$reg),
9465 as_Register($src2$$reg),
9466 as_Register($src3$$reg));
9467 %}
9468
9469 ins_pipe(imac_reg_reg);
9470 %}
9471
9472 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
9473 match(Set dst (SubI src3 (MulI src1 src2)));
9474
9475 ins_cost(INSN_COST * 3);
9476 format %{ "msub $dst, $src1, $src2, $src3" %}
9477
9478 ins_encode %{
9479 __ msubw(as_Register($dst$$reg),
9480 as_Register($src1$$reg),
9481 as_Register($src2$$reg),
9482 as_Register($src3$$reg));
9483 %}
9484
9485 ins_pipe(imac_reg_reg);
9486 %}
9487
9488 // Combined Integer Multiply & Neg
9489
9490 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
9491 match(Set dst (MulI (SubI zero src1) src2));
9492
9493 ins_cost(INSN_COST * 3);
9494 format %{ "mneg $dst, $src1, $src2" %}
9495
9496 ins_encode %{
9497 __ mnegw(as_Register($dst$$reg),
9498 as_Register($src1$$reg),
9499 as_Register($src2$$reg));
9500 %}
9501
9502 ins_pipe(imac_reg_reg);
9503 %}
9504
9505 // Combined Long Multiply & Add/Sub
9506
9507 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9508 match(Set dst (AddL src3 (MulL src1 src2)));
9509
9510 ins_cost(INSN_COST * 5);
9511 format %{ "madd $dst, $src1, $src2, $src3" %}
9512
9513 ins_encode %{
9514 __ madd(as_Register($dst$$reg),
9515 as_Register($src1$$reg),
9516 as_Register($src2$$reg),
9517 as_Register($src3$$reg));
9518 %}
9519
9520 ins_pipe(lmac_reg_reg);
9521 %}
9522
9523 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
9524 match(Set dst (SubL src3 (MulL src1 src2)));
9525
9526 ins_cost(INSN_COST * 5);
9527 format %{ "msub $dst, $src1, $src2, $src3" %}
9528
9529 ins_encode %{
9530 __ msub(as_Register($dst$$reg),
9531 as_Register($src1$$reg),
9532 as_Register($src2$$reg),
9533 as_Register($src3$$reg));
9534 %}
9535
9536 ins_pipe(lmac_reg_reg);
9537 %}
9538
9539 // Combined Long Multiply & Neg
9540
9541 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
9542 match(Set dst (MulL (SubL zero src1) src2));
9543
9544 ins_cost(INSN_COST * 5);
9545 format %{ "mneg $dst, $src1, $src2" %}
9546
9547 ins_encode %{
9548 __ mneg(as_Register($dst$$reg),
9549 as_Register($src1$$reg),
9550 as_Register($src2$$reg));
9551 %}
9552
9553 ins_pipe(lmac_reg_reg);
9554 %}
9555
9556 // Combine Integer Signed Multiply & Add/Sub/Neg Long
9557
9558 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9559 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9560
9561 ins_cost(INSN_COST * 3);
9562 format %{ "smaddl $dst, $src1, $src2, $src3" %}
9563
9564 ins_encode %{
9565 __ smaddl(as_Register($dst$$reg),
9566 as_Register($src1$$reg),
9567 as_Register($src2$$reg),
9568 as_Register($src3$$reg));
9569 %}
9570
9571 ins_pipe(imac_reg_reg);
9572 %}
9573
9574 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
9575 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
9576
9577 ins_cost(INSN_COST * 3);
9578 format %{ "smsubl $dst, $src1, $src2, $src3" %}
9579
9580 ins_encode %{
9581 __ smsubl(as_Register($dst$$reg),
9582 as_Register($src1$$reg),
9583 as_Register($src2$$reg),
9584 as_Register($src3$$reg));
9585 %}
9586
9587 ins_pipe(imac_reg_reg);
9588 %}
9589
9590 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
9591 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
9592
9593 ins_cost(INSN_COST * 3);
9594 format %{ "smnegl $dst, $src1, $src2" %}
9595
9596 ins_encode %{
9597 __ smnegl(as_Register($dst$$reg),
9598 as_Register($src1$$reg),
9599 as_Register($src2$$reg));
9600 %}
9601
9602 ins_pipe(imac_reg_reg);
9603 %}
9604
9605 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
9606
9607 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
9608 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
9609
9610 ins_cost(INSN_COST * 5);
9611 format %{ "mulw rscratch1, $src1, $src2\n\t"
9612 "maddw $dst, $src3, $src4, rscratch1" %}
9613
9614 ins_encode %{
9615 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
9616 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
9617
9618 ins_pipe(imac_reg_reg);
9619 %}
9620
9621 // Integer Divide
9622
9623 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9624 match(Set dst (DivI src1 src2));
9625
9626 ins_cost(INSN_COST * 19);
9627 format %{ "sdivw $dst, $src1, $src2" %}
9628
9629 ins_encode(aarch64_enc_divw(dst, src1, src2));
9630 ins_pipe(idiv_reg_reg);
9631 %}
9632
9633 // Long Divide
9634
9635 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9636 match(Set dst (DivL src1 src2));
9637
9638 ins_cost(INSN_COST * 35);
9639 format %{ "sdiv $dst, $src1, $src2" %}
9640
9641 ins_encode(aarch64_enc_div(dst, src1, src2));
9642 ins_pipe(ldiv_reg_reg);
9643 %}
9644
9645 // Integer Remainder
9646
9647 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9648 match(Set dst (ModI src1 src2));
9649
9650 ins_cost(INSN_COST * 22);
9651 format %{ "sdivw rscratch1, $src1, $src2\n\t"
9652 "msubw $dst, rscratch1, $src2, $src1" %}
9653
9654 ins_encode(aarch64_enc_modw(dst, src1, src2));
9655 ins_pipe(idiv_reg_reg);
9656 %}
9657
9658 // Long Remainder
9659
9660 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9661 match(Set dst (ModL src1 src2));
9662
9663 ins_cost(INSN_COST * 38);
9664 format %{ "sdiv rscratch1, $src1, $src2\n"
9665 "msub $dst, rscratch1, $src2, $src1" %}
9666
9667 ins_encode(aarch64_enc_mod(dst, src1, src2));
9668 ins_pipe(ldiv_reg_reg);
9669 %}
9670
9671 // Unsigned Integer Divide
9672
9673 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9674 match(Set dst (UDivI src1 src2));
9675
9676 ins_cost(INSN_COST * 19);
9677 format %{ "udivw $dst, $src1, $src2" %}
9678
9679 ins_encode %{
9680 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
9681 %}
9682
9683 ins_pipe(idiv_reg_reg);
9684 %}
9685
9686 // Unsigned Long Divide
9687
9688 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9689 match(Set dst (UDivL src1 src2));
9690
9691 ins_cost(INSN_COST * 35);
9692 format %{ "udiv $dst, $src1, $src2" %}
9693
9694 ins_encode %{
9695 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
9696 %}
9697
9698 ins_pipe(ldiv_reg_reg);
9699 %}
9700
9701 // Unsigned Integer Remainder
9702
9703 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9704 match(Set dst (UModI src1 src2));
9705
9706 ins_cost(INSN_COST * 22);
9707 format %{ "udivw rscratch1, $src1, $src2\n\t"
9708 "msubw $dst, rscratch1, $src2, $src1" %}
9709
9710 ins_encode %{
9711 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
9712 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9713 %}
9714
9715 ins_pipe(idiv_reg_reg);
9716 %}
9717
9718 // Unsigned Long Remainder
9719
9720 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9721 match(Set dst (UModL src1 src2));
9722
9723 ins_cost(INSN_COST * 38);
9724 format %{ "udiv rscratch1, $src1, $src2\n"
9725 "msub $dst, rscratch1, $src2, $src1" %}
9726
9727 ins_encode %{
9728 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
9729 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
9730 %}
9731
9732 ins_pipe(ldiv_reg_reg);
9733 %}
9734
9735 // Integer Shifts
9736
9737 // Shift Left Register
9738 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9739 match(Set dst (LShiftI src1 src2));
9740
9741 ins_cost(INSN_COST * 2);
9742 format %{ "lslvw $dst, $src1, $src2" %}
9743
9744 ins_encode %{
9745 __ lslvw(as_Register($dst$$reg),
9746 as_Register($src1$$reg),
9747 as_Register($src2$$reg));
9748 %}
9749
9750 ins_pipe(ialu_reg_reg_vshift);
9751 %}
9752
9753 // Shift Left Immediate
9754 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9755 match(Set dst (LShiftI src1 src2));
9756
9757 ins_cost(INSN_COST);
9758 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
9759
9760 ins_encode %{
9761 __ lslw(as_Register($dst$$reg),
9762 as_Register($src1$$reg),
9763 $src2$$constant & 0x1f);
9764 %}
9765
9766 ins_pipe(ialu_reg_shift);
9767 %}
9768
9769 // Shift Right Logical Register
9770 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9771 match(Set dst (URShiftI src1 src2));
9772
9773 ins_cost(INSN_COST * 2);
9774 format %{ "lsrvw $dst, $src1, $src2" %}
9775
9776 ins_encode %{
9777 __ lsrvw(as_Register($dst$$reg),
9778 as_Register($src1$$reg),
9779 as_Register($src2$$reg));
9780 %}
9781
9782 ins_pipe(ialu_reg_reg_vshift);
9783 %}
9784
9785 // Shift Right Logical Immediate
9786 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9787 match(Set dst (URShiftI src1 src2));
9788
9789 ins_cost(INSN_COST);
9790 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
9791
9792 ins_encode %{
9793 __ lsrw(as_Register($dst$$reg),
9794 as_Register($src1$$reg),
9795 $src2$$constant & 0x1f);
9796 %}
9797
9798 ins_pipe(ialu_reg_shift);
9799 %}
9800
9801 // Shift Right Arithmetic Register
9802 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9803 match(Set dst (RShiftI src1 src2));
9804
9805 ins_cost(INSN_COST * 2);
9806 format %{ "asrvw $dst, $src1, $src2" %}
9807
9808 ins_encode %{
9809 __ asrvw(as_Register($dst$$reg),
9810 as_Register($src1$$reg),
9811 as_Register($src2$$reg));
9812 %}
9813
9814 ins_pipe(ialu_reg_reg_vshift);
9815 %}
9816
9817 // Shift Right Arithmetic Immediate
9818 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
9819 match(Set dst (RShiftI src1 src2));
9820
9821 ins_cost(INSN_COST);
9822 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
9823
9824 ins_encode %{
9825 __ asrw(as_Register($dst$$reg),
9826 as_Register($src1$$reg),
9827 $src2$$constant & 0x1f);
9828 %}
9829
9830 ins_pipe(ialu_reg_shift);
9831 %}
9832
9833 // Combined Int Mask and Right Shift (using UBFM)
9834 // TODO
9835
9836 // Long Shifts
9837
9838 // Shift Left Register
9839 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9840 match(Set dst (LShiftL src1 src2));
9841
9842 ins_cost(INSN_COST * 2);
9843 format %{ "lslv $dst, $src1, $src2" %}
9844
9845 ins_encode %{
9846 __ lslv(as_Register($dst$$reg),
9847 as_Register($src1$$reg),
9848 as_Register($src2$$reg));
9849 %}
9850
9851 ins_pipe(ialu_reg_reg_vshift);
9852 %}
9853
9854 // Shift Left Immediate
9855 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9856 match(Set dst (LShiftL src1 src2));
9857
9858 ins_cost(INSN_COST);
9859 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
9860
9861 ins_encode %{
9862 __ lsl(as_Register($dst$$reg),
9863 as_Register($src1$$reg),
9864 $src2$$constant & 0x3f);
9865 %}
9866
9867 ins_pipe(ialu_reg_shift);
9868 %}
9869
9870 // Shift Right Logical Register
9871 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9872 match(Set dst (URShiftL src1 src2));
9873
9874 ins_cost(INSN_COST * 2);
9875 format %{ "lsrv $dst, $src1, $src2" %}
9876
9877 ins_encode %{
9878 __ lsrv(as_Register($dst$$reg),
9879 as_Register($src1$$reg),
9880 as_Register($src2$$reg));
9881 %}
9882
9883 ins_pipe(ialu_reg_reg_vshift);
9884 %}
9885
9886 // Shift Right Logical Immediate
9887 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9888 match(Set dst (URShiftL src1 src2));
9889
9890 ins_cost(INSN_COST);
9891 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
9892
9893 ins_encode %{
9894 __ lsr(as_Register($dst$$reg),
9895 as_Register($src1$$reg),
9896 $src2$$constant & 0x3f);
9897 %}
9898
9899 ins_pipe(ialu_reg_shift);
9900 %}
9901
9902 // A special-case pattern for card table stores.
9903 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
9904 match(Set dst (URShiftL (CastP2X src1) src2));
9905
9906 ins_cost(INSN_COST);
9907 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
9908
9909 ins_encode %{
9910 __ lsr(as_Register($dst$$reg),
9911 as_Register($src1$$reg),
9912 $src2$$constant & 0x3f);
9913 %}
9914
9915 ins_pipe(ialu_reg_shift);
9916 %}
9917
9918 // Shift Right Arithmetic Register
9919 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
9920 match(Set dst (RShiftL src1 src2));
9921
9922 ins_cost(INSN_COST * 2);
9923 format %{ "asrv $dst, $src1, $src2" %}
9924
9925 ins_encode %{
9926 __ asrv(as_Register($dst$$reg),
9927 as_Register($src1$$reg),
9928 as_Register($src2$$reg));
9929 %}
9930
9931 ins_pipe(ialu_reg_reg_vshift);
9932 %}
9933
9934 // Shift Right Arithmetic Immediate
9935 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
9936 match(Set dst (RShiftL src1 src2));
9937
9938 ins_cost(INSN_COST);
9939 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
9940
9941 ins_encode %{
9942 __ asr(as_Register($dst$$reg),
9943 as_Register($src1$$reg),
9944 $src2$$constant & 0x3f);
9945 %}
9946
9947 ins_pipe(ialu_reg_shift);
9948 %}
9949
9950 // BEGIN This section of the file is automatically generated. Do not edit --------------
9951 // This section is generated from aarch64_ad.m4
9952
9953 // This pattern is automatically generated from aarch64_ad.m4
9954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9955 instruct regL_not_reg(iRegLNoSp dst,
9956 iRegL src1, immL_M1 m1,
9957 rFlagsReg cr) %{
9958 match(Set dst (XorL src1 m1));
9959 ins_cost(INSN_COST);
9960 format %{ "eon $dst, $src1, zr" %}
9961
9962 ins_encode %{
9963 __ eon(as_Register($dst$$reg),
9964 as_Register($src1$$reg),
9965 zr,
9966 Assembler::LSL, 0);
9967 %}
9968
9969 ins_pipe(ialu_reg);
9970 %}
9971
9972 // This pattern is automatically generated from aarch64_ad.m4
9973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9974 instruct regI_not_reg(iRegINoSp dst,
9975 iRegIorL2I src1, immI_M1 m1,
9976 rFlagsReg cr) %{
9977 match(Set dst (XorI src1 m1));
9978 ins_cost(INSN_COST);
9979 format %{ "eonw $dst, $src1, zr" %}
9980
9981 ins_encode %{
9982 __ eonw(as_Register($dst$$reg),
9983 as_Register($src1$$reg),
9984 zr,
9985 Assembler::LSL, 0);
9986 %}
9987
9988 ins_pipe(ialu_reg);
9989 %}
9990
9991 // This pattern is automatically generated from aarch64_ad.m4
9992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9993 instruct NegI_reg_URShift_reg(iRegINoSp dst,
9994 immI0 zero, iRegIorL2I src1, immI src2) %{
9995 match(Set dst (SubI zero (URShiftI src1 src2)));
9996
9997 ins_cost(1.9 * INSN_COST);
9998 format %{ "negw $dst, $src1, LSR $src2" %}
9999
10000 ins_encode %{
10001 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10002 Assembler::LSR, $src2$$constant & 0x1f);
10003 %}
10004
10005 ins_pipe(ialu_reg_shift);
10006 %}
10007
10008 // This pattern is automatically generated from aarch64_ad.m4
10009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10010 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10011 immI0 zero, iRegIorL2I src1, immI src2) %{
10012 match(Set dst (SubI zero (RShiftI src1 src2)));
10013
10014 ins_cost(1.9 * INSN_COST);
10015 format %{ "negw $dst, $src1, ASR $src2" %}
10016
10017 ins_encode %{
10018 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10019 Assembler::ASR, $src2$$constant & 0x1f);
10020 %}
10021
10022 ins_pipe(ialu_reg_shift);
10023 %}
10024
10025 // This pattern is automatically generated from aarch64_ad.m4
10026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10027 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10028 immI0 zero, iRegIorL2I src1, immI src2) %{
10029 match(Set dst (SubI zero (LShiftI src1 src2)));
10030
10031 ins_cost(1.9 * INSN_COST);
10032 format %{ "negw $dst, $src1, LSL $src2" %}
10033
10034 ins_encode %{
10035 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10036 Assembler::LSL, $src2$$constant & 0x1f);
10037 %}
10038
10039 ins_pipe(ialu_reg_shift);
10040 %}
10041
10042 // This pattern is automatically generated from aarch64_ad.m4
10043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10044 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10045 immL0 zero, iRegL src1, immI src2) %{
10046 match(Set dst (SubL zero (URShiftL src1 src2)));
10047
10048 ins_cost(1.9 * INSN_COST);
10049 format %{ "neg $dst, $src1, LSR $src2" %}
10050
10051 ins_encode %{
10052 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10053 Assembler::LSR, $src2$$constant & 0x3f);
10054 %}
10055
10056 ins_pipe(ialu_reg_shift);
10057 %}
10058
10059 // This pattern is automatically generated from aarch64_ad.m4
10060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10061 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10062 immL0 zero, iRegL src1, immI src2) %{
10063 match(Set dst (SubL zero (RShiftL src1 src2)));
10064
10065 ins_cost(1.9 * INSN_COST);
10066 format %{ "neg $dst, $src1, ASR $src2" %}
10067
10068 ins_encode %{
10069 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10070 Assembler::ASR, $src2$$constant & 0x3f);
10071 %}
10072
10073 ins_pipe(ialu_reg_shift);
10074 %}
10075
10076 // This pattern is automatically generated from aarch64_ad.m4
10077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10078 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10079 immL0 zero, iRegL src1, immI src2) %{
10080 match(Set dst (SubL zero (LShiftL src1 src2)));
10081
10082 ins_cost(1.9 * INSN_COST);
10083 format %{ "neg $dst, $src1, LSL $src2" %}
10084
10085 ins_encode %{
10086 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10087 Assembler::LSL, $src2$$constant & 0x3f);
10088 %}
10089
10090 ins_pipe(ialu_reg_shift);
10091 %}
10092
10093 // This pattern is automatically generated from aarch64_ad.m4
10094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10095 instruct AndI_reg_not_reg(iRegINoSp dst,
10096 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10097 match(Set dst (AndI src1 (XorI src2 m1)));
10098 ins_cost(INSN_COST);
10099 format %{ "bicw $dst, $src1, $src2" %}
10100
10101 ins_encode %{
10102 __ bicw(as_Register($dst$$reg),
10103 as_Register($src1$$reg),
10104 as_Register($src2$$reg),
10105 Assembler::LSL, 0);
10106 %}
10107
10108 ins_pipe(ialu_reg_reg);
10109 %}
10110
10111 // This pattern is automatically generated from aarch64_ad.m4
10112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10113 instruct AndL_reg_not_reg(iRegLNoSp dst,
10114 iRegL src1, iRegL src2, immL_M1 m1) %{
10115 match(Set dst (AndL src1 (XorL src2 m1)));
10116 ins_cost(INSN_COST);
10117 format %{ "bic $dst, $src1, $src2" %}
10118
10119 ins_encode %{
10120 __ bic(as_Register($dst$$reg),
10121 as_Register($src1$$reg),
10122 as_Register($src2$$reg),
10123 Assembler::LSL, 0);
10124 %}
10125
10126 ins_pipe(ialu_reg_reg);
10127 %}
10128
10129 // This pattern is automatically generated from aarch64_ad.m4
10130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10131 instruct OrI_reg_not_reg(iRegINoSp dst,
10132 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10133 match(Set dst (OrI src1 (XorI src2 m1)));
10134 ins_cost(INSN_COST);
10135 format %{ "ornw $dst, $src1, $src2" %}
10136
10137 ins_encode %{
10138 __ ornw(as_Register($dst$$reg),
10139 as_Register($src1$$reg),
10140 as_Register($src2$$reg),
10141 Assembler::LSL, 0);
10142 %}
10143
10144 ins_pipe(ialu_reg_reg);
10145 %}
10146
10147 // This pattern is automatically generated from aarch64_ad.m4
10148 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10149 instruct OrL_reg_not_reg(iRegLNoSp dst,
10150 iRegL src1, iRegL src2, immL_M1 m1) %{
10151 match(Set dst (OrL src1 (XorL src2 m1)));
10152 ins_cost(INSN_COST);
10153 format %{ "orn $dst, $src1, $src2" %}
10154
10155 ins_encode %{
10156 __ orn(as_Register($dst$$reg),
10157 as_Register($src1$$reg),
10158 as_Register($src2$$reg),
10159 Assembler::LSL, 0);
10160 %}
10161
10162 ins_pipe(ialu_reg_reg);
10163 %}
10164
10165 // This pattern is automatically generated from aarch64_ad.m4
10166 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10167 instruct XorI_reg_not_reg(iRegINoSp dst,
10168 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10169 match(Set dst (XorI m1 (XorI src2 src1)));
10170 ins_cost(INSN_COST);
10171 format %{ "eonw $dst, $src1, $src2" %}
10172
10173 ins_encode %{
10174 __ eonw(as_Register($dst$$reg),
10175 as_Register($src1$$reg),
10176 as_Register($src2$$reg),
10177 Assembler::LSL, 0);
10178 %}
10179
10180 ins_pipe(ialu_reg_reg);
10181 %}
10182
10183 // This pattern is automatically generated from aarch64_ad.m4
10184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10185 instruct XorL_reg_not_reg(iRegLNoSp dst,
10186 iRegL src1, iRegL src2, immL_M1 m1) %{
10187 match(Set dst (XorL m1 (XorL src2 src1)));
10188 ins_cost(INSN_COST);
10189 format %{ "eon $dst, $src1, $src2" %}
10190
10191 ins_encode %{
10192 __ eon(as_Register($dst$$reg),
10193 as_Register($src1$$reg),
10194 as_Register($src2$$reg),
10195 Assembler::LSL, 0);
10196 %}
10197
10198 ins_pipe(ialu_reg_reg);
10199 %}
10200
10201 // This pattern is automatically generated from aarch64_ad.m4
10202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10203 // val & (-1 ^ (val >>> shift)) ==> bicw
10204 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10205 iRegIorL2I src1, iRegIorL2I src2,
10206 immI src3, immI_M1 src4) %{
10207 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10208 ins_cost(1.9 * INSN_COST);
10209 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10210
10211 ins_encode %{
10212 __ bicw(as_Register($dst$$reg),
10213 as_Register($src1$$reg),
10214 as_Register($src2$$reg),
10215 Assembler::LSR,
10216 $src3$$constant & 0x1f);
10217 %}
10218
10219 ins_pipe(ialu_reg_reg_shift);
10220 %}
10221
10222 // This pattern is automatically generated from aarch64_ad.m4
10223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10224 // val & (-1 ^ (val >>> shift)) ==> bic
10225 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10226 iRegL src1, iRegL src2,
10227 immI src3, immL_M1 src4) %{
10228 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10229 ins_cost(1.9 * INSN_COST);
10230 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10231
10232 ins_encode %{
10233 __ bic(as_Register($dst$$reg),
10234 as_Register($src1$$reg),
10235 as_Register($src2$$reg),
10236 Assembler::LSR,
10237 $src3$$constant & 0x3f);
10238 %}
10239
10240 ins_pipe(ialu_reg_reg_shift);
10241 %}
10242
10243 // This pattern is automatically generated from aarch64_ad.m4
10244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10245 // val & (-1 ^ (val >> shift)) ==> bicw
10246 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10247 iRegIorL2I src1, iRegIorL2I src2,
10248 immI src3, immI_M1 src4) %{
10249 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10250 ins_cost(1.9 * INSN_COST);
10251 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10252
10253 ins_encode %{
10254 __ bicw(as_Register($dst$$reg),
10255 as_Register($src1$$reg),
10256 as_Register($src2$$reg),
10257 Assembler::ASR,
10258 $src3$$constant & 0x1f);
10259 %}
10260
10261 ins_pipe(ialu_reg_reg_shift);
10262 %}
10263
10264 // This pattern is automatically generated from aarch64_ad.m4
10265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10266 // val & (-1 ^ (val >> shift)) ==> bic
10267 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10268 iRegL src1, iRegL src2,
10269 immI src3, immL_M1 src4) %{
10270 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10271 ins_cost(1.9 * INSN_COST);
10272 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10273
10274 ins_encode %{
10275 __ bic(as_Register($dst$$reg),
10276 as_Register($src1$$reg),
10277 as_Register($src2$$reg),
10278 Assembler::ASR,
10279 $src3$$constant & 0x3f);
10280 %}
10281
10282 ins_pipe(ialu_reg_reg_shift);
10283 %}
10284
10285 // This pattern is automatically generated from aarch64_ad.m4
10286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10287 // val & (-1 ^ (val ror shift)) ==> bicw
10288 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10289 iRegIorL2I src1, iRegIorL2I src2,
10290 immI src3, immI_M1 src4) %{
10291 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10292 ins_cost(1.9 * INSN_COST);
10293 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10294
10295 ins_encode %{
10296 __ bicw(as_Register($dst$$reg),
10297 as_Register($src1$$reg),
10298 as_Register($src2$$reg),
10299 Assembler::ROR,
10300 $src3$$constant & 0x1f);
10301 %}
10302
10303 ins_pipe(ialu_reg_reg_shift);
10304 %}
10305
10306 // This pattern is automatically generated from aarch64_ad.m4
10307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10308 // val & (-1 ^ (val ror shift)) ==> bic
10309 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10310 iRegL src1, iRegL src2,
10311 immI src3, immL_M1 src4) %{
10312 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10313 ins_cost(1.9 * INSN_COST);
10314 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10315
10316 ins_encode %{
10317 __ bic(as_Register($dst$$reg),
10318 as_Register($src1$$reg),
10319 as_Register($src2$$reg),
10320 Assembler::ROR,
10321 $src3$$constant & 0x3f);
10322 %}
10323
10324 ins_pipe(ialu_reg_reg_shift);
10325 %}
10326
10327 // This pattern is automatically generated from aarch64_ad.m4
10328 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10329 // val & (-1 ^ (val << shift)) ==> bicw
10330 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
10331 iRegIorL2I src1, iRegIorL2I src2,
10332 immI src3, immI_M1 src4) %{
10333 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
10334 ins_cost(1.9 * INSN_COST);
10335 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
10336
10337 ins_encode %{
10338 __ bicw(as_Register($dst$$reg),
10339 as_Register($src1$$reg),
10340 as_Register($src2$$reg),
10341 Assembler::LSL,
10342 $src3$$constant & 0x1f);
10343 %}
10344
10345 ins_pipe(ialu_reg_reg_shift);
10346 %}
10347
10348 // This pattern is automatically generated from aarch64_ad.m4
10349 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10350 // val & (-1 ^ (val << shift)) ==> bic
10351 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
10352 iRegL src1, iRegL src2,
10353 immI src3, immL_M1 src4) %{
10354 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
10355 ins_cost(1.9 * INSN_COST);
10356 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
10357
10358 ins_encode %{
10359 __ bic(as_Register($dst$$reg),
10360 as_Register($src1$$reg),
10361 as_Register($src2$$reg),
10362 Assembler::LSL,
10363 $src3$$constant & 0x3f);
10364 %}
10365
10366 ins_pipe(ialu_reg_reg_shift);
10367 %}
10368
10369 // This pattern is automatically generated from aarch64_ad.m4
10370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10371 // val ^ (-1 ^ (val >>> shift)) ==> eonw
10372 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
10373 iRegIorL2I src1, iRegIorL2I src2,
10374 immI src3, immI_M1 src4) %{
10375 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
10376 ins_cost(1.9 * INSN_COST);
10377 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
10378
10379 ins_encode %{
10380 __ eonw(as_Register($dst$$reg),
10381 as_Register($src1$$reg),
10382 as_Register($src2$$reg),
10383 Assembler::LSR,
10384 $src3$$constant & 0x1f);
10385 %}
10386
10387 ins_pipe(ialu_reg_reg_shift);
10388 %}
10389
10390 // This pattern is automatically generated from aarch64_ad.m4
10391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10392 // val ^ (-1 ^ (val >>> shift)) ==> eon
10393 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
10394 iRegL src1, iRegL src2,
10395 immI src3, immL_M1 src4) %{
10396 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
10397 ins_cost(1.9 * INSN_COST);
10398 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
10399
10400 ins_encode %{
10401 __ eon(as_Register($dst$$reg),
10402 as_Register($src1$$reg),
10403 as_Register($src2$$reg),
10404 Assembler::LSR,
10405 $src3$$constant & 0x3f);
10406 %}
10407
10408 ins_pipe(ialu_reg_reg_shift);
10409 %}
10410
10411 // This pattern is automatically generated from aarch64_ad.m4
10412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10413 // val ^ (-1 ^ (val >> shift)) ==> eonw
10414 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
10415 iRegIorL2I src1, iRegIorL2I src2,
10416 immI src3, immI_M1 src4) %{
10417 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
10418 ins_cost(1.9 * INSN_COST);
10419 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
10420
10421 ins_encode %{
10422 __ eonw(as_Register($dst$$reg),
10423 as_Register($src1$$reg),
10424 as_Register($src2$$reg),
10425 Assembler::ASR,
10426 $src3$$constant & 0x1f);
10427 %}
10428
10429 ins_pipe(ialu_reg_reg_shift);
10430 %}
10431
10432 // This pattern is automatically generated from aarch64_ad.m4
10433 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10434 // val ^ (-1 ^ (val >> shift)) ==> eon
10435 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
10436 iRegL src1, iRegL src2,
10437 immI src3, immL_M1 src4) %{
10438 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
10439 ins_cost(1.9 * INSN_COST);
10440 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
10441
10442 ins_encode %{
10443 __ eon(as_Register($dst$$reg),
10444 as_Register($src1$$reg),
10445 as_Register($src2$$reg),
10446 Assembler::ASR,
10447 $src3$$constant & 0x3f);
10448 %}
10449
10450 ins_pipe(ialu_reg_reg_shift);
10451 %}
10452
10453 // This pattern is automatically generated from aarch64_ad.m4
10454 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10455 // val ^ (-1 ^ (val ror shift)) ==> eonw
10456 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
10457 iRegIorL2I src1, iRegIorL2I src2,
10458 immI src3, immI_M1 src4) %{
10459 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
10460 ins_cost(1.9 * INSN_COST);
10461 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
10462
10463 ins_encode %{
10464 __ eonw(as_Register($dst$$reg),
10465 as_Register($src1$$reg),
10466 as_Register($src2$$reg),
10467 Assembler::ROR,
10468 $src3$$constant & 0x1f);
10469 %}
10470
10471 ins_pipe(ialu_reg_reg_shift);
10472 %}
10473
10474 // This pattern is automatically generated from aarch64_ad.m4
10475 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10476 // val ^ (-1 ^ (val ror shift)) ==> eon
10477 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
10478 iRegL src1, iRegL src2,
10479 immI src3, immL_M1 src4) %{
10480 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
10481 ins_cost(1.9 * INSN_COST);
10482 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
10483
10484 ins_encode %{
10485 __ eon(as_Register($dst$$reg),
10486 as_Register($src1$$reg),
10487 as_Register($src2$$reg),
10488 Assembler::ROR,
10489 $src3$$constant & 0x3f);
10490 %}
10491
10492 ins_pipe(ialu_reg_reg_shift);
10493 %}
10494
10495 // This pattern is automatically generated from aarch64_ad.m4
10496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10497 // val ^ (-1 ^ (val << shift)) ==> eonw
10498 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
10499 iRegIorL2I src1, iRegIorL2I src2,
10500 immI src3, immI_M1 src4) %{
10501 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
10502 ins_cost(1.9 * INSN_COST);
10503 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
10504
10505 ins_encode %{
10506 __ eonw(as_Register($dst$$reg),
10507 as_Register($src1$$reg),
10508 as_Register($src2$$reg),
10509 Assembler::LSL,
10510 $src3$$constant & 0x1f);
10511 %}
10512
10513 ins_pipe(ialu_reg_reg_shift);
10514 %}
10515
10516 // This pattern is automatically generated from aarch64_ad.m4
10517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10518 // val ^ (-1 ^ (val << shift)) ==> eon
10519 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
10520 iRegL src1, iRegL src2,
10521 immI src3, immL_M1 src4) %{
10522 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
10523 ins_cost(1.9 * INSN_COST);
10524 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
10525
10526 ins_encode %{
10527 __ eon(as_Register($dst$$reg),
10528 as_Register($src1$$reg),
10529 as_Register($src2$$reg),
10530 Assembler::LSL,
10531 $src3$$constant & 0x3f);
10532 %}
10533
10534 ins_pipe(ialu_reg_reg_shift);
10535 %}
10536
10537 // This pattern is automatically generated from aarch64_ad.m4
10538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10539 // val | (-1 ^ (val >>> shift)) ==> ornw
10540 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
10541 iRegIorL2I src1, iRegIorL2I src2,
10542 immI src3, immI_M1 src4) %{
10543 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
10544 ins_cost(1.9 * INSN_COST);
10545 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
10546
10547 ins_encode %{
10548 __ ornw(as_Register($dst$$reg),
10549 as_Register($src1$$reg),
10550 as_Register($src2$$reg),
10551 Assembler::LSR,
10552 $src3$$constant & 0x1f);
10553 %}
10554
10555 ins_pipe(ialu_reg_reg_shift);
10556 %}
10557
10558 // This pattern is automatically generated from aarch64_ad.m4
10559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10560 // val | (-1 ^ (val >>> shift)) ==> orn
10561 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
10562 iRegL src1, iRegL src2,
10563 immI src3, immL_M1 src4) %{
10564 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
10565 ins_cost(1.9 * INSN_COST);
10566 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
10567
10568 ins_encode %{
10569 __ orn(as_Register($dst$$reg),
10570 as_Register($src1$$reg),
10571 as_Register($src2$$reg),
10572 Assembler::LSR,
10573 $src3$$constant & 0x3f);
10574 %}
10575
10576 ins_pipe(ialu_reg_reg_shift);
10577 %}
10578
10579 // This pattern is automatically generated from aarch64_ad.m4
10580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10581 // val | (-1 ^ (val >> shift)) ==> ornw
10582 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
10583 iRegIorL2I src1, iRegIorL2I src2,
10584 immI src3, immI_M1 src4) %{
10585 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
10586 ins_cost(1.9 * INSN_COST);
10587 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
10588
10589 ins_encode %{
10590 __ ornw(as_Register($dst$$reg),
10591 as_Register($src1$$reg),
10592 as_Register($src2$$reg),
10593 Assembler::ASR,
10594 $src3$$constant & 0x1f);
10595 %}
10596
10597 ins_pipe(ialu_reg_reg_shift);
10598 %}
10599
10600 // This pattern is automatically generated from aarch64_ad.m4
10601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10602 // val | (-1 ^ (val >> shift)) ==> orn
10603 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
10604 iRegL src1, iRegL src2,
10605 immI src3, immL_M1 src4) %{
10606 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
10607 ins_cost(1.9 * INSN_COST);
10608 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
10609
10610 ins_encode %{
10611 __ orn(as_Register($dst$$reg),
10612 as_Register($src1$$reg),
10613 as_Register($src2$$reg),
10614 Assembler::ASR,
10615 $src3$$constant & 0x3f);
10616 %}
10617
10618 ins_pipe(ialu_reg_reg_shift);
10619 %}
10620
10621 // This pattern is automatically generated from aarch64_ad.m4
10622 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10623 // val | (-1 ^ (val ror shift)) ==> ornw
10624 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
10625 iRegIorL2I src1, iRegIorL2I src2,
10626 immI src3, immI_M1 src4) %{
10627 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
10628 ins_cost(1.9 * INSN_COST);
10629 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
10630
10631 ins_encode %{
10632 __ ornw(as_Register($dst$$reg),
10633 as_Register($src1$$reg),
10634 as_Register($src2$$reg),
10635 Assembler::ROR,
10636 $src3$$constant & 0x1f);
10637 %}
10638
10639 ins_pipe(ialu_reg_reg_shift);
10640 %}
10641
10642 // This pattern is automatically generated from aarch64_ad.m4
10643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10644 // val | (-1 ^ (val ror shift)) ==> orn
10645 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
10646 iRegL src1, iRegL src2,
10647 immI src3, immL_M1 src4) %{
10648 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
10649 ins_cost(1.9 * INSN_COST);
10650 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
10651
10652 ins_encode %{
10653 __ orn(as_Register($dst$$reg),
10654 as_Register($src1$$reg),
10655 as_Register($src2$$reg),
10656 Assembler::ROR,
10657 $src3$$constant & 0x3f);
10658 %}
10659
10660 ins_pipe(ialu_reg_reg_shift);
10661 %}
10662
10663 // This pattern is automatically generated from aarch64_ad.m4
10664 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10665 // val | (-1 ^ (val << shift)) ==> ornw
10666 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
10667 iRegIorL2I src1, iRegIorL2I src2,
10668 immI src3, immI_M1 src4) %{
10669 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
10670 ins_cost(1.9 * INSN_COST);
10671 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
10672
10673 ins_encode %{
10674 __ ornw(as_Register($dst$$reg),
10675 as_Register($src1$$reg),
10676 as_Register($src2$$reg),
10677 Assembler::LSL,
10678 $src3$$constant & 0x1f);
10679 %}
10680
10681 ins_pipe(ialu_reg_reg_shift);
10682 %}
10683
10684 // This pattern is automatically generated from aarch64_ad.m4
10685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10686 // val | (-1 ^ (val << shift)) ==> orn
10687 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
10688 iRegL src1, iRegL src2,
10689 immI src3, immL_M1 src4) %{
10690 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
10691 ins_cost(1.9 * INSN_COST);
10692 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
10693
10694 ins_encode %{
10695 __ orn(as_Register($dst$$reg),
10696 as_Register($src1$$reg),
10697 as_Register($src2$$reg),
10698 Assembler::LSL,
10699 $src3$$constant & 0x3f);
10700 %}
10701
10702 ins_pipe(ialu_reg_reg_shift);
10703 %}
10704
10705 // This pattern is automatically generated from aarch64_ad.m4
10706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10707 instruct AndI_reg_URShift_reg(iRegINoSp dst,
10708 iRegIorL2I src1, iRegIorL2I src2,
10709 immI src3) %{
10710 match(Set dst (AndI src1 (URShiftI src2 src3)));
10711
10712 ins_cost(1.9 * INSN_COST);
10713 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
10714
10715 ins_encode %{
10716 __ andw(as_Register($dst$$reg),
10717 as_Register($src1$$reg),
10718 as_Register($src2$$reg),
10719 Assembler::LSR,
10720 $src3$$constant & 0x1f);
10721 %}
10722
10723 ins_pipe(ialu_reg_reg_shift);
10724 %}
10725
10726 // This pattern is automatically generated from aarch64_ad.m4
10727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10728 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
10729 iRegL src1, iRegL src2,
10730 immI src3) %{
10731 match(Set dst (AndL src1 (URShiftL src2 src3)));
10732
10733 ins_cost(1.9 * INSN_COST);
10734 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
10735
10736 ins_encode %{
10737 __ andr(as_Register($dst$$reg),
10738 as_Register($src1$$reg),
10739 as_Register($src2$$reg),
10740 Assembler::LSR,
10741 $src3$$constant & 0x3f);
10742 %}
10743
10744 ins_pipe(ialu_reg_reg_shift);
10745 %}
10746
10747 // This pattern is automatically generated from aarch64_ad.m4
10748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10749 instruct AndI_reg_RShift_reg(iRegINoSp dst,
10750 iRegIorL2I src1, iRegIorL2I src2,
10751 immI src3) %{
10752 match(Set dst (AndI src1 (RShiftI src2 src3)));
10753
10754 ins_cost(1.9 * INSN_COST);
10755 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
10756
10757 ins_encode %{
10758 __ andw(as_Register($dst$$reg),
10759 as_Register($src1$$reg),
10760 as_Register($src2$$reg),
10761 Assembler::ASR,
10762 $src3$$constant & 0x1f);
10763 %}
10764
10765 ins_pipe(ialu_reg_reg_shift);
10766 %}
10767
10768 // This pattern is automatically generated from aarch64_ad.m4
10769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10770 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
10771 iRegL src1, iRegL src2,
10772 immI src3) %{
10773 match(Set dst (AndL src1 (RShiftL src2 src3)));
10774
10775 ins_cost(1.9 * INSN_COST);
10776 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
10777
10778 ins_encode %{
10779 __ andr(as_Register($dst$$reg),
10780 as_Register($src1$$reg),
10781 as_Register($src2$$reg),
10782 Assembler::ASR,
10783 $src3$$constant & 0x3f);
10784 %}
10785
10786 ins_pipe(ialu_reg_reg_shift);
10787 %}
10788
10789 // This pattern is automatically generated from aarch64_ad.m4
10790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10791 instruct AndI_reg_LShift_reg(iRegINoSp dst,
10792 iRegIorL2I src1, iRegIorL2I src2,
10793 immI src3) %{
10794 match(Set dst (AndI src1 (LShiftI src2 src3)));
10795
10796 ins_cost(1.9 * INSN_COST);
10797 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
10798
10799 ins_encode %{
10800 __ andw(as_Register($dst$$reg),
10801 as_Register($src1$$reg),
10802 as_Register($src2$$reg),
10803 Assembler::LSL,
10804 $src3$$constant & 0x1f);
10805 %}
10806
10807 ins_pipe(ialu_reg_reg_shift);
10808 %}
10809
10810 // This pattern is automatically generated from aarch64_ad.m4
10811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10812 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
10813 iRegL src1, iRegL src2,
10814 immI src3) %{
10815 match(Set dst (AndL src1 (LShiftL src2 src3)));
10816
10817 ins_cost(1.9 * INSN_COST);
10818 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
10819
10820 ins_encode %{
10821 __ andr(as_Register($dst$$reg),
10822 as_Register($src1$$reg),
10823 as_Register($src2$$reg),
10824 Assembler::LSL,
10825 $src3$$constant & 0x3f);
10826 %}
10827
10828 ins_pipe(ialu_reg_reg_shift);
10829 %}
10830
10831 // This pattern is automatically generated from aarch64_ad.m4
10832 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10833 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
10834 iRegIorL2I src1, iRegIorL2I src2,
10835 immI src3) %{
10836 match(Set dst (AndI src1 (RotateRight src2 src3)));
10837
10838 ins_cost(1.9 * INSN_COST);
10839 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
10840
10841 ins_encode %{
10842 __ andw(as_Register($dst$$reg),
10843 as_Register($src1$$reg),
10844 as_Register($src2$$reg),
10845 Assembler::ROR,
10846 $src3$$constant & 0x1f);
10847 %}
10848
10849 ins_pipe(ialu_reg_reg_shift);
10850 %}
10851
10852 // This pattern is automatically generated from aarch64_ad.m4
10853 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10854 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
10855 iRegL src1, iRegL src2,
10856 immI src3) %{
10857 match(Set dst (AndL src1 (RotateRight src2 src3)));
10858
10859 ins_cost(1.9 * INSN_COST);
10860 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
10861
10862 ins_encode %{
10863 __ andr(as_Register($dst$$reg),
10864 as_Register($src1$$reg),
10865 as_Register($src2$$reg),
10866 Assembler::ROR,
10867 $src3$$constant & 0x3f);
10868 %}
10869
10870 ins_pipe(ialu_reg_reg_shift);
10871 %}
10872
10873 // This pattern is automatically generated from aarch64_ad.m4
10874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10875 instruct XorI_reg_URShift_reg(iRegINoSp dst,
10876 iRegIorL2I src1, iRegIorL2I src2,
10877 immI src3) %{
10878 match(Set dst (XorI src1 (URShiftI src2 src3)));
10879
10880 ins_cost(1.9 * INSN_COST);
10881 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
10882
10883 ins_encode %{
10884 __ eorw(as_Register($dst$$reg),
10885 as_Register($src1$$reg),
10886 as_Register($src2$$reg),
10887 Assembler::LSR,
10888 $src3$$constant & 0x1f);
10889 %}
10890
10891 ins_pipe(ialu_reg_reg_shift);
10892 %}
10893
10894 // This pattern is automatically generated from aarch64_ad.m4
10895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10896 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
10897 iRegL src1, iRegL src2,
10898 immI src3) %{
10899 match(Set dst (XorL src1 (URShiftL src2 src3)));
10900
10901 ins_cost(1.9 * INSN_COST);
10902 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
10903
10904 ins_encode %{
10905 __ eor(as_Register($dst$$reg),
10906 as_Register($src1$$reg),
10907 as_Register($src2$$reg),
10908 Assembler::LSR,
10909 $src3$$constant & 0x3f);
10910 %}
10911
10912 ins_pipe(ialu_reg_reg_shift);
10913 %}
10914
10915 // This pattern is automatically generated from aarch64_ad.m4
10916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10917 instruct XorI_reg_RShift_reg(iRegINoSp dst,
10918 iRegIorL2I src1, iRegIorL2I src2,
10919 immI src3) %{
10920 match(Set dst (XorI src1 (RShiftI src2 src3)));
10921
10922 ins_cost(1.9 * INSN_COST);
10923 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
10924
10925 ins_encode %{
10926 __ eorw(as_Register($dst$$reg),
10927 as_Register($src1$$reg),
10928 as_Register($src2$$reg),
10929 Assembler::ASR,
10930 $src3$$constant & 0x1f);
10931 %}
10932
10933 ins_pipe(ialu_reg_reg_shift);
10934 %}
10935
10936 // This pattern is automatically generated from aarch64_ad.m4
10937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10938 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
10939 iRegL src1, iRegL src2,
10940 immI src3) %{
10941 match(Set dst (XorL src1 (RShiftL src2 src3)));
10942
10943 ins_cost(1.9 * INSN_COST);
10944 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
10945
10946 ins_encode %{
10947 __ eor(as_Register($dst$$reg),
10948 as_Register($src1$$reg),
10949 as_Register($src2$$reg),
10950 Assembler::ASR,
10951 $src3$$constant & 0x3f);
10952 %}
10953
10954 ins_pipe(ialu_reg_reg_shift);
10955 %}
10956
10957 // This pattern is automatically generated from aarch64_ad.m4
10958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10959 instruct XorI_reg_LShift_reg(iRegINoSp dst,
10960 iRegIorL2I src1, iRegIorL2I src2,
10961 immI src3) %{
10962 match(Set dst (XorI src1 (LShiftI src2 src3)));
10963
10964 ins_cost(1.9 * INSN_COST);
10965 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
10966
10967 ins_encode %{
10968 __ eorw(as_Register($dst$$reg),
10969 as_Register($src1$$reg),
10970 as_Register($src2$$reg),
10971 Assembler::LSL,
10972 $src3$$constant & 0x1f);
10973 %}
10974
10975 ins_pipe(ialu_reg_reg_shift);
10976 %}
10977
10978 // This pattern is automatically generated from aarch64_ad.m4
10979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10980 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
10981 iRegL src1, iRegL src2,
10982 immI src3) %{
10983 match(Set dst (XorL src1 (LShiftL src2 src3)));
10984
10985 ins_cost(1.9 * INSN_COST);
10986 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
10987
10988 ins_encode %{
10989 __ eor(as_Register($dst$$reg),
10990 as_Register($src1$$reg),
10991 as_Register($src2$$reg),
10992 Assembler::LSL,
10993 $src3$$constant & 0x3f);
10994 %}
10995
10996 ins_pipe(ialu_reg_reg_shift);
10997 %}
10998
10999 // This pattern is automatically generated from aarch64_ad.m4
11000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11001 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11002 iRegIorL2I src1, iRegIorL2I src2,
11003 immI src3) %{
11004 match(Set dst (XorI src1 (RotateRight src2 src3)));
11005
11006 ins_cost(1.9 * INSN_COST);
11007 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11008
11009 ins_encode %{
11010 __ eorw(as_Register($dst$$reg),
11011 as_Register($src1$$reg),
11012 as_Register($src2$$reg),
11013 Assembler::ROR,
11014 $src3$$constant & 0x1f);
11015 %}
11016
11017 ins_pipe(ialu_reg_reg_shift);
11018 %}
11019
11020 // This pattern is automatically generated from aarch64_ad.m4
11021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11022 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11023 iRegL src1, iRegL src2,
11024 immI src3) %{
11025 match(Set dst (XorL src1 (RotateRight src2 src3)));
11026
11027 ins_cost(1.9 * INSN_COST);
11028 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11029
11030 ins_encode %{
11031 __ eor(as_Register($dst$$reg),
11032 as_Register($src1$$reg),
11033 as_Register($src2$$reg),
11034 Assembler::ROR,
11035 $src3$$constant & 0x3f);
11036 %}
11037
11038 ins_pipe(ialu_reg_reg_shift);
11039 %}
11040
11041 // This pattern is automatically generated from aarch64_ad.m4
11042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11043 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11044 iRegIorL2I src1, iRegIorL2I src2,
11045 immI src3) %{
11046 match(Set dst (OrI src1 (URShiftI src2 src3)));
11047
11048 ins_cost(1.9 * INSN_COST);
11049 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11050
11051 ins_encode %{
11052 __ orrw(as_Register($dst$$reg),
11053 as_Register($src1$$reg),
11054 as_Register($src2$$reg),
11055 Assembler::LSR,
11056 $src3$$constant & 0x1f);
11057 %}
11058
11059 ins_pipe(ialu_reg_reg_shift);
11060 %}
11061
11062 // This pattern is automatically generated from aarch64_ad.m4
11063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11064 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11065 iRegL src1, iRegL src2,
11066 immI src3) %{
11067 match(Set dst (OrL src1 (URShiftL src2 src3)));
11068
11069 ins_cost(1.9 * INSN_COST);
11070 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11071
11072 ins_encode %{
11073 __ orr(as_Register($dst$$reg),
11074 as_Register($src1$$reg),
11075 as_Register($src2$$reg),
11076 Assembler::LSR,
11077 $src3$$constant & 0x3f);
11078 %}
11079
11080 ins_pipe(ialu_reg_reg_shift);
11081 %}
11082
11083 // This pattern is automatically generated from aarch64_ad.m4
11084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11085 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11086 iRegIorL2I src1, iRegIorL2I src2,
11087 immI src3) %{
11088 match(Set dst (OrI src1 (RShiftI src2 src3)));
11089
11090 ins_cost(1.9 * INSN_COST);
11091 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11092
11093 ins_encode %{
11094 __ orrw(as_Register($dst$$reg),
11095 as_Register($src1$$reg),
11096 as_Register($src2$$reg),
11097 Assembler::ASR,
11098 $src3$$constant & 0x1f);
11099 %}
11100
11101 ins_pipe(ialu_reg_reg_shift);
11102 %}
11103
11104 // This pattern is automatically generated from aarch64_ad.m4
11105 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11106 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11107 iRegL src1, iRegL src2,
11108 immI src3) %{
11109 match(Set dst (OrL src1 (RShiftL src2 src3)));
11110
11111 ins_cost(1.9 * INSN_COST);
11112 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11113
11114 ins_encode %{
11115 __ orr(as_Register($dst$$reg),
11116 as_Register($src1$$reg),
11117 as_Register($src2$$reg),
11118 Assembler::ASR,
11119 $src3$$constant & 0x3f);
11120 %}
11121
11122 ins_pipe(ialu_reg_reg_shift);
11123 %}
11124
11125 // This pattern is automatically generated from aarch64_ad.m4
11126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11127 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11128 iRegIorL2I src1, iRegIorL2I src2,
11129 immI src3) %{
11130 match(Set dst (OrI src1 (LShiftI src2 src3)));
11131
11132 ins_cost(1.9 * INSN_COST);
11133 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11134
11135 ins_encode %{
11136 __ orrw(as_Register($dst$$reg),
11137 as_Register($src1$$reg),
11138 as_Register($src2$$reg),
11139 Assembler::LSL,
11140 $src3$$constant & 0x1f);
11141 %}
11142
11143 ins_pipe(ialu_reg_reg_shift);
11144 %}
11145
11146 // This pattern is automatically generated from aarch64_ad.m4
11147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11148 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11149 iRegL src1, iRegL src2,
11150 immI src3) %{
11151 match(Set dst (OrL src1 (LShiftL src2 src3)));
11152
11153 ins_cost(1.9 * INSN_COST);
11154 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11155
11156 ins_encode %{
11157 __ orr(as_Register($dst$$reg),
11158 as_Register($src1$$reg),
11159 as_Register($src2$$reg),
11160 Assembler::LSL,
11161 $src3$$constant & 0x3f);
11162 %}
11163
11164 ins_pipe(ialu_reg_reg_shift);
11165 %}
11166
11167 // This pattern is automatically generated from aarch64_ad.m4
11168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11169 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11170 iRegIorL2I src1, iRegIorL2I src2,
11171 immI src3) %{
11172 match(Set dst (OrI src1 (RotateRight src2 src3)));
11173
11174 ins_cost(1.9 * INSN_COST);
11175 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11176
11177 ins_encode %{
11178 __ orrw(as_Register($dst$$reg),
11179 as_Register($src1$$reg),
11180 as_Register($src2$$reg),
11181 Assembler::ROR,
11182 $src3$$constant & 0x1f);
11183 %}
11184
11185 ins_pipe(ialu_reg_reg_shift);
11186 %}
11187
11188 // This pattern is automatically generated from aarch64_ad.m4
11189 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11190 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11191 iRegL src1, iRegL src2,
11192 immI src3) %{
11193 match(Set dst (OrL src1 (RotateRight src2 src3)));
11194
11195 ins_cost(1.9 * INSN_COST);
11196 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11197
11198 ins_encode %{
11199 __ orr(as_Register($dst$$reg),
11200 as_Register($src1$$reg),
11201 as_Register($src2$$reg),
11202 Assembler::ROR,
11203 $src3$$constant & 0x3f);
11204 %}
11205
11206 ins_pipe(ialu_reg_reg_shift);
11207 %}
11208
11209 // This pattern is automatically generated from aarch64_ad.m4
11210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11211 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11212 iRegIorL2I src1, iRegIorL2I src2,
11213 immI src3) %{
11214 match(Set dst (AddI src1 (URShiftI src2 src3)));
11215
11216 ins_cost(1.9 * INSN_COST);
11217 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11218
11219 ins_encode %{
11220 __ addw(as_Register($dst$$reg),
11221 as_Register($src1$$reg),
11222 as_Register($src2$$reg),
11223 Assembler::LSR,
11224 $src3$$constant & 0x1f);
11225 %}
11226
11227 ins_pipe(ialu_reg_reg_shift);
11228 %}
11229
11230 // This pattern is automatically generated from aarch64_ad.m4
11231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11232 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11233 iRegL src1, iRegL src2,
11234 immI src3) %{
11235 match(Set dst (AddL src1 (URShiftL src2 src3)));
11236
11237 ins_cost(1.9 * INSN_COST);
11238 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11239
11240 ins_encode %{
11241 __ add(as_Register($dst$$reg),
11242 as_Register($src1$$reg),
11243 as_Register($src2$$reg),
11244 Assembler::LSR,
11245 $src3$$constant & 0x3f);
11246 %}
11247
11248 ins_pipe(ialu_reg_reg_shift);
11249 %}
11250
11251 // This pattern is automatically generated from aarch64_ad.m4
11252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11253 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11254 iRegIorL2I src1, iRegIorL2I src2,
11255 immI src3) %{
11256 match(Set dst (AddI src1 (RShiftI src2 src3)));
11257
11258 ins_cost(1.9 * INSN_COST);
11259 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11260
11261 ins_encode %{
11262 __ addw(as_Register($dst$$reg),
11263 as_Register($src1$$reg),
11264 as_Register($src2$$reg),
11265 Assembler::ASR,
11266 $src3$$constant & 0x1f);
11267 %}
11268
11269 ins_pipe(ialu_reg_reg_shift);
11270 %}
11271
11272 // This pattern is automatically generated from aarch64_ad.m4
11273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11274 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11275 iRegL src1, iRegL src2,
11276 immI src3) %{
11277 match(Set dst (AddL src1 (RShiftL src2 src3)));
11278
11279 ins_cost(1.9 * INSN_COST);
11280 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11281
11282 ins_encode %{
11283 __ add(as_Register($dst$$reg),
11284 as_Register($src1$$reg),
11285 as_Register($src2$$reg),
11286 Assembler::ASR,
11287 $src3$$constant & 0x3f);
11288 %}
11289
11290 ins_pipe(ialu_reg_reg_shift);
11291 %}
11292
11293 // This pattern is automatically generated from aarch64_ad.m4
11294 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11295 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11296 iRegIorL2I src1, iRegIorL2I src2,
11297 immI src3) %{
11298 match(Set dst (AddI src1 (LShiftI src2 src3)));
11299
11300 ins_cost(1.9 * INSN_COST);
11301 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11302
11303 ins_encode %{
11304 __ addw(as_Register($dst$$reg),
11305 as_Register($src1$$reg),
11306 as_Register($src2$$reg),
11307 Assembler::LSL,
11308 $src3$$constant & 0x1f);
11309 %}
11310
11311 ins_pipe(ialu_reg_reg_shift);
11312 %}
11313
11314 // This pattern is automatically generated from aarch64_ad.m4
11315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11316 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11317 iRegL src1, iRegL src2,
11318 immI src3) %{
11319 match(Set dst (AddL src1 (LShiftL src2 src3)));
11320
11321 ins_cost(1.9 * INSN_COST);
11322 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11323
11324 ins_encode %{
11325 __ add(as_Register($dst$$reg),
11326 as_Register($src1$$reg),
11327 as_Register($src2$$reg),
11328 Assembler::LSL,
11329 $src3$$constant & 0x3f);
11330 %}
11331
11332 ins_pipe(ialu_reg_reg_shift);
11333 %}
11334
11335 // This pattern is automatically generated from aarch64_ad.m4
11336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11337 instruct SubI_reg_URShift_reg(iRegINoSp dst,
11338 iRegIorL2I src1, iRegIorL2I src2,
11339 immI src3) %{
11340 match(Set dst (SubI src1 (URShiftI src2 src3)));
11341
11342 ins_cost(1.9 * INSN_COST);
11343 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
11344
11345 ins_encode %{
11346 __ subw(as_Register($dst$$reg),
11347 as_Register($src1$$reg),
11348 as_Register($src2$$reg),
11349 Assembler::LSR,
11350 $src3$$constant & 0x1f);
11351 %}
11352
11353 ins_pipe(ialu_reg_reg_shift);
11354 %}
11355
11356 // This pattern is automatically generated from aarch64_ad.m4
11357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11358 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
11359 iRegL src1, iRegL src2,
11360 immI src3) %{
11361 match(Set dst (SubL src1 (URShiftL src2 src3)));
11362
11363 ins_cost(1.9 * INSN_COST);
11364 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
11365
11366 ins_encode %{
11367 __ sub(as_Register($dst$$reg),
11368 as_Register($src1$$reg),
11369 as_Register($src2$$reg),
11370 Assembler::LSR,
11371 $src3$$constant & 0x3f);
11372 %}
11373
11374 ins_pipe(ialu_reg_reg_shift);
11375 %}
11376
11377 // This pattern is automatically generated from aarch64_ad.m4
11378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11379 instruct SubI_reg_RShift_reg(iRegINoSp dst,
11380 iRegIorL2I src1, iRegIorL2I src2,
11381 immI src3) %{
11382 match(Set dst (SubI src1 (RShiftI src2 src3)));
11383
11384 ins_cost(1.9 * INSN_COST);
11385 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
11386
11387 ins_encode %{
11388 __ subw(as_Register($dst$$reg),
11389 as_Register($src1$$reg),
11390 as_Register($src2$$reg),
11391 Assembler::ASR,
11392 $src3$$constant & 0x1f);
11393 %}
11394
11395 ins_pipe(ialu_reg_reg_shift);
11396 %}
11397
11398 // This pattern is automatically generated from aarch64_ad.m4
11399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11400 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
11401 iRegL src1, iRegL src2,
11402 immI src3) %{
11403 match(Set dst (SubL src1 (RShiftL src2 src3)));
11404
11405 ins_cost(1.9 * INSN_COST);
11406 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
11407
11408 ins_encode %{
11409 __ sub(as_Register($dst$$reg),
11410 as_Register($src1$$reg),
11411 as_Register($src2$$reg),
11412 Assembler::ASR,
11413 $src3$$constant & 0x3f);
11414 %}
11415
11416 ins_pipe(ialu_reg_reg_shift);
11417 %}
11418
11419 // This pattern is automatically generated from aarch64_ad.m4
11420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11421 instruct SubI_reg_LShift_reg(iRegINoSp dst,
11422 iRegIorL2I src1, iRegIorL2I src2,
11423 immI src3) %{
11424 match(Set dst (SubI src1 (LShiftI src2 src3)));
11425
11426 ins_cost(1.9 * INSN_COST);
11427 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
11428
11429 ins_encode %{
11430 __ subw(as_Register($dst$$reg),
11431 as_Register($src1$$reg),
11432 as_Register($src2$$reg),
11433 Assembler::LSL,
11434 $src3$$constant & 0x1f);
11435 %}
11436
11437 ins_pipe(ialu_reg_reg_shift);
11438 %}
11439
11440 // This pattern is automatically generated from aarch64_ad.m4
11441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11442 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
11443 iRegL src1, iRegL src2,
11444 immI src3) %{
11445 match(Set dst (SubL src1 (LShiftL src2 src3)));
11446
11447 ins_cost(1.9 * INSN_COST);
11448 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
11449
11450 ins_encode %{
11451 __ sub(as_Register($dst$$reg),
11452 as_Register($src1$$reg),
11453 as_Register($src2$$reg),
11454 Assembler::LSL,
11455 $src3$$constant & 0x3f);
11456 %}
11457
11458 ins_pipe(ialu_reg_reg_shift);
11459 %}
11460
11461 // This pattern is automatically generated from aarch64_ad.m4
11462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11463
11464 // Shift Left followed by Shift Right.
11465 // This idiom is used by the compiler for the i2b bytecode etc.
11466 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11467 %{
11468 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
11469 ins_cost(INSN_COST * 2);
11470 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11471 ins_encode %{
11472 int lshift = $lshift_count$$constant & 63;
11473 int rshift = $rshift_count$$constant & 63;
11474 int s = 63 - lshift;
11475 int r = (rshift - lshift) & 63;
11476 __ sbfm(as_Register($dst$$reg),
11477 as_Register($src$$reg),
11478 r, s);
11479 %}
11480
11481 ins_pipe(ialu_reg_shift);
11482 %}
11483
11484 // This pattern is automatically generated from aarch64_ad.m4
11485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11486
11487 // Shift Left followed by Shift Right.
11488 // This idiom is used by the compiler for the i2b bytecode etc.
11489 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11490 %{
11491 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
11492 ins_cost(INSN_COST * 2);
11493 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11494 ins_encode %{
11495 int lshift = $lshift_count$$constant & 31;
11496 int rshift = $rshift_count$$constant & 31;
11497 int s = 31 - lshift;
11498 int r = (rshift - lshift) & 31;
11499 __ sbfmw(as_Register($dst$$reg),
11500 as_Register($src$$reg),
11501 r, s);
11502 %}
11503
11504 ins_pipe(ialu_reg_shift);
11505 %}
11506
11507 // This pattern is automatically generated from aarch64_ad.m4
11508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11509
11510 // Shift Left followed by Shift Right.
11511 // This idiom is used by the compiler for the i2b bytecode etc.
11512 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
11513 %{
11514 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
11515 ins_cost(INSN_COST * 2);
11516 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
11517 ins_encode %{
11518 int lshift = $lshift_count$$constant & 63;
11519 int rshift = $rshift_count$$constant & 63;
11520 int s = 63 - lshift;
11521 int r = (rshift - lshift) & 63;
11522 __ ubfm(as_Register($dst$$reg),
11523 as_Register($src$$reg),
11524 r, s);
11525 %}
11526
11527 ins_pipe(ialu_reg_shift);
11528 %}
11529
11530 // This pattern is automatically generated from aarch64_ad.m4
11531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11532
11533 // Shift Left followed by Shift Right.
11534 // This idiom is used by the compiler for the i2b bytecode etc.
11535 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
11536 %{
11537 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
11538 ins_cost(INSN_COST * 2);
11539 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
11540 ins_encode %{
11541 int lshift = $lshift_count$$constant & 31;
11542 int rshift = $rshift_count$$constant & 31;
11543 int s = 31 - lshift;
11544 int r = (rshift - lshift) & 31;
11545 __ ubfmw(as_Register($dst$$reg),
11546 as_Register($src$$reg),
11547 r, s);
11548 %}
11549
11550 ins_pipe(ialu_reg_shift);
11551 %}
11552
11553 // Bitfield extract with shift & mask
11554
11555 // This pattern is automatically generated from aarch64_ad.m4
11556 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11557 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11558 %{
11559 match(Set dst (AndI (URShiftI src rshift) mask));
11560 // Make sure we are not going to exceed what ubfxw can do.
11561 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11562
11563 ins_cost(INSN_COST);
11564 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
11565 ins_encode %{
11566 int rshift = $rshift$$constant & 31;
11567 intptr_t mask = $mask$$constant;
11568 int width = exact_log2(mask+1);
11569 __ ubfxw(as_Register($dst$$reg),
11570 as_Register($src$$reg), rshift, width);
11571 %}
11572 ins_pipe(ialu_reg_shift);
11573 %}
11574
11575 // This pattern is automatically generated from aarch64_ad.m4
11576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11577 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
11578 %{
11579 match(Set dst (AndL (URShiftL src rshift) mask));
11580 // Make sure we are not going to exceed what ubfx can do.
11581 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
11582
11583 ins_cost(INSN_COST);
11584 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11585 ins_encode %{
11586 int rshift = $rshift$$constant & 63;
11587 intptr_t mask = $mask$$constant;
11588 int width = exact_log2_long(mask+1);
11589 __ ubfx(as_Register($dst$$reg),
11590 as_Register($src$$reg), rshift, width);
11591 %}
11592 ins_pipe(ialu_reg_shift);
11593 %}
11594
11595
11596 // This pattern is automatically generated from aarch64_ad.m4
11597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11598
11599 // We can use ubfx when extending an And with a mask when we know mask
11600 // is positive. We know that because immI_bitmask guarantees it.
11601 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
11602 %{
11603 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
11604 // Make sure we are not going to exceed what ubfxw can do.
11605 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
11606
11607 ins_cost(INSN_COST * 2);
11608 format %{ "ubfx $dst, $src, $rshift, $mask" %}
11609 ins_encode %{
11610 int rshift = $rshift$$constant & 31;
11611 intptr_t mask = $mask$$constant;
11612 int width = exact_log2(mask+1);
11613 __ ubfx(as_Register($dst$$reg),
11614 as_Register($src$$reg), rshift, width);
11615 %}
11616 ins_pipe(ialu_reg_shift);
11617 %}
11618
11619
11620 // This pattern is automatically generated from aarch64_ad.m4
11621 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11622
11623 // We can use ubfiz when masking by a positive number and then left shifting the result.
11624 // We know that the mask is positive because immI_bitmask guarantees it.
11625 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11626 %{
11627 match(Set dst (LShiftI (AndI src mask) lshift));
11628 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
11629
11630 ins_cost(INSN_COST);
11631 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11632 ins_encode %{
11633 int lshift = $lshift$$constant & 31;
11634 intptr_t mask = $mask$$constant;
11635 int width = exact_log2(mask+1);
11636 __ ubfizw(as_Register($dst$$reg),
11637 as_Register($src$$reg), lshift, width);
11638 %}
11639 ins_pipe(ialu_reg_shift);
11640 %}
11641
11642 // This pattern is automatically generated from aarch64_ad.m4
11643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11644
11645 // We can use ubfiz when masking by a positive number and then left shifting the result.
11646 // We know that the mask is positive because immL_bitmask guarantees it.
11647 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
11648 %{
11649 match(Set dst (LShiftL (AndL src mask) lshift));
11650 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11651
11652 ins_cost(INSN_COST);
11653 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11654 ins_encode %{
11655 int lshift = $lshift$$constant & 63;
11656 intptr_t mask = $mask$$constant;
11657 int width = exact_log2_long(mask+1);
11658 __ ubfiz(as_Register($dst$$reg),
11659 as_Register($src$$reg), lshift, width);
11660 %}
11661 ins_pipe(ialu_reg_shift);
11662 %}
11663
11664 // This pattern is automatically generated from aarch64_ad.m4
11665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11666
11667 // We can use ubfiz when masking by a positive number and then left shifting the result.
11668 // We know that the mask is positive because immI_bitmask guarantees it.
11669 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11670 %{
11671 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
11672 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
11673
11674 ins_cost(INSN_COST);
11675 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
11676 ins_encode %{
11677 int lshift = $lshift$$constant & 31;
11678 intptr_t mask = $mask$$constant;
11679 int width = exact_log2(mask+1);
11680 __ ubfizw(as_Register($dst$$reg),
11681 as_Register($src$$reg), lshift, width);
11682 %}
11683 ins_pipe(ialu_reg_shift);
11684 %}
11685
11686 // This pattern is automatically generated from aarch64_ad.m4
11687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11688
11689 // We can use ubfiz when masking by a positive number and then left shifting the result.
11690 // We know that the mask is positive because immL_bitmask guarantees it.
11691 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11692 %{
11693 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
11694 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
11695
11696 ins_cost(INSN_COST);
11697 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11698 ins_encode %{
11699 int lshift = $lshift$$constant & 63;
11700 intptr_t mask = $mask$$constant;
11701 int width = exact_log2_long(mask+1);
11702 __ ubfiz(as_Register($dst$$reg),
11703 as_Register($src$$reg), lshift, width);
11704 %}
11705 ins_pipe(ialu_reg_shift);
11706 %}
11707
11708
11709 // This pattern is automatically generated from aarch64_ad.m4
11710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11711
11712 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
11713 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
11714 %{
11715 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
11716 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
11717
11718 ins_cost(INSN_COST);
11719 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11720 ins_encode %{
11721 int lshift = $lshift$$constant & 63;
11722 intptr_t mask = $mask$$constant;
11723 int width = exact_log2(mask+1);
11724 __ ubfiz(as_Register($dst$$reg),
11725 as_Register($src$$reg), lshift, width);
11726 %}
11727 ins_pipe(ialu_reg_shift);
11728 %}
11729
11730 // This pattern is automatically generated from aarch64_ad.m4
11731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11732
11733 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
11734 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
11735 %{
11736 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
11737 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
11738
11739 ins_cost(INSN_COST);
11740 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
11741 ins_encode %{
11742 int lshift = $lshift$$constant & 31;
11743 intptr_t mask = $mask$$constant;
11744 int width = exact_log2(mask+1);
11745 __ ubfiz(as_Register($dst$$reg),
11746 as_Register($src$$reg), lshift, width);
11747 %}
11748 ins_pipe(ialu_reg_shift);
11749 %}
11750
11751 // This pattern is automatically generated from aarch64_ad.m4
11752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11753
11754 // Can skip int2long conversions after AND with small bitmask
11755 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
11756 %{
11757 match(Set dst (ConvI2L (AndI src msk)));
11758 ins_cost(INSN_COST);
11759 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
11760 ins_encode %{
11761 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
11762 %}
11763 ins_pipe(ialu_reg_shift);
11764 %}
11765
11766
11767 // Rotations
11768
11769 // This pattern is automatically generated from aarch64_ad.m4
11770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11771 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11772 %{
11773 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11774 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11775
11776 ins_cost(INSN_COST);
11777 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11778
11779 ins_encode %{
11780 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11781 $rshift$$constant & 63);
11782 %}
11783 ins_pipe(ialu_reg_reg_extr);
11784 %}
11785
11786
11787 // This pattern is automatically generated from aarch64_ad.m4
11788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11789 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11790 %{
11791 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11792 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11793
11794 ins_cost(INSN_COST);
11795 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11796
11797 ins_encode %{
11798 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11799 $rshift$$constant & 31);
11800 %}
11801 ins_pipe(ialu_reg_reg_extr);
11802 %}
11803
11804
11805 // This pattern is automatically generated from aarch64_ad.m4
11806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11807 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
11808 %{
11809 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
11810 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
11811
11812 ins_cost(INSN_COST);
11813 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11814
11815 ins_encode %{
11816 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11817 $rshift$$constant & 63);
11818 %}
11819 ins_pipe(ialu_reg_reg_extr);
11820 %}
11821
11822
11823 // This pattern is automatically generated from aarch64_ad.m4
11824 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11825 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
11826 %{
11827 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
11828 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
11829
11830 ins_cost(INSN_COST);
11831 format %{ "extr $dst, $src1, $src2, #$rshift" %}
11832
11833 ins_encode %{
11834 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
11835 $rshift$$constant & 31);
11836 %}
11837 ins_pipe(ialu_reg_reg_extr);
11838 %}
11839
11840 // This pattern is automatically generated from aarch64_ad.m4
11841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11842 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
11843 %{
11844 match(Set dst (RotateRight src shift));
11845
11846 ins_cost(INSN_COST);
11847 format %{ "ror $dst, $src, $shift" %}
11848
11849 ins_encode %{
11850 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11851 $shift$$constant & 0x1f);
11852 %}
11853 ins_pipe(ialu_reg_reg_vshift);
11854 %}
11855
11856 // This pattern is automatically generated from aarch64_ad.m4
11857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11858 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
11859 %{
11860 match(Set dst (RotateRight src shift));
11861
11862 ins_cost(INSN_COST);
11863 format %{ "ror $dst, $src, $shift" %}
11864
11865 ins_encode %{
11866 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
11867 $shift$$constant & 0x3f);
11868 %}
11869 ins_pipe(ialu_reg_reg_vshift);
11870 %}
11871
11872 // This pattern is automatically generated from aarch64_ad.m4
11873 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11874 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11875 %{
11876 match(Set dst (RotateRight src shift));
11877
11878 ins_cost(INSN_COST);
11879 format %{ "ror $dst, $src, $shift" %}
11880
11881 ins_encode %{
11882 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11883 %}
11884 ins_pipe(ialu_reg_reg_vshift);
11885 %}
11886
11887 // This pattern is automatically generated from aarch64_ad.m4
11888 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11889 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11890 %{
11891 match(Set dst (RotateRight src shift));
11892
11893 ins_cost(INSN_COST);
11894 format %{ "ror $dst, $src, $shift" %}
11895
11896 ins_encode %{
11897 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
11898 %}
11899 ins_pipe(ialu_reg_reg_vshift);
11900 %}
11901
11902 // This pattern is automatically generated from aarch64_ad.m4
11903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11904 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
11905 %{
11906 match(Set dst (RotateLeft src shift));
11907
11908 ins_cost(INSN_COST);
11909 format %{ "rol $dst, $src, $shift" %}
11910
11911 ins_encode %{
11912 __ subw(rscratch1, zr, as_Register($shift$$reg));
11913 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11914 %}
11915 ins_pipe(ialu_reg_reg_vshift);
11916 %}
11917
11918 // This pattern is automatically generated from aarch64_ad.m4
11919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11920 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
11921 %{
11922 match(Set dst (RotateLeft src shift));
11923
11924 ins_cost(INSN_COST);
11925 format %{ "rol $dst, $src, $shift" %}
11926
11927 ins_encode %{
11928 __ subw(rscratch1, zr, as_Register($shift$$reg));
11929 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
11930 %}
11931 ins_pipe(ialu_reg_reg_vshift);
11932 %}
11933
11934
11935 // Add/subtract (extended)
11936
11937 // This pattern is automatically generated from aarch64_ad.m4
11938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11939 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11940 %{
11941 match(Set dst (AddL src1 (ConvI2L src2)));
11942 ins_cost(INSN_COST);
11943 format %{ "add $dst, $src1, $src2, sxtw" %}
11944
11945 ins_encode %{
11946 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11947 as_Register($src2$$reg), ext::sxtw);
11948 %}
11949 ins_pipe(ialu_reg_reg);
11950 %}
11951
11952 // This pattern is automatically generated from aarch64_ad.m4
11953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11954 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
11955 %{
11956 match(Set dst (SubL src1 (ConvI2L src2)));
11957 ins_cost(INSN_COST);
11958 format %{ "sub $dst, $src1, $src2, sxtw" %}
11959
11960 ins_encode %{
11961 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
11962 as_Register($src2$$reg), ext::sxtw);
11963 %}
11964 ins_pipe(ialu_reg_reg);
11965 %}
11966
11967 // This pattern is automatically generated from aarch64_ad.m4
11968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11969 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
11970 %{
11971 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11972 ins_cost(INSN_COST);
11973 format %{ "add $dst, $src1, $src2, sxth" %}
11974
11975 ins_encode %{
11976 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11977 as_Register($src2$$reg), ext::sxth);
11978 %}
11979 ins_pipe(ialu_reg_reg);
11980 %}
11981
11982 // This pattern is automatically generated from aarch64_ad.m4
11983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11984 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
11985 %{
11986 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
11987 ins_cost(INSN_COST);
11988 format %{ "add $dst, $src1, $src2, sxtb" %}
11989
11990 ins_encode %{
11991 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
11992 as_Register($src2$$reg), ext::sxtb);
11993 %}
11994 ins_pipe(ialu_reg_reg);
11995 %}
11996
11997 // This pattern is automatically generated from aarch64_ad.m4
11998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11999 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12000 %{
12001 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12002 ins_cost(INSN_COST);
12003 format %{ "add $dst, $src1, $src2, uxtb" %}
12004
12005 ins_encode %{
12006 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12007 as_Register($src2$$reg), ext::uxtb);
12008 %}
12009 ins_pipe(ialu_reg_reg);
12010 %}
12011
12012 // This pattern is automatically generated from aarch64_ad.m4
12013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12014 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12015 %{
12016 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12017 ins_cost(INSN_COST);
12018 format %{ "add $dst, $src1, $src2, sxth" %}
12019
12020 ins_encode %{
12021 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12022 as_Register($src2$$reg), ext::sxth);
12023 %}
12024 ins_pipe(ialu_reg_reg);
12025 %}
12026
12027 // This pattern is automatically generated from aarch64_ad.m4
12028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12029 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12030 %{
12031 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12032 ins_cost(INSN_COST);
12033 format %{ "add $dst, $src1, $src2, sxtw" %}
12034
12035 ins_encode %{
12036 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12037 as_Register($src2$$reg), ext::sxtw);
12038 %}
12039 ins_pipe(ialu_reg_reg);
12040 %}
12041
12042 // This pattern is automatically generated from aarch64_ad.m4
12043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12044 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12045 %{
12046 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12047 ins_cost(INSN_COST);
12048 format %{ "add $dst, $src1, $src2, sxtb" %}
12049
12050 ins_encode %{
12051 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12052 as_Register($src2$$reg), ext::sxtb);
12053 %}
12054 ins_pipe(ialu_reg_reg);
12055 %}
12056
12057 // This pattern is automatically generated from aarch64_ad.m4
12058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12059 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12060 %{
12061 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12062 ins_cost(INSN_COST);
12063 format %{ "add $dst, $src1, $src2, uxtb" %}
12064
12065 ins_encode %{
12066 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12067 as_Register($src2$$reg), ext::uxtb);
12068 %}
12069 ins_pipe(ialu_reg_reg);
12070 %}
12071
12072 // This pattern is automatically generated from aarch64_ad.m4
12073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12074 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12075 %{
12076 match(Set dst (AddI src1 (AndI src2 mask)));
12077 ins_cost(INSN_COST);
12078 format %{ "addw $dst, $src1, $src2, uxtb" %}
12079
12080 ins_encode %{
12081 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12082 as_Register($src2$$reg), ext::uxtb);
12083 %}
12084 ins_pipe(ialu_reg_reg);
12085 %}
12086
12087 // This pattern is automatically generated from aarch64_ad.m4
12088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12089 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12090 %{
12091 match(Set dst (AddI src1 (AndI src2 mask)));
12092 ins_cost(INSN_COST);
12093 format %{ "addw $dst, $src1, $src2, uxth" %}
12094
12095 ins_encode %{
12096 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12097 as_Register($src2$$reg), ext::uxth);
12098 %}
12099 ins_pipe(ialu_reg_reg);
12100 %}
12101
12102 // This pattern is automatically generated from aarch64_ad.m4
12103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12104 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12105 %{
12106 match(Set dst (AddL src1 (AndL src2 mask)));
12107 ins_cost(INSN_COST);
12108 format %{ "add $dst, $src1, $src2, uxtb" %}
12109
12110 ins_encode %{
12111 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12112 as_Register($src2$$reg), ext::uxtb);
12113 %}
12114 ins_pipe(ialu_reg_reg);
12115 %}
12116
12117 // This pattern is automatically generated from aarch64_ad.m4
12118 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12119 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12120 %{
12121 match(Set dst (AddL src1 (AndL src2 mask)));
12122 ins_cost(INSN_COST);
12123 format %{ "add $dst, $src1, $src2, uxth" %}
12124
12125 ins_encode %{
12126 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12127 as_Register($src2$$reg), ext::uxth);
12128 %}
12129 ins_pipe(ialu_reg_reg);
12130 %}
12131
12132 // This pattern is automatically generated from aarch64_ad.m4
12133 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12134 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12135 %{
12136 match(Set dst (AddL src1 (AndL src2 mask)));
12137 ins_cost(INSN_COST);
12138 format %{ "add $dst, $src1, $src2, uxtw" %}
12139
12140 ins_encode %{
12141 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12142 as_Register($src2$$reg), ext::uxtw);
12143 %}
12144 ins_pipe(ialu_reg_reg);
12145 %}
12146
12147 // This pattern is automatically generated from aarch64_ad.m4
12148 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12149 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12150 %{
12151 match(Set dst (SubI src1 (AndI src2 mask)));
12152 ins_cost(INSN_COST);
12153 format %{ "subw $dst, $src1, $src2, uxtb" %}
12154
12155 ins_encode %{
12156 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12157 as_Register($src2$$reg), ext::uxtb);
12158 %}
12159 ins_pipe(ialu_reg_reg);
12160 %}
12161
12162 // This pattern is automatically generated from aarch64_ad.m4
12163 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12164 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12165 %{
12166 match(Set dst (SubI src1 (AndI src2 mask)));
12167 ins_cost(INSN_COST);
12168 format %{ "subw $dst, $src1, $src2, uxth" %}
12169
12170 ins_encode %{
12171 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12172 as_Register($src2$$reg), ext::uxth);
12173 %}
12174 ins_pipe(ialu_reg_reg);
12175 %}
12176
12177 // This pattern is automatically generated from aarch64_ad.m4
12178 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12179 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12180 %{
12181 match(Set dst (SubL src1 (AndL src2 mask)));
12182 ins_cost(INSN_COST);
12183 format %{ "sub $dst, $src1, $src2, uxtb" %}
12184
12185 ins_encode %{
12186 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12187 as_Register($src2$$reg), ext::uxtb);
12188 %}
12189 ins_pipe(ialu_reg_reg);
12190 %}
12191
12192 // This pattern is automatically generated from aarch64_ad.m4
12193 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12194 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12195 %{
12196 match(Set dst (SubL src1 (AndL src2 mask)));
12197 ins_cost(INSN_COST);
12198 format %{ "sub $dst, $src1, $src2, uxth" %}
12199
12200 ins_encode %{
12201 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12202 as_Register($src2$$reg), ext::uxth);
12203 %}
12204 ins_pipe(ialu_reg_reg);
12205 %}
12206
12207 // This pattern is automatically generated from aarch64_ad.m4
12208 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12209 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12210 %{
12211 match(Set dst (SubL src1 (AndL src2 mask)));
12212 ins_cost(INSN_COST);
12213 format %{ "sub $dst, $src1, $src2, uxtw" %}
12214
12215 ins_encode %{
12216 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12217 as_Register($src2$$reg), ext::uxtw);
12218 %}
12219 ins_pipe(ialu_reg_reg);
12220 %}
12221
12222
12223 // This pattern is automatically generated from aarch64_ad.m4
12224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12225 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12226 %{
12227 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12228 ins_cost(1.9 * INSN_COST);
12229 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12230
12231 ins_encode %{
12232 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12233 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12234 %}
12235 ins_pipe(ialu_reg_reg_shift);
12236 %}
12237
12238 // This pattern is automatically generated from aarch64_ad.m4
12239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12240 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12241 %{
12242 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12243 ins_cost(1.9 * INSN_COST);
12244 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12245
12246 ins_encode %{
12247 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12248 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12249 %}
12250 ins_pipe(ialu_reg_reg_shift);
12251 %}
12252
12253 // This pattern is automatically generated from aarch64_ad.m4
12254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12255 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12256 %{
12257 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12258 ins_cost(1.9 * INSN_COST);
12259 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12260
12261 ins_encode %{
12262 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12263 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12264 %}
12265 ins_pipe(ialu_reg_reg_shift);
12266 %}
12267
12268 // This pattern is automatically generated from aarch64_ad.m4
12269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12270 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12271 %{
12272 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12273 ins_cost(1.9 * INSN_COST);
12274 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12275
12276 ins_encode %{
12277 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12278 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12279 %}
12280 ins_pipe(ialu_reg_reg_shift);
12281 %}
12282
12283 // This pattern is automatically generated from aarch64_ad.m4
12284 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12285 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12286 %{
12287 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12288 ins_cost(1.9 * INSN_COST);
12289 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12290
12291 ins_encode %{
12292 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12293 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12294 %}
12295 ins_pipe(ialu_reg_reg_shift);
12296 %}
12297
12298 // This pattern is automatically generated from aarch64_ad.m4
12299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12300 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12301 %{
12302 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12303 ins_cost(1.9 * INSN_COST);
12304 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12305
12306 ins_encode %{
12307 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12308 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12309 %}
12310 ins_pipe(ialu_reg_reg_shift);
12311 %}
12312
12313 // This pattern is automatically generated from aarch64_ad.m4
12314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12315 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12316 %{
12317 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12318 ins_cost(1.9 * INSN_COST);
12319 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12320
12321 ins_encode %{
12322 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12323 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12324 %}
12325 ins_pipe(ialu_reg_reg_shift);
12326 %}
12327
12328 // This pattern is automatically generated from aarch64_ad.m4
12329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12330 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12331 %{
12332 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12333 ins_cost(1.9 * INSN_COST);
12334 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
12335
12336 ins_encode %{
12337 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12338 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12339 %}
12340 ins_pipe(ialu_reg_reg_shift);
12341 %}
12342
12343 // This pattern is automatically generated from aarch64_ad.m4
12344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12345 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12346 %{
12347 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12348 ins_cost(1.9 * INSN_COST);
12349 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
12350
12351 ins_encode %{
12352 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12353 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12354 %}
12355 ins_pipe(ialu_reg_reg_shift);
12356 %}
12357
12358 // This pattern is automatically generated from aarch64_ad.m4
12359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12360 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
12361 %{
12362 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12363 ins_cost(1.9 * INSN_COST);
12364 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
12365
12366 ins_encode %{
12367 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12368 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12369 %}
12370 ins_pipe(ialu_reg_reg_shift);
12371 %}
12372
12373 // This pattern is automatically generated from aarch64_ad.m4
12374 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12375 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12376 %{
12377 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
12378 ins_cost(1.9 * INSN_COST);
12379 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
12380
12381 ins_encode %{
12382 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12383 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12384 %}
12385 ins_pipe(ialu_reg_reg_shift);
12386 %}
12387
12388 // This pattern is automatically generated from aarch64_ad.m4
12389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12390 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
12391 %{
12392 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
12393 ins_cost(1.9 * INSN_COST);
12394 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
12395
12396 ins_encode %{
12397 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12398 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
12399 %}
12400 ins_pipe(ialu_reg_reg_shift);
12401 %}
12402
12403 // This pattern is automatically generated from aarch64_ad.m4
12404 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12405 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12406 %{
12407 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12408 ins_cost(1.9 * INSN_COST);
12409 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
12410
12411 ins_encode %{
12412 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12413 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12414 %}
12415 ins_pipe(ialu_reg_reg_shift);
12416 %}
12417
12418 // This pattern is automatically generated from aarch64_ad.m4
12419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12420 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12421 %{
12422 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12423 ins_cost(1.9 * INSN_COST);
12424 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
12425
12426 ins_encode %{
12427 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12428 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12429 %}
12430 ins_pipe(ialu_reg_reg_shift);
12431 %}
12432
12433 // This pattern is automatically generated from aarch64_ad.m4
12434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12435 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12436 %{
12437 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
12438 ins_cost(1.9 * INSN_COST);
12439 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
12440
12441 ins_encode %{
12442 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12443 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12444 %}
12445 ins_pipe(ialu_reg_reg_shift);
12446 %}
12447
12448 // This pattern is automatically generated from aarch64_ad.m4
12449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12450 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
12451 %{
12452 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12453 ins_cost(1.9 * INSN_COST);
12454 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
12455
12456 ins_encode %{
12457 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12458 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12459 %}
12460 ins_pipe(ialu_reg_reg_shift);
12461 %}
12462
12463 // This pattern is automatically generated from aarch64_ad.m4
12464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12465 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
12466 %{
12467 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12468 ins_cost(1.9 * INSN_COST);
12469 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
12470
12471 ins_encode %{
12472 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12473 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12474 %}
12475 ins_pipe(ialu_reg_reg_shift);
12476 %}
12477
12478 // This pattern is automatically generated from aarch64_ad.m4
12479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12480 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
12481 %{
12482 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
12483 ins_cost(1.9 * INSN_COST);
12484 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
12485
12486 ins_encode %{
12487 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12488 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
12489 %}
12490 ins_pipe(ialu_reg_reg_shift);
12491 %}
12492
12493 // This pattern is automatically generated from aarch64_ad.m4
12494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12495 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12496 %{
12497 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12498 ins_cost(1.9 * INSN_COST);
12499 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
12500
12501 ins_encode %{
12502 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12503 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12504 %}
12505 ins_pipe(ialu_reg_reg_shift);
12506 %}
12507
12508 // This pattern is automatically generated from aarch64_ad.m4
12509 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12510 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12511 %{
12512 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
12513 ins_cost(1.9 * INSN_COST);
12514 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
12515
12516 ins_encode %{
12517 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12518 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12519 %}
12520 ins_pipe(ialu_reg_reg_shift);
12521 %}
12522
12523 // This pattern is automatically generated from aarch64_ad.m4
12524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12525 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
12526 %{
12527 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12528 ins_cost(1.9 * INSN_COST);
12529 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
12530
12531 ins_encode %{
12532 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12533 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
12534 %}
12535 ins_pipe(ialu_reg_reg_shift);
12536 %}
12537
12538 // This pattern is automatically generated from aarch64_ad.m4
12539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12540 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
12541 %{
12542 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
12543 ins_cost(1.9 * INSN_COST);
12544 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
12545
12546 ins_encode %{
12547 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12548 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
12549 %}
12550 ins_pipe(ialu_reg_reg_shift);
12551 %}
12552
12553 // This pattern is automatically generated from aarch64_ad.m4
12554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12555 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12556 %{
12557 effect(DEF dst, USE src1, USE src2, USE cr);
12558 ins_cost(INSN_COST * 2);
12559 format %{ "cselw $dst, $src1, $src2 lt\t" %}
12560
12561 ins_encode %{
12562 __ cselw($dst$$Register,
12563 $src1$$Register,
12564 $src2$$Register,
12565 Assembler::LT);
12566 %}
12567 ins_pipe(icond_reg_reg);
12568 %}
12569
12570 // This pattern is automatically generated from aarch64_ad.m4
12571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12572 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
12573 %{
12574 effect(DEF dst, USE src1, USE src2, USE cr);
12575 ins_cost(INSN_COST * 2);
12576 format %{ "cselw $dst, $src1, $src2 gt\t" %}
12577
12578 ins_encode %{
12579 __ cselw($dst$$Register,
12580 $src1$$Register,
12581 $src2$$Register,
12582 Assembler::GT);
12583 %}
12584 ins_pipe(icond_reg_reg);
12585 %}
12586
12587 // This pattern is automatically generated from aarch64_ad.m4
12588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12589 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12590 %{
12591 effect(DEF dst, USE src1, USE cr);
12592 ins_cost(INSN_COST * 2);
12593 format %{ "cselw $dst, $src1, zr lt\t" %}
12594
12595 ins_encode %{
12596 __ cselw($dst$$Register,
12597 $src1$$Register,
12598 zr,
12599 Assembler::LT);
12600 %}
12601 ins_pipe(icond_reg);
12602 %}
12603
12604 // This pattern is automatically generated from aarch64_ad.m4
12605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12606 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12607 %{
12608 effect(DEF dst, USE src1, USE cr);
12609 ins_cost(INSN_COST * 2);
12610 format %{ "cselw $dst, $src1, zr gt\t" %}
12611
12612 ins_encode %{
12613 __ cselw($dst$$Register,
12614 $src1$$Register,
12615 zr,
12616 Assembler::GT);
12617 %}
12618 ins_pipe(icond_reg);
12619 %}
12620
12621 // This pattern is automatically generated from aarch64_ad.m4
12622 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12623 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12624 %{
12625 effect(DEF dst, USE src1, USE cr);
12626 ins_cost(INSN_COST * 2);
12627 format %{ "csincw $dst, $src1, zr le\t" %}
12628
12629 ins_encode %{
12630 __ csincw($dst$$Register,
12631 $src1$$Register,
12632 zr,
12633 Assembler::LE);
12634 %}
12635 ins_pipe(icond_reg);
12636 %}
12637
12638 // This pattern is automatically generated from aarch64_ad.m4
12639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12640 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12641 %{
12642 effect(DEF dst, USE src1, USE cr);
12643 ins_cost(INSN_COST * 2);
12644 format %{ "csincw $dst, $src1, zr gt\t" %}
12645
12646 ins_encode %{
12647 __ csincw($dst$$Register,
12648 $src1$$Register,
12649 zr,
12650 Assembler::GT);
12651 %}
12652 ins_pipe(icond_reg);
12653 %}
12654
12655 // This pattern is automatically generated from aarch64_ad.m4
12656 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12657 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12658 %{
12659 effect(DEF dst, USE src1, USE cr);
12660 ins_cost(INSN_COST * 2);
12661 format %{ "csinvw $dst, $src1, zr lt\t" %}
12662
12663 ins_encode %{
12664 __ csinvw($dst$$Register,
12665 $src1$$Register,
12666 zr,
12667 Assembler::LT);
12668 %}
12669 ins_pipe(icond_reg);
12670 %}
12671
12672 // This pattern is automatically generated from aarch64_ad.m4
12673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12674 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
12675 %{
12676 effect(DEF dst, USE src1, USE cr);
12677 ins_cost(INSN_COST * 2);
12678 format %{ "csinvw $dst, $src1, zr ge\t" %}
12679
12680 ins_encode %{
12681 __ csinvw($dst$$Register,
12682 $src1$$Register,
12683 zr,
12684 Assembler::GE);
12685 %}
12686 ins_pipe(icond_reg);
12687 %}
12688
12689 // This pattern is automatically generated from aarch64_ad.m4
12690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12691 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12692 %{
12693 match(Set dst (MinI src imm));
12694 ins_cost(INSN_COST * 3);
12695 expand %{
12696 rFlagsReg cr;
12697 compI_reg_imm0(cr, src);
12698 cmovI_reg_imm0_lt(dst, src, cr);
12699 %}
12700 %}
12701
12702 // This pattern is automatically generated from aarch64_ad.m4
12703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12704 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12705 %{
12706 match(Set dst (MinI imm src));
12707 ins_cost(INSN_COST * 3);
12708 expand %{
12709 rFlagsReg cr;
12710 compI_reg_imm0(cr, src);
12711 cmovI_reg_imm0_lt(dst, src, cr);
12712 %}
12713 %}
12714
12715 // This pattern is automatically generated from aarch64_ad.m4
12716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12717 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12718 %{
12719 match(Set dst (MinI src imm));
12720 ins_cost(INSN_COST * 3);
12721 expand %{
12722 rFlagsReg cr;
12723 compI_reg_imm0(cr, src);
12724 cmovI_reg_imm1_le(dst, src, cr);
12725 %}
12726 %}
12727
12728 // This pattern is automatically generated from aarch64_ad.m4
12729 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12730 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12731 %{
12732 match(Set dst (MinI imm src));
12733 ins_cost(INSN_COST * 3);
12734 expand %{
12735 rFlagsReg cr;
12736 compI_reg_imm0(cr, src);
12737 cmovI_reg_imm1_le(dst, src, cr);
12738 %}
12739 %}
12740
12741 // This pattern is automatically generated from aarch64_ad.m4
12742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12743 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12744 %{
12745 match(Set dst (MinI src imm));
12746 ins_cost(INSN_COST * 3);
12747 expand %{
12748 rFlagsReg cr;
12749 compI_reg_imm0(cr, src);
12750 cmovI_reg_immM1_lt(dst, src, cr);
12751 %}
12752 %}
12753
12754 // This pattern is automatically generated from aarch64_ad.m4
12755 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12756 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12757 %{
12758 match(Set dst (MinI imm src));
12759 ins_cost(INSN_COST * 3);
12760 expand %{
12761 rFlagsReg cr;
12762 compI_reg_imm0(cr, src);
12763 cmovI_reg_immM1_lt(dst, src, cr);
12764 %}
12765 %}
12766
12767 // This pattern is automatically generated from aarch64_ad.m4
12768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12769 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
12770 %{
12771 match(Set dst (MaxI src imm));
12772 ins_cost(INSN_COST * 3);
12773 expand %{
12774 rFlagsReg cr;
12775 compI_reg_imm0(cr, src);
12776 cmovI_reg_imm0_gt(dst, src, cr);
12777 %}
12778 %}
12779
12780 // This pattern is automatically generated from aarch64_ad.m4
12781 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12782 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
12783 %{
12784 match(Set dst (MaxI imm src));
12785 ins_cost(INSN_COST * 3);
12786 expand %{
12787 rFlagsReg cr;
12788 compI_reg_imm0(cr, src);
12789 cmovI_reg_imm0_gt(dst, src, cr);
12790 %}
12791 %}
12792
12793 // This pattern is automatically generated from aarch64_ad.m4
12794 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12795 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
12796 %{
12797 match(Set dst (MaxI src imm));
12798 ins_cost(INSN_COST * 3);
12799 expand %{
12800 rFlagsReg cr;
12801 compI_reg_imm0(cr, src);
12802 cmovI_reg_imm1_gt(dst, src, cr);
12803 %}
12804 %}
12805
12806 // This pattern is automatically generated from aarch64_ad.m4
12807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12808 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
12809 %{
12810 match(Set dst (MaxI imm src));
12811 ins_cost(INSN_COST * 3);
12812 expand %{
12813 rFlagsReg cr;
12814 compI_reg_imm0(cr, src);
12815 cmovI_reg_imm1_gt(dst, src, cr);
12816 %}
12817 %}
12818
12819 // This pattern is automatically generated from aarch64_ad.m4
12820 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12821 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
12822 %{
12823 match(Set dst (MaxI src imm));
12824 ins_cost(INSN_COST * 3);
12825 expand %{
12826 rFlagsReg cr;
12827 compI_reg_imm0(cr, src);
12828 cmovI_reg_immM1_ge(dst, src, cr);
12829 %}
12830 %}
12831
12832 // This pattern is automatically generated from aarch64_ad.m4
12833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12834 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
12835 %{
12836 match(Set dst (MaxI imm src));
12837 ins_cost(INSN_COST * 3);
12838 expand %{
12839 rFlagsReg cr;
12840 compI_reg_imm0(cr, src);
12841 cmovI_reg_immM1_ge(dst, src, cr);
12842 %}
12843 %}
12844
12845 // This pattern is automatically generated from aarch64_ad.m4
12846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12847 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
12848 %{
12849 match(Set dst (ReverseI src));
12850 ins_cost(INSN_COST);
12851 format %{ "rbitw $dst, $src" %}
12852 ins_encode %{
12853 __ rbitw($dst$$Register, $src$$Register);
12854 %}
12855 ins_pipe(ialu_reg);
12856 %}
12857
12858 // This pattern is automatically generated from aarch64_ad.m4
12859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12860 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
12861 %{
12862 match(Set dst (ReverseL src));
12863 ins_cost(INSN_COST);
12864 format %{ "rbit $dst, $src" %}
12865 ins_encode %{
12866 __ rbit($dst$$Register, $src$$Register);
12867 %}
12868 ins_pipe(ialu_reg);
12869 %}
12870
12871
12872 // END This section of the file is automatically generated. Do not edit --------------
12873
12874
12875 // ============================================================================
12876 // Floating Point Arithmetic Instructions
12877
12878 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12879 match(Set dst (AddHF src1 src2));
12880 format %{ "faddh $dst, $src1, $src2" %}
12881 ins_encode %{
12882 __ faddh($dst$$FloatRegister,
12883 $src1$$FloatRegister,
12884 $src2$$FloatRegister);
12885 %}
12886 ins_pipe(fp_dop_reg_reg_s);
12887 %}
12888
12889 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12890 match(Set dst (AddF src1 src2));
12891
12892 ins_cost(INSN_COST * 5);
12893 format %{ "fadds $dst, $src1, $src2" %}
12894
12895 ins_encode %{
12896 __ fadds(as_FloatRegister($dst$$reg),
12897 as_FloatRegister($src1$$reg),
12898 as_FloatRegister($src2$$reg));
12899 %}
12900
12901 ins_pipe(fp_dop_reg_reg_s);
12902 %}
12903
12904 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12905 match(Set dst (AddD src1 src2));
12906
12907 ins_cost(INSN_COST * 5);
12908 format %{ "faddd $dst, $src1, $src2" %}
12909
12910 ins_encode %{
12911 __ faddd(as_FloatRegister($dst$$reg),
12912 as_FloatRegister($src1$$reg),
12913 as_FloatRegister($src2$$reg));
12914 %}
12915
12916 ins_pipe(fp_dop_reg_reg_d);
12917 %}
12918
12919 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12920 match(Set dst (SubHF src1 src2));
12921 format %{ "fsubh $dst, $src1, $src2" %}
12922 ins_encode %{
12923 __ fsubh($dst$$FloatRegister,
12924 $src1$$FloatRegister,
12925 $src2$$FloatRegister);
12926 %}
12927 ins_pipe(fp_dop_reg_reg_s);
12928 %}
12929
12930 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12931 match(Set dst (SubF src1 src2));
12932
12933 ins_cost(INSN_COST * 5);
12934 format %{ "fsubs $dst, $src1, $src2" %}
12935
12936 ins_encode %{
12937 __ fsubs(as_FloatRegister($dst$$reg),
12938 as_FloatRegister($src1$$reg),
12939 as_FloatRegister($src2$$reg));
12940 %}
12941
12942 ins_pipe(fp_dop_reg_reg_s);
12943 %}
12944
12945 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12946 match(Set dst (SubD src1 src2));
12947
12948 ins_cost(INSN_COST * 5);
12949 format %{ "fsubd $dst, $src1, $src2" %}
12950
12951 ins_encode %{
12952 __ fsubd(as_FloatRegister($dst$$reg),
12953 as_FloatRegister($src1$$reg),
12954 as_FloatRegister($src2$$reg));
12955 %}
12956
12957 ins_pipe(fp_dop_reg_reg_d);
12958 %}
12959
12960 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12961 match(Set dst (MulHF src1 src2));
12962 format %{ "fmulh $dst, $src1, $src2" %}
12963 ins_encode %{
12964 __ fmulh($dst$$FloatRegister,
12965 $src1$$FloatRegister,
12966 $src2$$FloatRegister);
12967 %}
12968 ins_pipe(fp_dop_reg_reg_s);
12969 %}
12970
12971 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
12972 match(Set dst (MulF src1 src2));
12973
12974 ins_cost(INSN_COST * 6);
12975 format %{ "fmuls $dst, $src1, $src2" %}
12976
12977 ins_encode %{
12978 __ fmuls(as_FloatRegister($dst$$reg),
12979 as_FloatRegister($src1$$reg),
12980 as_FloatRegister($src2$$reg));
12981 %}
12982
12983 ins_pipe(fp_dop_reg_reg_s);
12984 %}
12985
12986 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
12987 match(Set dst (MulD src1 src2));
12988
12989 ins_cost(INSN_COST * 6);
12990 format %{ "fmuld $dst, $src1, $src2" %}
12991
12992 ins_encode %{
12993 __ fmuld(as_FloatRegister($dst$$reg),
12994 as_FloatRegister($src1$$reg),
12995 as_FloatRegister($src2$$reg));
12996 %}
12997
12998 ins_pipe(fp_dop_reg_reg_d);
12999 %}
13000
13001 // src1 * src2 + src3 (half-precision float)
13002 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13003 match(Set dst (FmaHF src3 (Binary src1 src2)));
13004 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13005 ins_encode %{
13006 assert(UseFMA, "Needs FMA instructions support.");
13007 __ fmaddh($dst$$FloatRegister,
13008 $src1$$FloatRegister,
13009 $src2$$FloatRegister,
13010 $src3$$FloatRegister);
13011 %}
13012 ins_pipe(pipe_class_default);
13013 %}
13014
13015 // src1 * src2 + src3
13016 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13017 match(Set dst (FmaF src3 (Binary src1 src2)));
13018
13019 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13020
13021 ins_encode %{
13022 assert(UseFMA, "Needs FMA instructions support.");
13023 __ fmadds(as_FloatRegister($dst$$reg),
13024 as_FloatRegister($src1$$reg),
13025 as_FloatRegister($src2$$reg),
13026 as_FloatRegister($src3$$reg));
13027 %}
13028
13029 ins_pipe(pipe_class_default);
13030 %}
13031
13032 // src1 * src2 + src3
13033 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13034 match(Set dst (FmaD src3 (Binary src1 src2)));
13035
13036 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13037
13038 ins_encode %{
13039 assert(UseFMA, "Needs FMA instructions support.");
13040 __ fmaddd(as_FloatRegister($dst$$reg),
13041 as_FloatRegister($src1$$reg),
13042 as_FloatRegister($src2$$reg),
13043 as_FloatRegister($src3$$reg));
13044 %}
13045
13046 ins_pipe(pipe_class_default);
13047 %}
13048
13049 // src1 * (-src2) + src3
13050 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13051 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13052 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13053
13054 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13055
13056 ins_encode %{
13057 assert(UseFMA, "Needs FMA instructions support.");
13058 __ fmsubs(as_FloatRegister($dst$$reg),
13059 as_FloatRegister($src1$$reg),
13060 as_FloatRegister($src2$$reg),
13061 as_FloatRegister($src3$$reg));
13062 %}
13063
13064 ins_pipe(pipe_class_default);
13065 %}
13066
13067 // src1 * (-src2) + src3
13068 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13069 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13070 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13071
13072 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13073
13074 ins_encode %{
13075 assert(UseFMA, "Needs FMA instructions support.");
13076 __ fmsubd(as_FloatRegister($dst$$reg),
13077 as_FloatRegister($src1$$reg),
13078 as_FloatRegister($src2$$reg),
13079 as_FloatRegister($src3$$reg));
13080 %}
13081
13082 ins_pipe(pipe_class_default);
13083 %}
13084
13085 // src1 * (-src2) - src3
13086 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13087 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13088 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13089
13090 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13091
13092 ins_encode %{
13093 assert(UseFMA, "Needs FMA instructions support.");
13094 __ fnmadds(as_FloatRegister($dst$$reg),
13095 as_FloatRegister($src1$$reg),
13096 as_FloatRegister($src2$$reg),
13097 as_FloatRegister($src3$$reg));
13098 %}
13099
13100 ins_pipe(pipe_class_default);
13101 %}
13102
13103 // src1 * (-src2) - src3
13104 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13105 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13106 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13107
13108 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13109
13110 ins_encode %{
13111 assert(UseFMA, "Needs FMA instructions support.");
13112 __ fnmaddd(as_FloatRegister($dst$$reg),
13113 as_FloatRegister($src1$$reg),
13114 as_FloatRegister($src2$$reg),
13115 as_FloatRegister($src3$$reg));
13116 %}
13117
13118 ins_pipe(pipe_class_default);
13119 %}
13120
13121 // src1 * src2 - src3
13122 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13123 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13124
13125 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13126
13127 ins_encode %{
13128 assert(UseFMA, "Needs FMA instructions support.");
13129 __ fnmsubs(as_FloatRegister($dst$$reg),
13130 as_FloatRegister($src1$$reg),
13131 as_FloatRegister($src2$$reg),
13132 as_FloatRegister($src3$$reg));
13133 %}
13134
13135 ins_pipe(pipe_class_default);
13136 %}
13137
13138 // src1 * src2 - src3
13139 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13140 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13141
13142 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13143
13144 ins_encode %{
13145 assert(UseFMA, "Needs FMA instructions support.");
13146 // n.b. insn name should be fnmsubd
13147 __ fnmsub(as_FloatRegister($dst$$reg),
13148 as_FloatRegister($src1$$reg),
13149 as_FloatRegister($src2$$reg),
13150 as_FloatRegister($src3$$reg));
13151 %}
13152
13153 ins_pipe(pipe_class_default);
13154 %}
13155
13156 // Math.max(HH)H (half-precision float)
13157 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13158 match(Set dst (MaxHF src1 src2));
13159 format %{ "fmaxh $dst, $src1, $src2" %}
13160 ins_encode %{
13161 __ fmaxh($dst$$FloatRegister,
13162 $src1$$FloatRegister,
13163 $src2$$FloatRegister);
13164 %}
13165 ins_pipe(fp_dop_reg_reg_s);
13166 %}
13167
13168 // Math.min(HH)H (half-precision float)
13169 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13170 match(Set dst (MinHF src1 src2));
13171 format %{ "fminh $dst, $src1, $src2" %}
13172 ins_encode %{
13173 __ fminh($dst$$FloatRegister,
13174 $src1$$FloatRegister,
13175 $src2$$FloatRegister);
13176 %}
13177 ins_pipe(fp_dop_reg_reg_s);
13178 %}
13179
13180 // Math.max(FF)F
13181 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13182 match(Set dst (MaxF src1 src2));
13183
13184 format %{ "fmaxs $dst, $src1, $src2" %}
13185 ins_encode %{
13186 __ fmaxs(as_FloatRegister($dst$$reg),
13187 as_FloatRegister($src1$$reg),
13188 as_FloatRegister($src2$$reg));
13189 %}
13190
13191 ins_pipe(fp_dop_reg_reg_s);
13192 %}
13193
13194 // Math.min(FF)F
13195 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13196 match(Set dst (MinF src1 src2));
13197
13198 format %{ "fmins $dst, $src1, $src2" %}
13199 ins_encode %{
13200 __ fmins(as_FloatRegister($dst$$reg),
13201 as_FloatRegister($src1$$reg),
13202 as_FloatRegister($src2$$reg));
13203 %}
13204
13205 ins_pipe(fp_dop_reg_reg_s);
13206 %}
13207
13208 // Math.max(DD)D
13209 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13210 match(Set dst (MaxD src1 src2));
13211
13212 format %{ "fmaxd $dst, $src1, $src2" %}
13213 ins_encode %{
13214 __ fmaxd(as_FloatRegister($dst$$reg),
13215 as_FloatRegister($src1$$reg),
13216 as_FloatRegister($src2$$reg));
13217 %}
13218
13219 ins_pipe(fp_dop_reg_reg_d);
13220 %}
13221
13222 // Math.min(DD)D
13223 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13224 match(Set dst (MinD src1 src2));
13225
13226 format %{ "fmind $dst, $src1, $src2" %}
13227 ins_encode %{
13228 __ fmind(as_FloatRegister($dst$$reg),
13229 as_FloatRegister($src1$$reg),
13230 as_FloatRegister($src2$$reg));
13231 %}
13232
13233 ins_pipe(fp_dop_reg_reg_d);
13234 %}
13235
13236 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13237 match(Set dst (DivHF src1 src2));
13238 format %{ "fdivh $dst, $src1, $src2" %}
13239 ins_encode %{
13240 __ fdivh($dst$$FloatRegister,
13241 $src1$$FloatRegister,
13242 $src2$$FloatRegister);
13243 %}
13244 ins_pipe(fp_div_s);
13245 %}
13246
13247 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13248 match(Set dst (DivF src1 src2));
13249
13250 ins_cost(INSN_COST * 18);
13251 format %{ "fdivs $dst, $src1, $src2" %}
13252
13253 ins_encode %{
13254 __ fdivs(as_FloatRegister($dst$$reg),
13255 as_FloatRegister($src1$$reg),
13256 as_FloatRegister($src2$$reg));
13257 %}
13258
13259 ins_pipe(fp_div_s);
13260 %}
13261
13262 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13263 match(Set dst (DivD src1 src2));
13264
13265 ins_cost(INSN_COST * 32);
13266 format %{ "fdivd $dst, $src1, $src2" %}
13267
13268 ins_encode %{
13269 __ fdivd(as_FloatRegister($dst$$reg),
13270 as_FloatRegister($src1$$reg),
13271 as_FloatRegister($src2$$reg));
13272 %}
13273
13274 ins_pipe(fp_div_d);
13275 %}
13276
13277 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13278 match(Set dst (NegF src));
13279
13280 ins_cost(INSN_COST * 3);
13281 format %{ "fneg $dst, $src" %}
13282
13283 ins_encode %{
13284 __ fnegs(as_FloatRegister($dst$$reg),
13285 as_FloatRegister($src$$reg));
13286 %}
13287
13288 ins_pipe(fp_uop_s);
13289 %}
13290
13291 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13292 match(Set dst (NegD src));
13293
13294 ins_cost(INSN_COST * 3);
13295 format %{ "fnegd $dst, $src" %}
13296
13297 ins_encode %{
13298 __ fnegd(as_FloatRegister($dst$$reg),
13299 as_FloatRegister($src$$reg));
13300 %}
13301
13302 ins_pipe(fp_uop_d);
13303 %}
13304
13305 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13306 %{
13307 match(Set dst (AbsI src));
13308
13309 effect(KILL cr);
13310 ins_cost(INSN_COST * 2);
13311 format %{ "cmpw $src, zr\n\t"
13312 "cnegw $dst, $src, Assembler::LT\t# int abs"
13313 %}
13314
13315 ins_encode %{
13316 __ cmpw(as_Register($src$$reg), zr);
13317 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13318 %}
13319 ins_pipe(pipe_class_default);
13320 %}
13321
13322 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13323 %{
13324 match(Set dst (AbsL src));
13325
13326 effect(KILL cr);
13327 ins_cost(INSN_COST * 2);
13328 format %{ "cmp $src, zr\n\t"
13329 "cneg $dst, $src, Assembler::LT\t# long abs"
13330 %}
13331
13332 ins_encode %{
13333 __ cmp(as_Register($src$$reg), zr);
13334 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13335 %}
13336 ins_pipe(pipe_class_default);
13337 %}
13338
13339 instruct absF_reg(vRegF dst, vRegF src) %{
13340 match(Set dst (AbsF src));
13341
13342 ins_cost(INSN_COST * 3);
13343 format %{ "fabss $dst, $src" %}
13344 ins_encode %{
13345 __ fabss(as_FloatRegister($dst$$reg),
13346 as_FloatRegister($src$$reg));
13347 %}
13348
13349 ins_pipe(fp_uop_s);
13350 %}
13351
13352 instruct absD_reg(vRegD dst, vRegD src) %{
13353 match(Set dst (AbsD src));
13354
13355 ins_cost(INSN_COST * 3);
13356 format %{ "fabsd $dst, $src" %}
13357 ins_encode %{
13358 __ fabsd(as_FloatRegister($dst$$reg),
13359 as_FloatRegister($src$$reg));
13360 %}
13361
13362 ins_pipe(fp_uop_d);
13363 %}
13364
13365 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13366 match(Set dst (AbsF (SubF src1 src2)));
13367
13368 ins_cost(INSN_COST * 3);
13369 format %{ "fabds $dst, $src1, $src2" %}
13370 ins_encode %{
13371 __ fabds(as_FloatRegister($dst$$reg),
13372 as_FloatRegister($src1$$reg),
13373 as_FloatRegister($src2$$reg));
13374 %}
13375
13376 ins_pipe(fp_uop_s);
13377 %}
13378
13379 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13380 match(Set dst (AbsD (SubD src1 src2)));
13381
13382 ins_cost(INSN_COST * 3);
13383 format %{ "fabdd $dst, $src1, $src2" %}
13384 ins_encode %{
13385 __ fabdd(as_FloatRegister($dst$$reg),
13386 as_FloatRegister($src1$$reg),
13387 as_FloatRegister($src2$$reg));
13388 %}
13389
13390 ins_pipe(fp_uop_d);
13391 %}
13392
13393 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13394 match(Set dst (SqrtD src));
13395
13396 ins_cost(INSN_COST * 50);
13397 format %{ "fsqrtd $dst, $src" %}
13398 ins_encode %{
13399 __ fsqrtd(as_FloatRegister($dst$$reg),
13400 as_FloatRegister($src$$reg));
13401 %}
13402
13403 ins_pipe(fp_div_s);
13404 %}
13405
13406 instruct sqrtF_reg(vRegF dst, vRegF src) %{
13407 match(Set dst (SqrtF src));
13408
13409 ins_cost(INSN_COST * 50);
13410 format %{ "fsqrts $dst, $src" %}
13411 ins_encode %{
13412 __ fsqrts(as_FloatRegister($dst$$reg),
13413 as_FloatRegister($src$$reg));
13414 %}
13415
13416 ins_pipe(fp_div_d);
13417 %}
13418
13419 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
13420 match(Set dst (SqrtHF src));
13421 format %{ "fsqrth $dst, $src" %}
13422 ins_encode %{
13423 __ fsqrth($dst$$FloatRegister,
13424 $src$$FloatRegister);
13425 %}
13426 ins_pipe(fp_div_s);
13427 %}
13428
13429 // Math.rint, floor, ceil
13430 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
13431 match(Set dst (RoundDoubleMode src rmode));
13432 format %{ "frint $dst, $src, $rmode" %}
13433 ins_encode %{
13434 switch ($rmode$$constant) {
13435 case RoundDoubleModeNode::rmode_rint:
13436 __ frintnd(as_FloatRegister($dst$$reg),
13437 as_FloatRegister($src$$reg));
13438 break;
13439 case RoundDoubleModeNode::rmode_floor:
13440 __ frintmd(as_FloatRegister($dst$$reg),
13441 as_FloatRegister($src$$reg));
13442 break;
13443 case RoundDoubleModeNode::rmode_ceil:
13444 __ frintpd(as_FloatRegister($dst$$reg),
13445 as_FloatRegister($src$$reg));
13446 break;
13447 }
13448 %}
13449 ins_pipe(fp_uop_d);
13450 %}
13451
13452 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
13453 match(Set dst (CopySignD src1 (Binary src2 zero)));
13454 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
13455 format %{ "CopySignD $dst $src1 $src2" %}
13456 ins_encode %{
13457 FloatRegister dst = as_FloatRegister($dst$$reg),
13458 src1 = as_FloatRegister($src1$$reg),
13459 src2 = as_FloatRegister($src2$$reg),
13460 zero = as_FloatRegister($zero$$reg);
13461 __ fnegd(dst, zero);
13462 __ bsl(dst, __ T8B, src2, src1);
13463 %}
13464 ins_pipe(fp_uop_d);
13465 %}
13466
13467 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13468 match(Set dst (CopySignF src1 src2));
13469 effect(TEMP_DEF dst, USE src1, USE src2);
13470 format %{ "CopySignF $dst $src1 $src2" %}
13471 ins_encode %{
13472 FloatRegister dst = as_FloatRegister($dst$$reg),
13473 src1 = as_FloatRegister($src1$$reg),
13474 src2 = as_FloatRegister($src2$$reg);
13475 __ movi(dst, __ T2S, 0x80, 24);
13476 __ bsl(dst, __ T8B, src2, src1);
13477 %}
13478 ins_pipe(fp_uop_d);
13479 %}
13480
13481 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
13482 match(Set dst (SignumD src (Binary zero one)));
13483 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13484 format %{ "signumD $dst, $src" %}
13485 ins_encode %{
13486 FloatRegister src = as_FloatRegister($src$$reg),
13487 dst = as_FloatRegister($dst$$reg),
13488 zero = as_FloatRegister($zero$$reg),
13489 one = as_FloatRegister($one$$reg);
13490 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13491 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13492 // Bit selection instruction gets bit from "one" for each enabled bit in
13493 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13494 // NaN the whole "src" will be copied because "dst" is zero. For all other
13495 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13496 // from "src", and all other bits are copied from 1.0.
13497 __ bsl(dst, __ T8B, one, src);
13498 %}
13499 ins_pipe(fp_uop_d);
13500 %}
13501
13502 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
13503 match(Set dst (SignumF src (Binary zero one)));
13504 effect(TEMP_DEF dst, USE src, USE zero, USE one);
13505 format %{ "signumF $dst, $src" %}
13506 ins_encode %{
13507 FloatRegister src = as_FloatRegister($src$$reg),
13508 dst = as_FloatRegister($dst$$reg),
13509 zero = as_FloatRegister($zero$$reg),
13510 one = as_FloatRegister($one$$reg);
13511 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
13512 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
13513 // Bit selection instruction gets bit from "one" for each enabled bit in
13514 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
13515 // NaN the whole "src" will be copied because "dst" is zero. For all other
13516 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
13517 // from "src", and all other bits are copied from 1.0.
13518 __ bsl(dst, __ T8B, one, src);
13519 %}
13520 ins_pipe(fp_uop_d);
13521 %}
13522
13523 instruct onspinwait() %{
13524 match(OnSpinWait);
13525 ins_cost(INSN_COST);
13526
13527 format %{ "onspinwait" %}
13528
13529 ins_encode %{
13530 __ spin_wait();
13531 %}
13532 ins_pipe(pipe_class_empty);
13533 %}
13534
13535 // ============================================================================
13536 // Logical Instructions
13537
13538 // Integer Logical Instructions
13539
13540 // And Instructions
13541
13542
13543 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
13544 match(Set dst (AndI src1 src2));
13545
13546 format %{ "andw $dst, $src1, $src2\t# int" %}
13547
13548 ins_cost(INSN_COST);
13549 ins_encode %{
13550 __ andw(as_Register($dst$$reg),
13551 as_Register($src1$$reg),
13552 as_Register($src2$$reg));
13553 %}
13554
13555 ins_pipe(ialu_reg_reg);
13556 %}
13557
13558 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
13559 match(Set dst (AndI src1 src2));
13560
13561 format %{ "andsw $dst, $src1, $src2\t# int" %}
13562
13563 ins_cost(INSN_COST);
13564 ins_encode %{
13565 __ andw(as_Register($dst$$reg),
13566 as_Register($src1$$reg),
13567 (uint64_t)($src2$$constant));
13568 %}
13569
13570 ins_pipe(ialu_reg_imm);
13571 %}
13572
13573 // Or Instructions
13574
13575 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13576 match(Set dst (OrI src1 src2));
13577
13578 format %{ "orrw $dst, $src1, $src2\t# int" %}
13579
13580 ins_cost(INSN_COST);
13581 ins_encode %{
13582 __ orrw(as_Register($dst$$reg),
13583 as_Register($src1$$reg),
13584 as_Register($src2$$reg));
13585 %}
13586
13587 ins_pipe(ialu_reg_reg);
13588 %}
13589
13590 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13591 match(Set dst (OrI src1 src2));
13592
13593 format %{ "orrw $dst, $src1, $src2\t# int" %}
13594
13595 ins_cost(INSN_COST);
13596 ins_encode %{
13597 __ orrw(as_Register($dst$$reg),
13598 as_Register($src1$$reg),
13599 (uint64_t)($src2$$constant));
13600 %}
13601
13602 ins_pipe(ialu_reg_imm);
13603 %}
13604
13605 // Xor Instructions
13606
13607 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
13608 match(Set dst (XorI src1 src2));
13609
13610 format %{ "eorw $dst, $src1, $src2\t# int" %}
13611
13612 ins_cost(INSN_COST);
13613 ins_encode %{
13614 __ eorw(as_Register($dst$$reg),
13615 as_Register($src1$$reg),
13616 as_Register($src2$$reg));
13617 %}
13618
13619 ins_pipe(ialu_reg_reg);
13620 %}
13621
13622 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
13623 match(Set dst (XorI src1 src2));
13624
13625 format %{ "eorw $dst, $src1, $src2\t# int" %}
13626
13627 ins_cost(INSN_COST);
13628 ins_encode %{
13629 __ eorw(as_Register($dst$$reg),
13630 as_Register($src1$$reg),
13631 (uint64_t)($src2$$constant));
13632 %}
13633
13634 ins_pipe(ialu_reg_imm);
13635 %}
13636
13637 // Long Logical Instructions
13638 // TODO
13639
13640 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
13641 match(Set dst (AndL src1 src2));
13642
13643 format %{ "and $dst, $src1, $src2\t# int" %}
13644
13645 ins_cost(INSN_COST);
13646 ins_encode %{
13647 __ andr(as_Register($dst$$reg),
13648 as_Register($src1$$reg),
13649 as_Register($src2$$reg));
13650 %}
13651
13652 ins_pipe(ialu_reg_reg);
13653 %}
13654
13655 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
13656 match(Set dst (AndL src1 src2));
13657
13658 format %{ "and $dst, $src1, $src2\t# int" %}
13659
13660 ins_cost(INSN_COST);
13661 ins_encode %{
13662 __ andr(as_Register($dst$$reg),
13663 as_Register($src1$$reg),
13664 (uint64_t)($src2$$constant));
13665 %}
13666
13667 ins_pipe(ialu_reg_imm);
13668 %}
13669
13670 // Or Instructions
13671
13672 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13673 match(Set dst (OrL src1 src2));
13674
13675 format %{ "orr $dst, $src1, $src2\t# int" %}
13676
13677 ins_cost(INSN_COST);
13678 ins_encode %{
13679 __ orr(as_Register($dst$$reg),
13680 as_Register($src1$$reg),
13681 as_Register($src2$$reg));
13682 %}
13683
13684 ins_pipe(ialu_reg_reg);
13685 %}
13686
13687 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13688 match(Set dst (OrL src1 src2));
13689
13690 format %{ "orr $dst, $src1, $src2\t# int" %}
13691
13692 ins_cost(INSN_COST);
13693 ins_encode %{
13694 __ orr(as_Register($dst$$reg),
13695 as_Register($src1$$reg),
13696 (uint64_t)($src2$$constant));
13697 %}
13698
13699 ins_pipe(ialu_reg_imm);
13700 %}
13701
13702 // Xor Instructions
13703
13704 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
13705 match(Set dst (XorL src1 src2));
13706
13707 format %{ "eor $dst, $src1, $src2\t# int" %}
13708
13709 ins_cost(INSN_COST);
13710 ins_encode %{
13711 __ eor(as_Register($dst$$reg),
13712 as_Register($src1$$reg),
13713 as_Register($src2$$reg));
13714 %}
13715
13716 ins_pipe(ialu_reg_reg);
13717 %}
13718
13719 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
13720 match(Set dst (XorL src1 src2));
13721
13722 ins_cost(INSN_COST);
13723 format %{ "eor $dst, $src1, $src2\t# int" %}
13724
13725 ins_encode %{
13726 __ eor(as_Register($dst$$reg),
13727 as_Register($src1$$reg),
13728 (uint64_t)($src2$$constant));
13729 %}
13730
13731 ins_pipe(ialu_reg_imm);
13732 %}
13733
13734 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
13735 %{
13736 match(Set dst (ConvI2L src));
13737
13738 ins_cost(INSN_COST);
13739 format %{ "sxtw $dst, $src\t# i2l" %}
13740 ins_encode %{
13741 __ sbfm($dst$$Register, $src$$Register, 0, 31);
13742 %}
13743 ins_pipe(ialu_reg_shift);
13744 %}
13745
13746 // this pattern occurs in bigmath arithmetic
13747 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
13748 %{
13749 match(Set dst (AndL (ConvI2L src) mask));
13750
13751 ins_cost(INSN_COST);
13752 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
13753 ins_encode %{
13754 __ ubfm($dst$$Register, $src$$Register, 0, 31);
13755 %}
13756
13757 ins_pipe(ialu_reg_shift);
13758 %}
13759
13760 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
13761 match(Set dst (ConvL2I src));
13762
13763 ins_cost(INSN_COST);
13764 format %{ "movw $dst, $src \t// l2i" %}
13765
13766 ins_encode %{
13767 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
13768 %}
13769
13770 ins_pipe(ialu_reg);
13771 %}
13772
13773 instruct convD2F_reg(vRegF dst, vRegD src) %{
13774 match(Set dst (ConvD2F src));
13775
13776 ins_cost(INSN_COST * 5);
13777 format %{ "fcvtd $dst, $src \t// d2f" %}
13778
13779 ins_encode %{
13780 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13781 %}
13782
13783 ins_pipe(fp_d2f);
13784 %}
13785
13786 instruct convF2D_reg(vRegD dst, vRegF src) %{
13787 match(Set dst (ConvF2D src));
13788
13789 ins_cost(INSN_COST * 5);
13790 format %{ "fcvts $dst, $src \t// f2d" %}
13791
13792 ins_encode %{
13793 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
13794 %}
13795
13796 ins_pipe(fp_f2d);
13797 %}
13798
13799 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
13800 match(Set dst (ConvF2I src));
13801
13802 ins_cost(INSN_COST * 5);
13803 format %{ "fcvtzsw $dst, $src \t// f2i" %}
13804
13805 ins_encode %{
13806 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13807 %}
13808
13809 ins_pipe(fp_f2i);
13810 %}
13811
13812 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
13813 match(Set dst (ConvF2L src));
13814
13815 ins_cost(INSN_COST * 5);
13816 format %{ "fcvtzs $dst, $src \t// f2l" %}
13817
13818 ins_encode %{
13819 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13820 %}
13821
13822 ins_pipe(fp_f2l);
13823 %}
13824
13825 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
13826 match(Set dst (ConvF2HF src));
13827 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
13828 "smov $dst, $tmp\t# move result from $tmp to $dst"
13829 %}
13830 effect(TEMP tmp);
13831 ins_encode %{
13832 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
13833 %}
13834 ins_pipe(pipe_slow);
13835 %}
13836
13837 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
13838 match(Set dst (ConvHF2F src));
13839 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
13840 "fcvt $dst, $tmp\t# convert half to single precision"
13841 %}
13842 effect(TEMP tmp);
13843 ins_encode %{
13844 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
13845 %}
13846 ins_pipe(pipe_slow);
13847 %}
13848
13849 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
13850 match(Set dst (ConvI2F src));
13851
13852 ins_cost(INSN_COST * 5);
13853 format %{ "scvtfws $dst, $src \t// i2f" %}
13854
13855 ins_encode %{
13856 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13857 %}
13858
13859 ins_pipe(fp_i2f);
13860 %}
13861
13862 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
13863 match(Set dst (ConvL2F src));
13864
13865 ins_cost(INSN_COST * 5);
13866 format %{ "scvtfs $dst, $src \t// l2f" %}
13867
13868 ins_encode %{
13869 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13870 %}
13871
13872 ins_pipe(fp_l2f);
13873 %}
13874
13875 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
13876 match(Set dst (ConvD2I src));
13877
13878 ins_cost(INSN_COST * 5);
13879 format %{ "fcvtzdw $dst, $src \t// d2i" %}
13880
13881 ins_encode %{
13882 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13883 %}
13884
13885 ins_pipe(fp_d2i);
13886 %}
13887
13888 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
13889 match(Set dst (ConvD2L src));
13890
13891 ins_cost(INSN_COST * 5);
13892 format %{ "fcvtzd $dst, $src \t// d2l" %}
13893
13894 ins_encode %{
13895 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
13896 %}
13897
13898 ins_pipe(fp_d2l);
13899 %}
13900
13901 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
13902 match(Set dst (ConvI2D src));
13903
13904 ins_cost(INSN_COST * 5);
13905 format %{ "scvtfwd $dst, $src \t// i2d" %}
13906
13907 ins_encode %{
13908 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13909 %}
13910
13911 ins_pipe(fp_i2d);
13912 %}
13913
13914 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
13915 match(Set dst (ConvL2D src));
13916
13917 ins_cost(INSN_COST * 5);
13918 format %{ "scvtfd $dst, $src \t// l2d" %}
13919
13920 ins_encode %{
13921 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
13922 %}
13923
13924 ins_pipe(fp_l2d);
13925 %}
13926
13927 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
13928 %{
13929 match(Set dst (RoundD src));
13930 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13931 format %{ "java_round_double $dst,$src"%}
13932 ins_encode %{
13933 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
13934 as_FloatRegister($ftmp$$reg));
13935 %}
13936 ins_pipe(pipe_slow);
13937 %}
13938
13939 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
13940 %{
13941 match(Set dst (RoundF src));
13942 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
13943 format %{ "java_round_float $dst,$src"%}
13944 ins_encode %{
13945 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
13946 as_FloatRegister($ftmp$$reg));
13947 %}
13948 ins_pipe(pipe_slow);
13949 %}
13950
13951 // stack <-> reg and reg <-> reg shuffles with no conversion
13952
13953 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
13954
13955 match(Set dst (MoveF2I src));
13956
13957 effect(DEF dst, USE src);
13958
13959 ins_cost(4 * INSN_COST);
13960
13961 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
13962
13963 ins_encode %{
13964 __ ldrw($dst$$Register, Address(sp, $src$$disp));
13965 %}
13966
13967 ins_pipe(iload_reg_reg);
13968
13969 %}
13970
13971 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
13972
13973 match(Set dst (MoveI2F src));
13974
13975 effect(DEF dst, USE src);
13976
13977 ins_cost(4 * INSN_COST);
13978
13979 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
13980
13981 ins_encode %{
13982 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
13983 %}
13984
13985 ins_pipe(pipe_class_memory);
13986
13987 %}
13988
13989 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
13990
13991 match(Set dst (MoveD2L src));
13992
13993 effect(DEF dst, USE src);
13994
13995 ins_cost(4 * INSN_COST);
13996
13997 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
13998
13999 ins_encode %{
14000 __ ldr($dst$$Register, Address(sp, $src$$disp));
14001 %}
14002
14003 ins_pipe(iload_reg_reg);
14004
14005 %}
14006
14007 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14008
14009 match(Set dst (MoveL2D src));
14010
14011 effect(DEF dst, USE src);
14012
14013 ins_cost(4 * INSN_COST);
14014
14015 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14016
14017 ins_encode %{
14018 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14019 %}
14020
14021 ins_pipe(pipe_class_memory);
14022
14023 %}
14024
14025 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14026
14027 match(Set dst (MoveF2I src));
14028
14029 effect(DEF dst, USE src);
14030
14031 ins_cost(INSN_COST);
14032
14033 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14034
14035 ins_encode %{
14036 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14037 %}
14038
14039 ins_pipe(pipe_class_memory);
14040
14041 %}
14042
14043 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14044
14045 match(Set dst (MoveI2F src));
14046
14047 effect(DEF dst, USE src);
14048
14049 ins_cost(INSN_COST);
14050
14051 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14052
14053 ins_encode %{
14054 __ strw($src$$Register, Address(sp, $dst$$disp));
14055 %}
14056
14057 ins_pipe(istore_reg_reg);
14058
14059 %}
14060
14061 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14062
14063 match(Set dst (MoveD2L src));
14064
14065 effect(DEF dst, USE src);
14066
14067 ins_cost(INSN_COST);
14068
14069 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14070
14071 ins_encode %{
14072 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14073 %}
14074
14075 ins_pipe(pipe_class_memory);
14076
14077 %}
14078
14079 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14080
14081 match(Set dst (MoveL2D src));
14082
14083 effect(DEF dst, USE src);
14084
14085 ins_cost(INSN_COST);
14086
14087 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14088
14089 ins_encode %{
14090 __ str($src$$Register, Address(sp, $dst$$disp));
14091 %}
14092
14093 ins_pipe(istore_reg_reg);
14094
14095 %}
14096
14097 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14098
14099 match(Set dst (MoveF2I src));
14100
14101 effect(DEF dst, USE src);
14102
14103 ins_cost(INSN_COST);
14104
14105 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14106
14107 ins_encode %{
14108 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14109 %}
14110
14111 ins_pipe(fp_f2i);
14112
14113 %}
14114
14115 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14116
14117 match(Set dst (MoveI2F src));
14118
14119 effect(DEF dst, USE src);
14120
14121 ins_cost(INSN_COST);
14122
14123 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14124
14125 ins_encode %{
14126 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14127 %}
14128
14129 ins_pipe(fp_i2f);
14130
14131 %}
14132
14133 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14134
14135 match(Set dst (MoveD2L src));
14136
14137 effect(DEF dst, USE src);
14138
14139 ins_cost(INSN_COST);
14140
14141 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14142
14143 ins_encode %{
14144 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14145 %}
14146
14147 ins_pipe(fp_d2l);
14148
14149 %}
14150
14151 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14152
14153 match(Set dst (MoveL2D src));
14154
14155 effect(DEF dst, USE src);
14156
14157 ins_cost(INSN_COST);
14158
14159 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14160
14161 ins_encode %{
14162 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14163 %}
14164
14165 ins_pipe(fp_l2d);
14166
14167 %}
14168
14169 // ============================================================================
14170 // clearing of an array
14171
14172 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14173 %{
14174 match(Set dummy (ClearArray (Binary cnt base) zero));
14175 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14176
14177 ins_cost(4 * INSN_COST);
14178 format %{ "ClearArray $cnt, $base" %}
14179
14180 ins_encode %{
14181 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14182 if (tpc == nullptr) {
14183 ciEnv::current()->record_failure("CodeCache is full");
14184 return;
14185 }
14186 %}
14187
14188 ins_pipe(pipe_class_memory);
14189 %}
14190
14191 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14192 %{
14193 predicate(((ClearArrayNode*)n)->word_copy_only());
14194 match(Set dummy (ClearArray (Binary cnt base) val));
14195 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14196
14197 ins_cost(4 * INSN_COST);
14198 format %{ "ClearArray $cnt, $base, $val" %}
14199
14200 ins_encode %{
14201 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14202 %}
14203
14204 ins_pipe(pipe_class_memory);
14205 %}
14206
14207 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, immL0 zero, Universe dummy, rFlagsReg cr)
14208 %{
14209 predicate((uint64_t)n->in(2)->in(1)->get_long()
14210 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14211 && !((ClearArrayNode*)n)->word_copy_only());
14212 match(Set dummy (ClearArray (Binary cnt base) zero));
14213 effect(TEMP temp, USE_KILL base, KILL cr);
14214
14215 ins_cost(4 * INSN_COST);
14216 format %{ "ClearArray $cnt, $base" %}
14217
14218 ins_encode %{
14219 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14220 if (tpc == nullptr) {
14221 ciEnv::current()->record_failure("CodeCache is full");
14222 return;
14223 }
14224 %}
14225
14226 ins_pipe(pipe_class_memory);
14227 %}
14228
14229 // ============================================================================
14230 // Overflow Math Instructions
14231
14232 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14233 %{
14234 match(Set cr (OverflowAddI op1 op2));
14235
14236 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14237 ins_cost(INSN_COST);
14238 ins_encode %{
14239 __ cmnw($op1$$Register, $op2$$Register);
14240 %}
14241
14242 ins_pipe(icmp_reg_reg);
14243 %}
14244
14245 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14246 %{
14247 match(Set cr (OverflowAddI op1 op2));
14248
14249 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14250 ins_cost(INSN_COST);
14251 ins_encode %{
14252 __ cmnw($op1$$Register, $op2$$constant);
14253 %}
14254
14255 ins_pipe(icmp_reg_imm);
14256 %}
14257
14258 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14259 %{
14260 match(Set cr (OverflowAddL op1 op2));
14261
14262 format %{ "cmn $op1, $op2\t# overflow check long" %}
14263 ins_cost(INSN_COST);
14264 ins_encode %{
14265 __ cmn($op1$$Register, $op2$$Register);
14266 %}
14267
14268 ins_pipe(icmp_reg_reg);
14269 %}
14270
14271 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14272 %{
14273 match(Set cr (OverflowAddL op1 op2));
14274
14275 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14276 ins_cost(INSN_COST);
14277 ins_encode %{
14278 __ adds(zr, $op1$$Register, $op2$$constant);
14279 %}
14280
14281 ins_pipe(icmp_reg_imm);
14282 %}
14283
14284 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14285 %{
14286 match(Set cr (OverflowSubI op1 op2));
14287
14288 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14289 ins_cost(INSN_COST);
14290 ins_encode %{
14291 __ cmpw($op1$$Register, $op2$$Register);
14292 %}
14293
14294 ins_pipe(icmp_reg_reg);
14295 %}
14296
14297 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14298 %{
14299 match(Set cr (OverflowSubI op1 op2));
14300
14301 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14302 ins_cost(INSN_COST);
14303 ins_encode %{
14304 __ cmpw($op1$$Register, $op2$$constant);
14305 %}
14306
14307 ins_pipe(icmp_reg_imm);
14308 %}
14309
14310 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14311 %{
14312 match(Set cr (OverflowSubL op1 op2));
14313
14314 format %{ "cmp $op1, $op2\t# overflow check long" %}
14315 ins_cost(INSN_COST);
14316 ins_encode %{
14317 __ cmp($op1$$Register, $op2$$Register);
14318 %}
14319
14320 ins_pipe(icmp_reg_reg);
14321 %}
14322
14323 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14324 %{
14325 match(Set cr (OverflowSubL op1 op2));
14326
14327 format %{ "cmp $op1, $op2\t# overflow check long" %}
14328 ins_cost(INSN_COST);
14329 ins_encode %{
14330 __ subs(zr, $op1$$Register, $op2$$constant);
14331 %}
14332
14333 ins_pipe(icmp_reg_imm);
14334 %}
14335
14336 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14337 %{
14338 match(Set cr (OverflowSubI zero op1));
14339
14340 format %{ "cmpw zr, $op1\t# overflow check int" %}
14341 ins_cost(INSN_COST);
14342 ins_encode %{
14343 __ cmpw(zr, $op1$$Register);
14344 %}
14345
14346 ins_pipe(icmp_reg_imm);
14347 %}
14348
14349 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14350 %{
14351 match(Set cr (OverflowSubL zero op1));
14352
14353 format %{ "cmp zr, $op1\t# overflow check long" %}
14354 ins_cost(INSN_COST);
14355 ins_encode %{
14356 __ cmp(zr, $op1$$Register);
14357 %}
14358
14359 ins_pipe(icmp_reg_imm);
14360 %}
14361
14362 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14363 %{
14364 match(Set cr (OverflowMulI op1 op2));
14365
14366 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14367 "cmp rscratch1, rscratch1, sxtw\n\t"
14368 "movw rscratch1, #0x80000000\n\t"
14369 "cselw rscratch1, rscratch1, zr, NE\n\t"
14370 "cmpw rscratch1, #1" %}
14371 ins_cost(5 * INSN_COST);
14372 ins_encode %{
14373 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14374 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14375 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14376 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14377 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14378 %}
14379
14380 ins_pipe(pipe_slow);
14381 %}
14382
14383 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14384 %{
14385 match(If cmp (OverflowMulI op1 op2));
14386 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14387 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14388 effect(USE labl, KILL cr);
14389
14390 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14391 "cmp rscratch1, rscratch1, sxtw\n\t"
14392 "b$cmp $labl" %}
14393 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14394 ins_encode %{
14395 Label* L = $labl$$label;
14396 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14397 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14398 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14399 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14400 %}
14401
14402 ins_pipe(pipe_serial);
14403 %}
14404
14405 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14406 %{
14407 match(Set cr (OverflowMulL op1 op2));
14408
14409 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14410 "smulh rscratch2, $op1, $op2\n\t"
14411 "cmp rscratch2, rscratch1, ASR #63\n\t"
14412 "movw rscratch1, #0x80000000\n\t"
14413 "cselw rscratch1, rscratch1, zr, NE\n\t"
14414 "cmpw rscratch1, #1" %}
14415 ins_cost(6 * INSN_COST);
14416 ins_encode %{
14417 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14418 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14419 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14420 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14421 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14422 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14423 %}
14424
14425 ins_pipe(pipe_slow);
14426 %}
14427
14428 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14429 %{
14430 match(If cmp (OverflowMulL op1 op2));
14431 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14432 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14433 effect(USE labl, KILL cr);
14434
14435 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14436 "smulh rscratch2, $op1, $op2\n\t"
14437 "cmp rscratch2, rscratch1, ASR #63\n\t"
14438 "b$cmp $labl" %}
14439 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
14440 ins_encode %{
14441 Label* L = $labl$$label;
14442 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14443 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14444 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14445 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14446 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14447 %}
14448
14449 ins_pipe(pipe_serial);
14450 %}
14451
14452 // ============================================================================
14453 // Compare Instructions
14454
14455 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
14456 %{
14457 match(Set cr (CmpI op1 op2));
14458
14459 effect(DEF cr, USE op1, USE op2);
14460
14461 ins_cost(INSN_COST);
14462 format %{ "cmpw $op1, $op2" %}
14463
14464 ins_encode(aarch64_enc_cmpw(op1, op2));
14465
14466 ins_pipe(icmp_reg_reg);
14467 %}
14468
14469 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
14470 %{
14471 match(Set cr (CmpI op1 zero));
14472
14473 effect(DEF cr, USE op1);
14474
14475 ins_cost(INSN_COST);
14476 format %{ "cmpw $op1, 0" %}
14477
14478 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14479
14480 ins_pipe(icmp_reg_imm);
14481 %}
14482
14483 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
14484 %{
14485 match(Set cr (CmpI op1 op2));
14486
14487 effect(DEF cr, USE op1);
14488
14489 ins_cost(INSN_COST);
14490 format %{ "cmpw $op1, $op2" %}
14491
14492 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14493
14494 ins_pipe(icmp_reg_imm);
14495 %}
14496
14497 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
14498 %{
14499 match(Set cr (CmpI op1 op2));
14500
14501 effect(DEF cr, USE op1);
14502
14503 ins_cost(INSN_COST * 2);
14504 format %{ "cmpw $op1, $op2" %}
14505
14506 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14507
14508 ins_pipe(icmp_reg_imm);
14509 %}
14510
14511 // Unsigned compare Instructions; really, same as signed compare
14512 // except it should only be used to feed an If or a CMovI which takes a
14513 // cmpOpU.
14514
14515 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
14516 %{
14517 match(Set cr (CmpU op1 op2));
14518
14519 effect(DEF cr, USE op1, USE op2);
14520
14521 ins_cost(INSN_COST);
14522 format %{ "cmpw $op1, $op2\t# unsigned" %}
14523
14524 ins_encode(aarch64_enc_cmpw(op1, op2));
14525
14526 ins_pipe(icmp_reg_reg);
14527 %}
14528
14529 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
14530 %{
14531 match(Set cr (CmpU op1 zero));
14532
14533 effect(DEF cr, USE op1);
14534
14535 ins_cost(INSN_COST);
14536 format %{ "cmpw $op1, #0\t# unsigned" %}
14537
14538 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
14539
14540 ins_pipe(icmp_reg_imm);
14541 %}
14542
14543 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
14544 %{
14545 match(Set cr (CmpU op1 op2));
14546
14547 effect(DEF cr, USE op1);
14548
14549 ins_cost(INSN_COST);
14550 format %{ "cmpw $op1, $op2\t# unsigned" %}
14551
14552 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
14553
14554 ins_pipe(icmp_reg_imm);
14555 %}
14556
14557 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
14558 %{
14559 match(Set cr (CmpU op1 op2));
14560
14561 effect(DEF cr, USE op1);
14562
14563 ins_cost(INSN_COST * 2);
14564 format %{ "cmpw $op1, $op2\t# unsigned" %}
14565
14566 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
14567
14568 ins_pipe(icmp_reg_imm);
14569 %}
14570
14571 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14572 %{
14573 match(Set cr (CmpL op1 op2));
14574
14575 effect(DEF cr, USE op1, USE op2);
14576
14577 ins_cost(INSN_COST);
14578 format %{ "cmp $op1, $op2" %}
14579
14580 ins_encode(aarch64_enc_cmp(op1, op2));
14581
14582 ins_pipe(icmp_reg_reg);
14583 %}
14584
14585 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
14586 %{
14587 match(Set cr (CmpL op1 zero));
14588
14589 effect(DEF cr, USE op1);
14590
14591 ins_cost(INSN_COST);
14592 format %{ "tst $op1" %}
14593
14594 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14595
14596 ins_pipe(icmp_reg_imm);
14597 %}
14598
14599 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
14600 %{
14601 match(Set cr (CmpL op1 op2));
14602
14603 effect(DEF cr, USE op1);
14604
14605 ins_cost(INSN_COST);
14606 format %{ "cmp $op1, $op2" %}
14607
14608 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14609
14610 ins_pipe(icmp_reg_imm);
14611 %}
14612
14613 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
14614 %{
14615 match(Set cr (CmpL op1 op2));
14616
14617 effect(DEF cr, USE op1);
14618
14619 ins_cost(INSN_COST * 2);
14620 format %{ "cmp $op1, $op2" %}
14621
14622 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14623
14624 ins_pipe(icmp_reg_imm);
14625 %}
14626
14627 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
14628 %{
14629 match(Set cr (CmpUL op1 op2));
14630
14631 effect(DEF cr, USE op1, USE op2);
14632
14633 ins_cost(INSN_COST);
14634 format %{ "cmp $op1, $op2" %}
14635
14636 ins_encode(aarch64_enc_cmp(op1, op2));
14637
14638 ins_pipe(icmp_reg_reg);
14639 %}
14640
14641 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
14642 %{
14643 match(Set cr (CmpUL op1 zero));
14644
14645 effect(DEF cr, USE op1);
14646
14647 ins_cost(INSN_COST);
14648 format %{ "tst $op1" %}
14649
14650 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
14651
14652 ins_pipe(icmp_reg_imm);
14653 %}
14654
14655 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
14656 %{
14657 match(Set cr (CmpUL op1 op2));
14658
14659 effect(DEF cr, USE op1);
14660
14661 ins_cost(INSN_COST);
14662 format %{ "cmp $op1, $op2" %}
14663
14664 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
14665
14666 ins_pipe(icmp_reg_imm);
14667 %}
14668
14669 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
14670 %{
14671 match(Set cr (CmpUL op1 op2));
14672
14673 effect(DEF cr, USE op1);
14674
14675 ins_cost(INSN_COST * 2);
14676 format %{ "cmp $op1, $op2" %}
14677
14678 ins_encode(aarch64_enc_cmp_imm(op1, op2));
14679
14680 ins_pipe(icmp_reg_imm);
14681 %}
14682
14683 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
14684 %{
14685 match(Set cr (CmpP op1 op2));
14686
14687 effect(DEF cr, USE op1, USE op2);
14688
14689 ins_cost(INSN_COST);
14690 format %{ "cmp $op1, $op2\t // ptr" %}
14691
14692 ins_encode(aarch64_enc_cmpp(op1, op2));
14693
14694 ins_pipe(icmp_reg_reg);
14695 %}
14696
14697 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
14698 %{
14699 match(Set cr (CmpN op1 op2));
14700
14701 effect(DEF cr, USE op1, USE op2);
14702
14703 ins_cost(INSN_COST);
14704 format %{ "cmp $op1, $op2\t // compressed ptr" %}
14705
14706 ins_encode(aarch64_enc_cmpn(op1, op2));
14707
14708 ins_pipe(icmp_reg_reg);
14709 %}
14710
14711 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
14712 %{
14713 match(Set cr (CmpP op1 zero));
14714
14715 effect(DEF cr, USE op1, USE zero);
14716
14717 ins_cost(INSN_COST);
14718 format %{ "cmp $op1, 0\t // ptr" %}
14719
14720 ins_encode(aarch64_enc_testp(op1));
14721
14722 ins_pipe(icmp_reg_imm);
14723 %}
14724
14725 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
14726 %{
14727 match(Set cr (CmpN op1 zero));
14728
14729 effect(DEF cr, USE op1, USE zero);
14730
14731 ins_cost(INSN_COST);
14732 format %{ "cmp $op1, 0\t // compressed ptr" %}
14733
14734 ins_encode(aarch64_enc_testn(op1));
14735
14736 ins_pipe(icmp_reg_imm);
14737 %}
14738
14739 // FP comparisons
14740 //
14741 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
14742 // using normal cmpOp. See declaration of rFlagsReg for details.
14743
14744 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
14745 %{
14746 match(Set cr (CmpF src1 src2));
14747
14748 ins_cost(3 * INSN_COST);
14749 format %{ "fcmps $src1, $src2" %}
14750
14751 ins_encode %{
14752 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14753 %}
14754
14755 ins_pipe(pipe_class_compare);
14756 %}
14757
14758 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
14759 %{
14760 match(Set cr (CmpF src1 src2));
14761
14762 ins_cost(3 * INSN_COST);
14763 format %{ "fcmps $src1, 0.0" %}
14764
14765 ins_encode %{
14766 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
14767 %}
14768
14769 ins_pipe(pipe_class_compare);
14770 %}
14771 // FROM HERE
14772
14773 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
14774 %{
14775 match(Set cr (CmpD src1 src2));
14776
14777 ins_cost(3 * INSN_COST);
14778 format %{ "fcmpd $src1, $src2" %}
14779
14780 ins_encode %{
14781 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
14782 %}
14783
14784 ins_pipe(pipe_class_compare);
14785 %}
14786
14787 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
14788 %{
14789 match(Set cr (CmpD src1 src2));
14790
14791 ins_cost(3 * INSN_COST);
14792 format %{ "fcmpd $src1, 0.0" %}
14793
14794 ins_encode %{
14795 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
14796 %}
14797
14798 ins_pipe(pipe_class_compare);
14799 %}
14800
14801 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
14802 %{
14803 match(Set dst (CmpF3 src1 src2));
14804 effect(KILL cr);
14805
14806 ins_cost(5 * INSN_COST);
14807 format %{ "fcmps $src1, $src2\n\t"
14808 "csinvw($dst, zr, zr, eq\n\t"
14809 "csnegw($dst, $dst, $dst, lt)"
14810 %}
14811
14812 ins_encode %{
14813 Label done;
14814 FloatRegister s1 = as_FloatRegister($src1$$reg);
14815 FloatRegister s2 = as_FloatRegister($src2$$reg);
14816 Register d = as_Register($dst$$reg);
14817 __ fcmps(s1, s2);
14818 // installs 0 if EQ else -1
14819 __ csinvw(d, zr, zr, Assembler::EQ);
14820 // keeps -1 if less or unordered else installs 1
14821 __ csnegw(d, d, d, Assembler::LT);
14822 __ bind(done);
14823 %}
14824
14825 ins_pipe(pipe_class_default);
14826
14827 %}
14828
14829 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
14830 %{
14831 match(Set dst (CmpD3 src1 src2));
14832 effect(KILL cr);
14833
14834 ins_cost(5 * INSN_COST);
14835 format %{ "fcmpd $src1, $src2\n\t"
14836 "csinvw($dst, zr, zr, eq\n\t"
14837 "csnegw($dst, $dst, $dst, lt)"
14838 %}
14839
14840 ins_encode %{
14841 Label done;
14842 FloatRegister s1 = as_FloatRegister($src1$$reg);
14843 FloatRegister s2 = as_FloatRegister($src2$$reg);
14844 Register d = as_Register($dst$$reg);
14845 __ fcmpd(s1, s2);
14846 // installs 0 if EQ else -1
14847 __ csinvw(d, zr, zr, Assembler::EQ);
14848 // keeps -1 if less or unordered else installs 1
14849 __ csnegw(d, d, d, Assembler::LT);
14850 __ bind(done);
14851 %}
14852 ins_pipe(pipe_class_default);
14853
14854 %}
14855
14856 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
14857 %{
14858 match(Set dst (CmpF3 src1 zero));
14859 effect(KILL cr);
14860
14861 ins_cost(5 * INSN_COST);
14862 format %{ "fcmps $src1, 0.0\n\t"
14863 "csinvw($dst, zr, zr, eq\n\t"
14864 "csnegw($dst, $dst, $dst, lt)"
14865 %}
14866
14867 ins_encode %{
14868 Label done;
14869 FloatRegister s1 = as_FloatRegister($src1$$reg);
14870 Register d = as_Register($dst$$reg);
14871 __ fcmps(s1, 0.0);
14872 // installs 0 if EQ else -1
14873 __ csinvw(d, zr, zr, Assembler::EQ);
14874 // keeps -1 if less or unordered else installs 1
14875 __ csnegw(d, d, d, Assembler::LT);
14876 __ bind(done);
14877 %}
14878
14879 ins_pipe(pipe_class_default);
14880
14881 %}
14882
14883 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
14884 %{
14885 match(Set dst (CmpD3 src1 zero));
14886 effect(KILL cr);
14887
14888 ins_cost(5 * INSN_COST);
14889 format %{ "fcmpd $src1, 0.0\n\t"
14890 "csinvw($dst, zr, zr, eq\n\t"
14891 "csnegw($dst, $dst, $dst, lt)"
14892 %}
14893
14894 ins_encode %{
14895 Label done;
14896 FloatRegister s1 = as_FloatRegister($src1$$reg);
14897 Register d = as_Register($dst$$reg);
14898 __ fcmpd(s1, 0.0);
14899 // installs 0 if EQ else -1
14900 __ csinvw(d, zr, zr, Assembler::EQ);
14901 // keeps -1 if less or unordered else installs 1
14902 __ csnegw(d, d, d, Assembler::LT);
14903 __ bind(done);
14904 %}
14905 ins_pipe(pipe_class_default);
14906
14907 %}
14908
14909 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
14910 %{
14911 match(Set dst (CmpLTMask p q));
14912 effect(KILL cr);
14913
14914 ins_cost(3 * INSN_COST);
14915
14916 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
14917 "csetw $dst, lt\n\t"
14918 "subw $dst, zr, $dst"
14919 %}
14920
14921 ins_encode %{
14922 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
14923 __ csetw(as_Register($dst$$reg), Assembler::LT);
14924 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
14925 %}
14926
14927 ins_pipe(ialu_reg_reg);
14928 %}
14929
14930 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
14931 %{
14932 match(Set dst (CmpLTMask src zero));
14933 effect(KILL cr);
14934
14935 ins_cost(INSN_COST);
14936
14937 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
14938
14939 ins_encode %{
14940 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
14941 %}
14942
14943 ins_pipe(ialu_reg_shift);
14944 %}
14945
14946 // ============================================================================
14947 // Max and Min
14948
14949 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
14950
14951 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
14952 %{
14953 effect(DEF cr, USE src);
14954 ins_cost(INSN_COST);
14955 format %{ "cmpw $src, 0" %}
14956
14957 ins_encode %{
14958 __ cmpw($src$$Register, 0);
14959 %}
14960 ins_pipe(icmp_reg_imm);
14961 %}
14962
14963 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14964 %{
14965 match(Set dst (MinI src1 src2));
14966 ins_cost(INSN_COST * 3);
14967
14968 expand %{
14969 rFlagsReg cr;
14970 compI_reg_reg(cr, src1, src2);
14971 cmovI_reg_reg_lt(dst, src1, src2, cr);
14972 %}
14973 %}
14974
14975 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
14976 %{
14977 match(Set dst (MaxI src1 src2));
14978 ins_cost(INSN_COST * 3);
14979
14980 expand %{
14981 rFlagsReg cr;
14982 compI_reg_reg(cr, src1, src2);
14983 cmovI_reg_reg_gt(dst, src1, src2, cr);
14984 %}
14985 %}
14986
14987
14988 // ============================================================================
14989 // Branch Instructions
14990
14991 // Direct Branch.
14992 instruct branch(label lbl)
14993 %{
14994 match(Goto);
14995
14996 effect(USE lbl);
14997
14998 ins_cost(BRANCH_COST);
14999 format %{ "b $lbl" %}
15000
15001 ins_encode(aarch64_enc_b(lbl));
15002
15003 ins_pipe(pipe_branch);
15004 %}
15005
15006 // Conditional Near Branch
15007 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15008 %{
15009 // Same match rule as `branchConFar'.
15010 match(If cmp cr);
15011
15012 effect(USE lbl);
15013
15014 ins_cost(BRANCH_COST);
15015 // If set to 1 this indicates that the current instruction is a
15016 // short variant of a long branch. This avoids using this
15017 // instruction in first-pass matching. It will then only be used in
15018 // the `Shorten_branches' pass.
15019 // ins_short_branch(1);
15020 format %{ "b$cmp $lbl" %}
15021
15022 ins_encode(aarch64_enc_br_con(cmp, lbl));
15023
15024 ins_pipe(pipe_branch_cond);
15025 %}
15026
15027 // Conditional Near Branch Unsigned
15028 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15029 %{
15030 // Same match rule as `branchConFar'.
15031 match(If cmp cr);
15032
15033 effect(USE lbl);
15034
15035 ins_cost(BRANCH_COST);
15036 // If set to 1 this indicates that the current instruction is a
15037 // short variant of a long branch. This avoids using this
15038 // instruction in first-pass matching. It will then only be used in
15039 // the `Shorten_branches' pass.
15040 // ins_short_branch(1);
15041 format %{ "b$cmp $lbl\t# unsigned" %}
15042
15043 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15044
15045 ins_pipe(pipe_branch_cond);
15046 %}
15047
15048 // Make use of CBZ and CBNZ. These instructions, as well as being
15049 // shorter than (cmp; branch), have the additional benefit of not
15050 // killing the flags.
15051
15052 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15053 match(If cmp (CmpI op1 op2));
15054 effect(USE labl);
15055
15056 ins_cost(BRANCH_COST);
15057 format %{ "cbw$cmp $op1, $labl" %}
15058 ins_encode %{
15059 Label* L = $labl$$label;
15060 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15061 if (cond == Assembler::EQ)
15062 __ cbzw($op1$$Register, *L);
15063 else
15064 __ cbnzw($op1$$Register, *L);
15065 %}
15066 ins_pipe(pipe_cmp_branch);
15067 %}
15068
15069 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15070 match(If cmp (CmpL op1 op2));
15071 effect(USE labl);
15072
15073 ins_cost(BRANCH_COST);
15074 format %{ "cb$cmp $op1, $labl" %}
15075 ins_encode %{
15076 Label* L = $labl$$label;
15077 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15078 if (cond == Assembler::EQ)
15079 __ cbz($op1$$Register, *L);
15080 else
15081 __ cbnz($op1$$Register, *L);
15082 %}
15083 ins_pipe(pipe_cmp_branch);
15084 %}
15085
15086 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15087 match(If cmp (CmpP op1 op2));
15088 effect(USE labl);
15089
15090 ins_cost(BRANCH_COST);
15091 format %{ "cb$cmp $op1, $labl" %}
15092 ins_encode %{
15093 Label* L = $labl$$label;
15094 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15095 if (cond == Assembler::EQ)
15096 __ cbz($op1$$Register, *L);
15097 else
15098 __ cbnz($op1$$Register, *L);
15099 %}
15100 ins_pipe(pipe_cmp_branch);
15101 %}
15102
15103 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15104 match(If cmp (CmpN op1 op2));
15105 effect(USE labl);
15106
15107 ins_cost(BRANCH_COST);
15108 format %{ "cbw$cmp $op1, $labl" %}
15109 ins_encode %{
15110 Label* L = $labl$$label;
15111 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15112 if (cond == Assembler::EQ)
15113 __ cbzw($op1$$Register, *L);
15114 else
15115 __ cbnzw($op1$$Register, *L);
15116 %}
15117 ins_pipe(pipe_cmp_branch);
15118 %}
15119
15120 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15121 match(If cmp (CmpP (DecodeN oop) zero));
15122 effect(USE labl);
15123
15124 ins_cost(BRANCH_COST);
15125 format %{ "cb$cmp $oop, $labl" %}
15126 ins_encode %{
15127 Label* L = $labl$$label;
15128 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15129 if (cond == Assembler::EQ)
15130 __ cbzw($oop$$Register, *L);
15131 else
15132 __ cbnzw($oop$$Register, *L);
15133 %}
15134 ins_pipe(pipe_cmp_branch);
15135 %}
15136
15137 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15138 match(If cmp (CmpU op1 op2));
15139 effect(USE labl);
15140
15141 ins_cost(BRANCH_COST);
15142 format %{ "cbw$cmp $op1, $labl" %}
15143 ins_encode %{
15144 Label* L = $labl$$label;
15145 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15146 if (cond == Assembler::EQ || cond == Assembler::LS) {
15147 __ cbzw($op1$$Register, *L);
15148 } else {
15149 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15150 __ cbnzw($op1$$Register, *L);
15151 }
15152 %}
15153 ins_pipe(pipe_cmp_branch);
15154 %}
15155
15156 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15157 match(If cmp (CmpUL op1 op2));
15158 effect(USE labl);
15159
15160 ins_cost(BRANCH_COST);
15161 format %{ "cb$cmp $op1, $labl" %}
15162 ins_encode %{
15163 Label* L = $labl$$label;
15164 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15165 if (cond == Assembler::EQ || cond == Assembler::LS) {
15166 __ cbz($op1$$Register, *L);
15167 } else {
15168 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15169 __ cbnz($op1$$Register, *L);
15170 }
15171 %}
15172 ins_pipe(pipe_cmp_branch);
15173 %}
15174
15175 // Test bit and Branch
15176
15177 // Patterns for short (< 32KiB) variants
15178 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15179 match(If cmp (CmpL op1 op2));
15180 effect(USE labl);
15181
15182 ins_cost(BRANCH_COST);
15183 format %{ "cb$cmp $op1, $labl # long" %}
15184 ins_encode %{
15185 Label* L = $labl$$label;
15186 Assembler::Condition cond =
15187 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15188 __ tbr(cond, $op1$$Register, 63, *L);
15189 %}
15190 ins_pipe(pipe_cmp_branch);
15191 ins_short_branch(1);
15192 %}
15193
15194 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15195 match(If cmp (CmpI op1 op2));
15196 effect(USE labl);
15197
15198 ins_cost(BRANCH_COST);
15199 format %{ "cb$cmp $op1, $labl # int" %}
15200 ins_encode %{
15201 Label* L = $labl$$label;
15202 Assembler::Condition cond =
15203 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15204 __ tbr(cond, $op1$$Register, 31, *L);
15205 %}
15206 ins_pipe(pipe_cmp_branch);
15207 ins_short_branch(1);
15208 %}
15209
15210 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15211 match(If cmp (CmpL (AndL op1 op2) op3));
15212 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15213 effect(USE labl);
15214
15215 ins_cost(BRANCH_COST);
15216 format %{ "tb$cmp $op1, $op2, $labl" %}
15217 ins_encode %{
15218 Label* L = $labl$$label;
15219 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15220 int bit = exact_log2_long($op2$$constant);
15221 __ tbr(cond, $op1$$Register, bit, *L);
15222 %}
15223 ins_pipe(pipe_cmp_branch);
15224 ins_short_branch(1);
15225 %}
15226
15227 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15228 match(If cmp (CmpI (AndI op1 op2) op3));
15229 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15230 effect(USE labl);
15231
15232 ins_cost(BRANCH_COST);
15233 format %{ "tb$cmp $op1, $op2, $labl" %}
15234 ins_encode %{
15235 Label* L = $labl$$label;
15236 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15237 int bit = exact_log2((juint)$op2$$constant);
15238 __ tbr(cond, $op1$$Register, bit, *L);
15239 %}
15240 ins_pipe(pipe_cmp_branch);
15241 ins_short_branch(1);
15242 %}
15243
15244 // And far variants
15245 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15246 match(If cmp (CmpL op1 op2));
15247 effect(USE labl);
15248
15249 ins_cost(BRANCH_COST);
15250 format %{ "cb$cmp $op1, $labl # long" %}
15251 ins_encode %{
15252 Label* L = $labl$$label;
15253 Assembler::Condition cond =
15254 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15255 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15256 %}
15257 ins_pipe(pipe_cmp_branch);
15258 %}
15259
15260 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15261 match(If cmp (CmpI op1 op2));
15262 effect(USE labl);
15263
15264 ins_cost(BRANCH_COST);
15265 format %{ "cb$cmp $op1, $labl # int" %}
15266 ins_encode %{
15267 Label* L = $labl$$label;
15268 Assembler::Condition cond =
15269 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15270 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15271 %}
15272 ins_pipe(pipe_cmp_branch);
15273 %}
15274
15275 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15276 match(If cmp (CmpL (AndL op1 op2) op3));
15277 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15278 effect(USE labl);
15279
15280 ins_cost(BRANCH_COST);
15281 format %{ "tb$cmp $op1, $op2, $labl" %}
15282 ins_encode %{
15283 Label* L = $labl$$label;
15284 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15285 int bit = exact_log2_long($op2$$constant);
15286 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15287 %}
15288 ins_pipe(pipe_cmp_branch);
15289 %}
15290
15291 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15292 match(If cmp (CmpI (AndI op1 op2) op3));
15293 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15294 effect(USE labl);
15295
15296 ins_cost(BRANCH_COST);
15297 format %{ "tb$cmp $op1, $op2, $labl" %}
15298 ins_encode %{
15299 Label* L = $labl$$label;
15300 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15301 int bit = exact_log2((juint)$op2$$constant);
15302 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15303 %}
15304 ins_pipe(pipe_cmp_branch);
15305 %}
15306
15307 // Test bits
15308
15309 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15310 match(Set cr (CmpL (AndL op1 op2) op3));
15311 predicate(Assembler::operand_valid_for_logical_immediate
15312 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15313
15314 ins_cost(INSN_COST);
15315 format %{ "tst $op1, $op2 # long" %}
15316 ins_encode %{
15317 __ tst($op1$$Register, $op2$$constant);
15318 %}
15319 ins_pipe(ialu_reg_reg);
15320 %}
15321
15322 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15323 match(Set cr (CmpI (AndI op1 op2) op3));
15324 predicate(Assembler::operand_valid_for_logical_immediate
15325 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15326
15327 ins_cost(INSN_COST);
15328 format %{ "tst $op1, $op2 # int" %}
15329 ins_encode %{
15330 __ tstw($op1$$Register, $op2$$constant);
15331 %}
15332 ins_pipe(ialu_reg_reg);
15333 %}
15334
15335 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15336 match(Set cr (CmpL (AndL op1 op2) op3));
15337
15338 ins_cost(INSN_COST);
15339 format %{ "tst $op1, $op2 # long" %}
15340 ins_encode %{
15341 __ tst($op1$$Register, $op2$$Register);
15342 %}
15343 ins_pipe(ialu_reg_reg);
15344 %}
15345
15346 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15347 match(Set cr (CmpI (AndI op1 op2) op3));
15348
15349 ins_cost(INSN_COST);
15350 format %{ "tstw $op1, $op2 # int" %}
15351 ins_encode %{
15352 __ tstw($op1$$Register, $op2$$Register);
15353 %}
15354 ins_pipe(ialu_reg_reg);
15355 %}
15356
15357
15358 // Conditional Far Branch
15359 // Conditional Far Branch Unsigned
15360 // TODO: fixme
15361
15362 // counted loop end branch near
15363 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15364 %{
15365 match(CountedLoopEnd cmp cr);
15366
15367 effect(USE lbl);
15368
15369 ins_cost(BRANCH_COST);
15370 // short variant.
15371 // ins_short_branch(1);
15372 format %{ "b$cmp $lbl \t// counted loop end" %}
15373
15374 ins_encode(aarch64_enc_br_con(cmp, lbl));
15375
15376 ins_pipe(pipe_branch);
15377 %}
15378
15379 // counted loop end branch far
15380 // TODO: fixme
15381
15382 // ============================================================================
15383 // inlined locking and unlocking
15384
15385 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15386 %{
15387 match(Set cr (FastLock object box));
15388 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15389
15390 ins_cost(5 * INSN_COST);
15391 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15392
15393 ins_encode %{
15394 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15395 %}
15396
15397 ins_pipe(pipe_serial);
15398 %}
15399
15400 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15401 %{
15402 match(Set cr (FastUnlock object box));
15403 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15404
15405 ins_cost(5 * INSN_COST);
15406 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
15407
15408 ins_encode %{
15409 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15410 %}
15411
15412 ins_pipe(pipe_serial);
15413 %}
15414
15415 // ============================================================================
15416 // Safepoint Instructions
15417
15418 // TODO
15419 // provide a near and far version of this code
15420
15421 instruct safePoint(rFlagsReg cr, iRegP poll)
15422 %{
15423 match(SafePoint poll);
15424 effect(KILL cr);
15425
15426 format %{
15427 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
15428 %}
15429 ins_encode %{
15430 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
15431 %}
15432 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
15433 %}
15434
15435
15436 // ============================================================================
15437 // Procedure Call/Return Instructions
15438
15439 // Call Java Static Instruction
15440
15441 instruct CallStaticJavaDirect(method meth)
15442 %{
15443 match(CallStaticJava);
15444
15445 effect(USE meth);
15446
15447 ins_cost(CALL_COST);
15448
15449 format %{ "call,static $meth \t// ==> " %}
15450
15451 ins_encode(aarch64_enc_java_static_call(meth),
15452 aarch64_enc_call_epilog);
15453
15454 ins_pipe(pipe_class_call);
15455 %}
15456
15457 // TO HERE
15458
15459 // Call Java Dynamic Instruction
15460 instruct CallDynamicJavaDirect(method meth)
15461 %{
15462 match(CallDynamicJava);
15463
15464 effect(USE meth);
15465
15466 ins_cost(CALL_COST);
15467
15468 format %{ "CALL,dynamic $meth \t// ==> " %}
15469
15470 ins_encode(aarch64_enc_java_dynamic_call(meth),
15471 aarch64_enc_call_epilog);
15472
15473 ins_pipe(pipe_class_call);
15474 %}
15475
15476 // Call Runtime Instruction
15477
15478 instruct CallRuntimeDirect(method meth)
15479 %{
15480 match(CallRuntime);
15481
15482 effect(USE meth);
15483
15484 ins_cost(CALL_COST);
15485
15486 format %{ "CALL, runtime $meth" %}
15487
15488 ins_encode( aarch64_enc_java_to_runtime(meth) );
15489
15490 ins_pipe(pipe_class_call);
15491 %}
15492
15493 // Call Runtime Instruction
15494
15495 instruct CallLeafDirect(method meth)
15496 %{
15497 match(CallLeaf);
15498
15499 effect(USE meth);
15500
15501 ins_cost(CALL_COST);
15502
15503 format %{ "CALL, runtime leaf $meth" %}
15504
15505 ins_encode( aarch64_enc_java_to_runtime(meth) );
15506
15507 ins_pipe(pipe_class_call);
15508 %}
15509
15510 // Call Runtime Instruction without safepoint and with vector arguments
15511 instruct CallLeafDirectVector(method meth)
15512 %{
15513 match(CallLeafVector);
15514
15515 effect(USE meth);
15516
15517 ins_cost(CALL_COST);
15518
15519 format %{ "CALL, runtime leaf vector $meth" %}
15520
15521 ins_encode(aarch64_enc_java_to_runtime(meth));
15522
15523 ins_pipe(pipe_class_call);
15524 %}
15525
15526 // Call Runtime Instruction
15527
15528 // entry point is null, target holds the address to call
15529 instruct CallLeafNoFPIndirect(iRegP target)
15530 %{
15531 predicate(n->as_Call()->entry_point() == nullptr);
15532
15533 match(CallLeafNoFP target);
15534
15535 ins_cost(CALL_COST);
15536
15537 format %{ "CALL, runtime leaf nofp indirect $target" %}
15538
15539 ins_encode %{
15540 __ blr($target$$Register);
15541 %}
15542
15543 ins_pipe(pipe_class_call);
15544 %}
15545
15546 instruct CallLeafNoFPDirect(method meth)
15547 %{
15548 predicate(n->as_Call()->entry_point() != nullptr);
15549
15550 match(CallLeafNoFP);
15551
15552 effect(USE meth);
15553
15554 ins_cost(CALL_COST);
15555
15556 format %{ "CALL, runtime leaf nofp $meth" %}
15557
15558 ins_encode( aarch64_enc_java_to_runtime(meth) );
15559
15560 ins_pipe(pipe_class_call);
15561 %}
15562
15563 // Tail Call; Jump from runtime stub to Java code.
15564 // Also known as an 'interprocedural jump'.
15565 // Target of jump will eventually return to caller.
15566 // TailJump below removes the return address.
15567 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15568 // emitted just above the TailCall which has reset rfp to the caller state.
15569 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
15570 %{
15571 match(TailCall jump_target method_ptr);
15572
15573 ins_cost(CALL_COST);
15574
15575 format %{ "br $jump_target\t# $method_ptr holds method" %}
15576
15577 ins_encode(aarch64_enc_tail_call(jump_target));
15578
15579 ins_pipe(pipe_class_call);
15580 %}
15581
15582 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
15583 %{
15584 match(TailJump jump_target ex_oop);
15585
15586 ins_cost(CALL_COST);
15587
15588 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
15589
15590 ins_encode(aarch64_enc_tail_jmp(jump_target));
15591
15592 ins_pipe(pipe_class_call);
15593 %}
15594
15595 // Forward exception.
15596 instruct ForwardExceptionjmp()
15597 %{
15598 match(ForwardException);
15599 ins_cost(CALL_COST);
15600
15601 format %{ "b forward_exception_stub" %}
15602 ins_encode %{
15603 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
15604 %}
15605 ins_pipe(pipe_class_call);
15606 %}
15607
15608 // Create exception oop: created by stack-crawling runtime code.
15609 // Created exception is now available to this handler, and is setup
15610 // just prior to jumping to this handler. No code emitted.
15611 // TODO check
15612 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
15613 instruct CreateException(iRegP_R0 ex_oop)
15614 %{
15615 match(Set ex_oop (CreateEx));
15616
15617 format %{ " -- \t// exception oop; no code emitted" %}
15618
15619 size(0);
15620
15621 ins_encode( /*empty*/ );
15622
15623 ins_pipe(pipe_class_empty);
15624 %}
15625
15626 // Rethrow exception: The exception oop will come in the first
15627 // argument position. Then JUMP (not call) to the rethrow stub code.
15628 instruct RethrowException() %{
15629 match(Rethrow);
15630 ins_cost(CALL_COST);
15631
15632 format %{ "b rethrow_stub" %}
15633
15634 ins_encode( aarch64_enc_rethrow() );
15635
15636 ins_pipe(pipe_class_call);
15637 %}
15638
15639
15640 // Return Instruction
15641 // epilog node loads ret address into lr as part of frame pop
15642 instruct Ret()
15643 %{
15644 match(Return);
15645
15646 format %{ "ret\t// return register" %}
15647
15648 ins_encode( aarch64_enc_ret() );
15649
15650 ins_pipe(pipe_branch);
15651 %}
15652
15653 // Die now.
15654 instruct ShouldNotReachHere() %{
15655 match(Halt);
15656
15657 ins_cost(CALL_COST);
15658 format %{ "ShouldNotReachHere" %}
15659
15660 ins_encode %{
15661 if (is_reachable()) {
15662 const char* str = __ code_string(_halt_reason);
15663 __ stop(str);
15664 }
15665 %}
15666
15667 ins_pipe(pipe_class_default);
15668 %}
15669
15670 // ============================================================================
15671 // Partial Subtype Check
15672 //
15673 // superklass array for an instance of the superklass. Set a hidden
15674 // internal cache on a hit (cache is checked with exposed code in
15675 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
15676 // encoding ALSO sets flags.
15677
15678 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
15679 %{
15680 match(Set result (PartialSubtypeCheck sub super));
15681 predicate(!UseSecondarySupersTable);
15682 effect(KILL cr, KILL temp);
15683
15684 ins_cost(20 * INSN_COST); // slightly larger than the next version
15685 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15686
15687 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
15688
15689 opcode(0x1); // Force zero of result reg on hit
15690
15691 ins_pipe(pipe_class_memory);
15692 %}
15693
15694 // Two versions of partialSubtypeCheck, both used when we need to
15695 // search for a super class in the secondary supers array. The first
15696 // is used when we don't know _a priori_ the class being searched
15697 // for. The second, far more common, is used when we do know: this is
15698 // used for instanceof, checkcast, and any case where C2 can determine
15699 // it by constant propagation.
15700
15701 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
15702 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15703 rFlagsReg cr)
15704 %{
15705 match(Set result (PartialSubtypeCheck sub super));
15706 predicate(UseSecondarySupersTable);
15707 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15708
15709 ins_cost(10 * INSN_COST); // slightly larger than the next version
15710 format %{ "partialSubtypeCheck $result, $sub, $super" %}
15711
15712 ins_encode %{
15713 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
15714 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15715 $vtemp$$FloatRegister,
15716 $result$$Register, /*L_success*/nullptr);
15717 %}
15718
15719 ins_pipe(pipe_class_memory);
15720 %}
15721
15722 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
15723 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
15724 rFlagsReg cr)
15725 %{
15726 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
15727 predicate(UseSecondarySupersTable);
15728 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
15729
15730 ins_cost(5 * INSN_COST); // smaller than the next version
15731 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
15732
15733 ins_encode %{
15734 bool success = false;
15735 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
15736 if (InlineSecondarySupersTest) {
15737 success =
15738 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
15739 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
15740 $vtemp$$FloatRegister,
15741 $result$$Register,
15742 super_klass_slot);
15743 } else {
15744 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
15745 success = (call != nullptr);
15746 }
15747 if (!success) {
15748 ciEnv::current()->record_failure("CodeCache is full");
15749 return;
15750 }
15751 %}
15752
15753 ins_pipe(pipe_class_memory);
15754 %}
15755
15756 // Intrisics for String.compareTo()
15757
15758 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15759 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15760 %{
15761 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15762 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15763 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15764
15765 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15766 ins_encode %{
15767 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15768 __ string_compare($str1$$Register, $str2$$Register,
15769 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15770 $tmp1$$Register, $tmp2$$Register,
15771 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
15772 %}
15773 ins_pipe(pipe_class_memory);
15774 %}
15775
15776 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15777 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
15778 %{
15779 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15780 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15781 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15782
15783 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
15784 ins_encode %{
15785 __ string_compare($str1$$Register, $str2$$Register,
15786 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15787 $tmp1$$Register, $tmp2$$Register,
15788 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
15789 %}
15790 ins_pipe(pipe_class_memory);
15791 %}
15792
15793 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15794 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15795 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15796 %{
15797 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15798 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15799 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15800 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15801
15802 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15803 ins_encode %{
15804 __ string_compare($str1$$Register, $str2$$Register,
15805 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15806 $tmp1$$Register, $tmp2$$Register,
15807 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15808 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
15809 %}
15810 ins_pipe(pipe_class_memory);
15811 %}
15812
15813 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15814 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15815 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
15816 %{
15817 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15818 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15819 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
15820 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15821
15822 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
15823 ins_encode %{
15824 __ string_compare($str1$$Register, $str2$$Register,
15825 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15826 $tmp1$$Register, $tmp2$$Register,
15827 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
15828 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
15829 %}
15830 ins_pipe(pipe_class_memory);
15831 %}
15832
15833 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
15834 // these string_compare variants as NEON register type for convenience so that the prototype of
15835 // string_compare can be shared with all variants.
15836
15837 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15838 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15839 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15840 pRegGov_P1 pgtmp2, rFlagsReg cr)
15841 %{
15842 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
15843 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15844 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15845 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15846
15847 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15848 ins_encode %{
15849 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15850 __ string_compare($str1$$Register, $str2$$Register,
15851 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15852 $tmp1$$Register, $tmp2$$Register,
15853 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15854 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15855 StrIntrinsicNode::LL);
15856 %}
15857 ins_pipe(pipe_class_memory);
15858 %}
15859
15860 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15861 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15862 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15863 pRegGov_P1 pgtmp2, rFlagsReg cr)
15864 %{
15865 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
15866 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15867 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15868 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15869
15870 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15871 ins_encode %{
15872 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15873 __ string_compare($str1$$Register, $str2$$Register,
15874 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15875 $tmp1$$Register, $tmp2$$Register,
15876 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15877 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15878 StrIntrinsicNode::LU);
15879 %}
15880 ins_pipe(pipe_class_memory);
15881 %}
15882
15883 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15884 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15885 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15886 pRegGov_P1 pgtmp2, rFlagsReg cr)
15887 %{
15888 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
15889 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15890 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15891 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15892
15893 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15894 ins_encode %{
15895 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15896 __ string_compare($str1$$Register, $str2$$Register,
15897 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15898 $tmp1$$Register, $tmp2$$Register,
15899 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15900 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15901 StrIntrinsicNode::UL);
15902 %}
15903 ins_pipe(pipe_class_memory);
15904 %}
15905
15906 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
15907 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
15908 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
15909 pRegGov_P1 pgtmp2, rFlagsReg cr)
15910 %{
15911 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
15912 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
15913 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
15914 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
15915
15916 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
15917 ins_encode %{
15918 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
15919 __ string_compare($str1$$Register, $str2$$Register,
15920 $cnt1$$Register, $cnt2$$Register, $result$$Register,
15921 $tmp1$$Register, $tmp2$$Register,
15922 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
15923 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
15924 StrIntrinsicNode::UU);
15925 %}
15926 ins_pipe(pipe_class_memory);
15927 %}
15928
15929 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15930 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
15931 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15932 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15933 %{
15934 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
15935 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15936 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15937 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15938 TEMP vtmp0, TEMP vtmp1, KILL cr);
15939 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
15940 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15941
15942 ins_encode %{
15943 __ string_indexof($str1$$Register, $str2$$Register,
15944 $cnt1$$Register, $cnt2$$Register,
15945 $tmp1$$Register, $tmp2$$Register,
15946 $tmp3$$Register, $tmp4$$Register,
15947 $tmp5$$Register, $tmp6$$Register,
15948 -1, $result$$Register, StrIntrinsicNode::UU);
15949 %}
15950 ins_pipe(pipe_class_memory);
15951 %}
15952
15953 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15954 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
15955 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15956 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15957 %{
15958 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
15959 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15960 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15961 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
15962 TEMP vtmp0, TEMP vtmp1, KILL cr);
15963 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
15964 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15965
15966 ins_encode %{
15967 __ string_indexof($str1$$Register, $str2$$Register,
15968 $cnt1$$Register, $cnt2$$Register,
15969 $tmp1$$Register, $tmp2$$Register,
15970 $tmp3$$Register, $tmp4$$Register,
15971 $tmp5$$Register, $tmp6$$Register,
15972 -1, $result$$Register, StrIntrinsicNode::LL);
15973 %}
15974 ins_pipe(pipe_class_memory);
15975 %}
15976
15977 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
15978 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
15979 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
15980 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
15981 %{
15982 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
15983 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
15984 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
15985 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
15986 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
15987 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
15988 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
15989
15990 ins_encode %{
15991 __ string_indexof($str1$$Register, $str2$$Register,
15992 $cnt1$$Register, $cnt2$$Register,
15993 $tmp1$$Register, $tmp2$$Register,
15994 $tmp3$$Register, $tmp4$$Register,
15995 $tmp5$$Register, $tmp6$$Register,
15996 -1, $result$$Register, StrIntrinsicNode::UL);
15997 %}
15998 ins_pipe(pipe_class_memory);
15999 %}
16000
16001 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16002 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16003 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16004 %{
16005 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16006 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16007 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16008 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16009 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16010 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16011
16012 ins_encode %{
16013 int icnt2 = (int)$int_cnt2$$constant;
16014 __ string_indexof($str1$$Register, $str2$$Register,
16015 $cnt1$$Register, zr,
16016 $tmp1$$Register, $tmp2$$Register,
16017 $tmp3$$Register, $tmp4$$Register, zr, zr,
16018 icnt2, $result$$Register, StrIntrinsicNode::UU);
16019 %}
16020 ins_pipe(pipe_class_memory);
16021 %}
16022
16023 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16024 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16025 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16026 %{
16027 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16028 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16029 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16030 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16031 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16032 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16033
16034 ins_encode %{
16035 int icnt2 = (int)$int_cnt2$$constant;
16036 __ string_indexof($str1$$Register, $str2$$Register,
16037 $cnt1$$Register, zr,
16038 $tmp1$$Register, $tmp2$$Register,
16039 $tmp3$$Register, $tmp4$$Register, zr, zr,
16040 icnt2, $result$$Register, StrIntrinsicNode::LL);
16041 %}
16042 ins_pipe(pipe_class_memory);
16043 %}
16044
16045 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16046 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16047 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16048 %{
16049 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16050 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16051 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16052 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16053 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16054 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16055
16056 ins_encode %{
16057 int icnt2 = (int)$int_cnt2$$constant;
16058 __ string_indexof($str1$$Register, $str2$$Register,
16059 $cnt1$$Register, zr,
16060 $tmp1$$Register, $tmp2$$Register,
16061 $tmp3$$Register, $tmp4$$Register, zr, zr,
16062 icnt2, $result$$Register, StrIntrinsicNode::UL);
16063 %}
16064 ins_pipe(pipe_class_memory);
16065 %}
16066
16067 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16068 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16069 iRegINoSp tmp3, rFlagsReg cr)
16070 %{
16071 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16072 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16073 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16074 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16075
16076 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16077
16078 ins_encode %{
16079 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16080 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16081 $tmp3$$Register);
16082 %}
16083 ins_pipe(pipe_class_memory);
16084 %}
16085
16086 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16087 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16088 iRegINoSp tmp3, rFlagsReg cr)
16089 %{
16090 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16091 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16092 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16093 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16094
16095 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16096
16097 ins_encode %{
16098 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16099 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16100 $tmp3$$Register);
16101 %}
16102 ins_pipe(pipe_class_memory);
16103 %}
16104
16105 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16106 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16107 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16108 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16109 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16110 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16111 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16112 ins_encode %{
16113 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16114 $result$$Register, $ztmp1$$FloatRegister,
16115 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16116 $ptmp$$PRegister, true /* isL */);
16117 %}
16118 ins_pipe(pipe_class_memory);
16119 %}
16120
16121 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16122 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16123 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16124 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16125 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16126 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16127 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16128 ins_encode %{
16129 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16130 $result$$Register, $ztmp1$$FloatRegister,
16131 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16132 $ptmp$$PRegister, false /* isL */);
16133 %}
16134 ins_pipe(pipe_class_memory);
16135 %}
16136
16137 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16138 iRegI_R0 result, rFlagsReg cr)
16139 %{
16140 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16141 match(Set result (StrEquals (Binary str1 str2) cnt));
16142 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16143
16144 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16145 ins_encode %{
16146 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16147 __ string_equals($str1$$Register, $str2$$Register,
16148 $result$$Register, $cnt$$Register);
16149 %}
16150 ins_pipe(pipe_class_memory);
16151 %}
16152
16153 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16154 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16155 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16156 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16157 iRegP_R10 tmp, rFlagsReg cr)
16158 %{
16159 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16160 match(Set result (AryEq ary1 ary2));
16161 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16162 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16163 TEMP vtmp6, TEMP vtmp7, KILL cr);
16164
16165 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16166 ins_encode %{
16167 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16168 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16169 $result$$Register, $tmp$$Register, 1);
16170 if (tpc == nullptr) {
16171 ciEnv::current()->record_failure("CodeCache is full");
16172 return;
16173 }
16174 %}
16175 ins_pipe(pipe_class_memory);
16176 %}
16177
16178 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16179 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16180 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16181 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16182 iRegP_R10 tmp, rFlagsReg cr)
16183 %{
16184 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16185 match(Set result (AryEq ary1 ary2));
16186 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16187 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16188 TEMP vtmp6, TEMP vtmp7, KILL cr);
16189
16190 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16191 ins_encode %{
16192 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16193 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16194 $result$$Register, $tmp$$Register, 2);
16195 if (tpc == nullptr) {
16196 ciEnv::current()->record_failure("CodeCache is full");
16197 return;
16198 }
16199 %}
16200 ins_pipe(pipe_class_memory);
16201 %}
16202
16203 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
16204 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16205 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16206 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
16207 %{
16208 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
16209 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
16210 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
16211
16212 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
16213 ins_encode %{
16214 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
16215 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
16216 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
16217 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
16218 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
16219 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
16220 (BasicType)$basic_type$$constant);
16221 if (tpc == nullptr) {
16222 ciEnv::current()->record_failure("CodeCache is full");
16223 return;
16224 }
16225 %}
16226 ins_pipe(pipe_class_memory);
16227 %}
16228
16229 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16230 %{
16231 match(Set result (CountPositives ary1 len));
16232 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16233 format %{ "count positives byte[] $ary1,$len -> $result" %}
16234 ins_encode %{
16235 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16236 if (tpc == nullptr) {
16237 ciEnv::current()->record_failure("CodeCache is full");
16238 return;
16239 }
16240 %}
16241 ins_pipe( pipe_slow );
16242 %}
16243
16244 // fast char[] to byte[] compression
16245 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16246 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16247 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16248 iRegI_R0 result, rFlagsReg cr)
16249 %{
16250 match(Set result (StrCompressedCopy src (Binary dst len)));
16251 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16252 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16253
16254 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16255 ins_encode %{
16256 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16257 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16258 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16259 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16260 %}
16261 ins_pipe(pipe_slow);
16262 %}
16263
16264 // fast byte[] to char[] inflation
16265 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16266 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16267 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16268 %{
16269 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16270 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16271 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16272 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16273
16274 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16275 ins_encode %{
16276 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16277 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16278 $vtmp2$$FloatRegister, $tmp$$Register);
16279 if (tpc == nullptr) {
16280 ciEnv::current()->record_failure("CodeCache is full");
16281 return;
16282 }
16283 %}
16284 ins_pipe(pipe_class_memory);
16285 %}
16286
16287 // encode char[] to byte[] in ISO_8859_1
16288 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16289 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16290 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16291 iRegI_R0 result, rFlagsReg cr)
16292 %{
16293 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16294 match(Set result (EncodeISOArray src (Binary dst len)));
16295 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16296 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16297
16298 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16299 ins_encode %{
16300 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16301 $result$$Register, false,
16302 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16303 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16304 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16305 %}
16306 ins_pipe(pipe_class_memory);
16307 %}
16308
16309 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16310 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16311 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16312 iRegI_R0 result, rFlagsReg cr)
16313 %{
16314 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16315 match(Set result (EncodeISOArray src (Binary dst len)));
16316 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16317 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16318
16319 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16320 ins_encode %{
16321 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16322 $result$$Register, true,
16323 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16324 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16325 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16326 %}
16327 ins_pipe(pipe_class_memory);
16328 %}
16329
16330 //----------------------------- CompressBits/ExpandBits ------------------------
16331
16332 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16333 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16334 match(Set dst (CompressBits src mask));
16335 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16336 format %{ "mov $tsrc, $src\n\t"
16337 "mov $tmask, $mask\n\t"
16338 "bext $tdst, $tsrc, $tmask\n\t"
16339 "mov $dst, $tdst"
16340 %}
16341 ins_encode %{
16342 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16343 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16344 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16345 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16346 %}
16347 ins_pipe(pipe_slow);
16348 %}
16349
16350 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16351 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16352 match(Set dst (CompressBits (LoadI mem) mask));
16353 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16354 format %{ "ldrs $tsrc, $mem\n\t"
16355 "ldrs $tmask, $mask\n\t"
16356 "bext $tdst, $tsrc, $tmask\n\t"
16357 "mov $dst, $tdst"
16358 %}
16359 ins_encode %{
16360 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16361 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16362 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16363 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16364 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16365 %}
16366 ins_pipe(pipe_slow);
16367 %}
16368
16369 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16370 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16371 match(Set dst (CompressBits src mask));
16372 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16373 format %{ "mov $tsrc, $src\n\t"
16374 "mov $tmask, $mask\n\t"
16375 "bext $tdst, $tsrc, $tmask\n\t"
16376 "mov $dst, $tdst"
16377 %}
16378 ins_encode %{
16379 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16380 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16381 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16382 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16383 %}
16384 ins_pipe(pipe_slow);
16385 %}
16386
16387 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16388 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16389 match(Set dst (CompressBits (LoadL mem) mask));
16390 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16391 format %{ "ldrd $tsrc, $mem\n\t"
16392 "ldrd $tmask, $mask\n\t"
16393 "bext $tdst, $tsrc, $tmask\n\t"
16394 "mov $dst, $tdst"
16395 %}
16396 ins_encode %{
16397 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16398 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16399 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16400 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16401 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16402 %}
16403 ins_pipe(pipe_slow);
16404 %}
16405
16406 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16407 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16408 match(Set dst (ExpandBits src mask));
16409 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16410 format %{ "mov $tsrc, $src\n\t"
16411 "mov $tmask, $mask\n\t"
16412 "bdep $tdst, $tsrc, $tmask\n\t"
16413 "mov $dst, $tdst"
16414 %}
16415 ins_encode %{
16416 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16417 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16418 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16419 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16420 %}
16421 ins_pipe(pipe_slow);
16422 %}
16423
16424 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16425 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16426 match(Set dst (ExpandBits (LoadI mem) mask));
16427 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16428 format %{ "ldrs $tsrc, $mem\n\t"
16429 "ldrs $tmask, $mask\n\t"
16430 "bdep $tdst, $tsrc, $tmask\n\t"
16431 "mov $dst, $tdst"
16432 %}
16433 ins_encode %{
16434 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16435 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16436 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16437 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16438 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16439 %}
16440 ins_pipe(pipe_slow);
16441 %}
16442
16443 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16444 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16445 match(Set dst (ExpandBits src mask));
16446 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16447 format %{ "mov $tsrc, $src\n\t"
16448 "mov $tmask, $mask\n\t"
16449 "bdep $tdst, $tsrc, $tmask\n\t"
16450 "mov $dst, $tdst"
16451 %}
16452 ins_encode %{
16453 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16454 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16455 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16456 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16457 %}
16458 ins_pipe(pipe_slow);
16459 %}
16460
16461
16462 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16463 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16464 match(Set dst (ExpandBits (LoadL mem) mask));
16465 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16466 format %{ "ldrd $tsrc, $mem\n\t"
16467 "ldrd $tmask, $mask\n\t"
16468 "bdep $tdst, $tsrc, $tmask\n\t"
16469 "mov $dst, $tdst"
16470 %}
16471 ins_encode %{
16472 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16473 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16474 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16475 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16476 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16477 %}
16478 ins_pipe(pipe_slow);
16479 %}
16480
16481 //----------------------------- Reinterpret ----------------------------------
16482 // Reinterpret a half-precision float value in a floating point register to a general purpose register
16483 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
16484 match(Set dst (ReinterpretHF2S src));
16485 format %{ "reinterpretHF2S $dst, $src" %}
16486 ins_encode %{
16487 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
16488 %}
16489 ins_pipe(pipe_slow);
16490 %}
16491
16492 // Reinterpret a half-precision float value in a general purpose register to a floating point register
16493 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
16494 match(Set dst (ReinterpretS2HF src));
16495 format %{ "reinterpretS2HF $dst, $src" %}
16496 ins_encode %{
16497 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
16498 %}
16499 ins_pipe(pipe_slow);
16500 %}
16501
16502 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
16503 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
16504 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
16505 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
16506 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
16507 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
16508 // can be omitted in this pattern, resulting in -
16509 // fcvt $dst, $src // Convert float to half-precision float
16510 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
16511 %{
16512 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
16513 format %{ "convF2HFAndS2HF $dst, $src" %}
16514 ins_encode %{
16515 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
16516 %}
16517 ins_pipe(pipe_slow);
16518 %}
16519
16520 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
16521 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
16522 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
16523 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
16524 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
16525 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
16526 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
16527 // resulting in -
16528 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
16529 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
16530 %{
16531 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
16532 format %{ "convHF2SAndHF2F $dst, $src" %}
16533 ins_encode %{
16534 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
16535 %}
16536 ins_pipe(pipe_slow);
16537 %}
16538
16539 // ============================================================================
16540 // This name is KNOWN by the ADLC and cannot be changed.
16541 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16542 // for this guy.
16543 instruct tlsLoadP(thread_RegP dst)
16544 %{
16545 match(Set dst (ThreadLocal));
16546
16547 ins_cost(0);
16548
16549 format %{ " -- \t// $dst=Thread::current(), empty" %}
16550
16551 size(0);
16552
16553 ins_encode( /*empty*/ );
16554
16555 ins_pipe(pipe_class_empty);
16556 %}
16557
16558 //----------PEEPHOLE RULES-----------------------------------------------------
16559 // These must follow all instruction definitions as they use the names
16560 // defined in the instructions definitions.
16561 //
16562 // peepmatch ( root_instr_name [preceding_instruction]* );
16563 //
16564 // peepconstraint %{
16565 // (instruction_number.operand_name relational_op instruction_number.operand_name
16566 // [, ...] );
16567 // // instruction numbers are zero-based using left to right order in peepmatch
16568 //
16569 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
16570 // // provide an instruction_number.operand_name for each operand that appears
16571 // // in the replacement instruction's match rule
16572 //
16573 // ---------VM FLAGS---------------------------------------------------------
16574 //
16575 // All peephole optimizations can be turned off using -XX:-OptoPeephole
16576 //
16577 // Each peephole rule is given an identifying number starting with zero and
16578 // increasing by one in the order seen by the parser. An individual peephole
16579 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
16580 // on the command-line.
16581 //
16582 // ---------CURRENT LIMITATIONS----------------------------------------------
16583 //
16584 // Only match adjacent instructions in same basic block
16585 // Only equality constraints
16586 // Only constraints between operands, not (0.dest_reg == RAX_enc)
16587 // Only one replacement instruction
16588 //
16589 // ---------EXAMPLE----------------------------------------------------------
16590 //
16591 // // pertinent parts of existing instructions in architecture description
16592 // instruct movI(iRegINoSp dst, iRegI src)
16593 // %{
16594 // match(Set dst (CopyI src));
16595 // %}
16596 //
16597 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
16598 // %{
16599 // match(Set dst (AddI dst src));
16600 // effect(KILL cr);
16601 // %}
16602 //
16603 // // Change (inc mov) to lea
16604 // peephole %{
16605 // // increment preceded by register-register move
16606 // peepmatch ( incI_iReg movI );
16607 // // require that the destination register of the increment
16608 // // match the destination register of the move
16609 // peepconstraint ( 0.dst == 1.dst );
16610 // // construct a replacement instruction that sets
16611 // // the destination to ( move's source register + one )
16612 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
16613 // %}
16614 //
16615
16616 // Implementation no longer uses movX instructions since
16617 // machine-independent system no longer uses CopyX nodes.
16618 //
16619 // peephole
16620 // %{
16621 // peepmatch (incI_iReg movI);
16622 // peepconstraint (0.dst == 1.dst);
16623 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16624 // %}
16625
16626 // peephole
16627 // %{
16628 // peepmatch (decI_iReg movI);
16629 // peepconstraint (0.dst == 1.dst);
16630 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16631 // %}
16632
16633 // peephole
16634 // %{
16635 // peepmatch (addI_iReg_imm movI);
16636 // peepconstraint (0.dst == 1.dst);
16637 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
16638 // %}
16639
16640 // peephole
16641 // %{
16642 // peepmatch (incL_iReg movL);
16643 // peepconstraint (0.dst == 1.dst);
16644 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16645 // %}
16646
16647 // peephole
16648 // %{
16649 // peepmatch (decL_iReg movL);
16650 // peepconstraint (0.dst == 1.dst);
16651 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16652 // %}
16653
16654 // peephole
16655 // %{
16656 // peepmatch (addL_iReg_imm movL);
16657 // peepconstraint (0.dst == 1.dst);
16658 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
16659 // %}
16660
16661 // peephole
16662 // %{
16663 // peepmatch (addP_iReg_imm movP);
16664 // peepconstraint (0.dst == 1.dst);
16665 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
16666 // %}
16667
16668 // // Change load of spilled value to only a spill
16669 // instruct storeI(memory mem, iRegI src)
16670 // %{
16671 // match(Set mem (StoreI mem src));
16672 // %}
16673 //
16674 // instruct loadI(iRegINoSp dst, memory mem)
16675 // %{
16676 // match(Set dst (LoadI mem));
16677 // %}
16678 //
16679
16680 //----------SMARTSPILL RULES---------------------------------------------------
16681 // These must follow all instruction definitions as they use the names
16682 // defined in the instructions definitions.
16683
16684 // Local Variables:
16685 // mode: c++
16686 // End: