1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BootTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else {
1694 return 6 * NativeInstruction::instruction_size;
1695 }
1696 }
1697
1698 //=============================================================================
1699
1700 #ifndef PRODUCT
1701 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1702 st->print("BREAKPOINT");
1703 }
1704 #endif
1705
1706 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1707 __ brk(0);
1708 }
1709
1710 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1711 return MachNode::size(ra_);
1712 }
1713
1714 //=============================================================================
1715
1716 #ifndef PRODUCT
1717 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1718 st->print("nop \t# %d bytes pad for loops and calls", _count);
1719 }
1720 #endif
1721
1722 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1723 for (int i = 0; i < _count; i++) {
1724 __ nop();
1725 }
1726 }
1727
1728 uint MachNopNode::size(PhaseRegAlloc*) const {
1729 return _count * NativeInstruction::instruction_size;
1730 }
1731
1732 //=============================================================================
1733 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1734
1735 int ConstantTable::calculate_table_base_offset() const {
1736 return 0; // absolute addressing, no offset
1737 }
1738
1739 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1740 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1741 ShouldNotReachHere();
1742 }
1743
1744 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1745 // Empty encoding
1746 }
1747
1748 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1749 return 0;
1750 }
1751
1752 #ifndef PRODUCT
1753 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1754 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1755 }
1756 #endif
1757
1758 #ifndef PRODUCT
1759 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1760 Compile* C = ra_->C;
1761
1762 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1763
1764 if (C->output()->need_stack_bang(framesize))
1765 st->print("# stack bang size=%d\n\t", framesize);
1766
1767 if (VM_Version::use_rop_protection()) {
1768 st->print("ldr zr, [lr]\n\t");
1769 st->print("paciaz\n\t");
1770 }
1771 if (framesize < ((1 << 9) + 2 * wordSize)) {
1772 st->print("sub sp, sp, #%d\n\t", framesize);
1773 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1774 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1775 } else {
1776 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1777 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1778 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1779 st->print("sub sp, sp, rscratch1");
1780 }
1781 if (C->stub_function() == nullptr) {
1782 st->print("\n\t");
1783 st->print("ldr rscratch1, [guard]\n\t");
1784 st->print("dmb ishld\n\t");
1785 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1786 st->print("cmp rscratch1, rscratch2\n\t");
1787 st->print("b.eq skip");
1788 st->print("\n\t");
1789 st->print("blr #nmethod_entry_barrier_stub\n\t");
1790 st->print("b skip\n\t");
1791 st->print("guard: int\n\t");
1792 st->print("\n\t");
1793 st->print("skip:\n\t");
1794 }
1795 }
1796 #endif
1797
1798 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1799 Compile* C = ra_->C;
1800
1801 // n.b. frame size includes space for return pc and rfp
1802 const int framesize = C->output()->frame_size_in_bytes();
1803
1804 if (C->clinit_barrier_on_entry()) {
1805 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1806
1807 Label L_skip_barrier;
1808
1809 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1810 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1811 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1812 __ bind(L_skip_barrier);
1813 }
1814
1815 if (C->max_vector_size() > 0) {
1816 __ reinitialize_ptrue();
1817 }
1818
1819 int bangsize = C->output()->bang_size_in_bytes();
1820 if (C->output()->need_stack_bang(bangsize))
1821 __ generate_stack_overflow_check(bangsize);
1822
1823 __ build_frame(framesize);
1824
1825 if (C->stub_function() == nullptr) {
1826 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1827 // Dummy labels for just measuring the code size
1828 Label dummy_slow_path;
1829 Label dummy_continuation;
1830 Label dummy_guard;
1831 Label* slow_path = &dummy_slow_path;
1832 Label* continuation = &dummy_continuation;
1833 Label* guard = &dummy_guard;
1834 if (!Compile::current()->output()->in_scratch_emit_size()) {
1835 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1836 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1837 Compile::current()->output()->add_stub(stub);
1838 slow_path = &stub->entry();
1839 continuation = &stub->continuation();
1840 guard = &stub->guard();
1841 }
1842 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1843 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1844 }
1845
1846 if (VerifyStackAtCalls) {
1847 Unimplemented();
1848 }
1849
1850 C->output()->set_frame_complete(__ offset());
1851
1852 if (C->has_mach_constant_base_node()) {
1853 // NOTE: We set the table base offset here because users might be
1854 // emitted before MachConstantBaseNode.
1855 ConstantTable& constant_table = C->output()->constant_table();
1856 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1857 }
1858 }
1859
1860 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1861 {
1862 return MachNode::size(ra_); // too many variables; just compute it
1863 // the hard way
1864 }
1865
1866 int MachPrologNode::reloc() const
1867 {
1868 return 0;
1869 }
1870
1871 //=============================================================================
1872
1873 #ifndef PRODUCT
1874 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 st->print("# pop frame %d\n\t",framesize);
1879
1880 if (framesize == 0) {
1881 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1882 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1883 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1884 st->print("add sp, sp, #%d\n\t", framesize);
1885 } else {
1886 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1887 st->print("add sp, sp, rscratch1\n\t");
1888 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1889 }
1890 if (VM_Version::use_rop_protection()) {
1891 st->print("autiaz\n\t");
1892 st->print("ldr zr, [lr]\n\t");
1893 }
1894
1895 if (do_polling() && C->is_method_compilation()) {
1896 st->print("# test polling word\n\t");
1897 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1898 st->print("cmp sp, rscratch1\n\t");
1899 st->print("bhi #slow_path");
1900 }
1901 }
1902 #endif
1903
1904 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1905 Compile* C = ra_->C;
1906 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1907
1908 __ remove_frame(framesize);
1909
1910 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1911 __ reserved_stack_check();
1912 }
1913
1914 if (do_polling() && C->is_method_compilation()) {
1915 Label dummy_label;
1916 Label* code_stub = &dummy_label;
1917 if (!C->output()->in_scratch_emit_size()) {
1918 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1919 C->output()->add_stub(stub);
1920 code_stub = &stub->entry();
1921 }
1922 __ relocate(relocInfo::poll_return_type);
1923 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1924 }
1925 }
1926
1927 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1928 // Variable size. Determine dynamically.
1929 return MachNode::size(ra_);
1930 }
1931
1932 int MachEpilogNode::reloc() const {
1933 // Return number of relocatable values contained in this instruction.
1934 return 1; // 1 for polling page.
1935 }
1936
1937 const Pipeline * MachEpilogNode::pipeline() const {
1938 return MachNode::pipeline_class();
1939 }
1940
1941 //=============================================================================
1942
1943 static enum RC rc_class(OptoReg::Name reg) {
1944
1945 if (reg == OptoReg::Bad) {
1946 return rc_bad;
1947 }
1948
1949 // we have 32 int registers * 2 halves
1950 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1951
1952 if (reg < slots_of_int_registers) {
1953 return rc_int;
1954 }
1955
1956 // we have 32 float register * 8 halves
1957 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1958 if (reg < slots_of_int_registers + slots_of_float_registers) {
1959 return rc_float;
1960 }
1961
1962 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1963 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1964 return rc_predicate;
1965 }
1966
1967 // Between predicate regs & stack is the flags.
1968 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1969
1970 return rc_stack;
1971 }
1972
1973 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1974 Compile* C = ra_->C;
1975
1976 // Get registers to move.
1977 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1978 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1979 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1980 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1981
1982 enum RC src_hi_rc = rc_class(src_hi);
1983 enum RC src_lo_rc = rc_class(src_lo);
1984 enum RC dst_hi_rc = rc_class(dst_hi);
1985 enum RC dst_lo_rc = rc_class(dst_lo);
1986
1987 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1988
1989 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1990 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1991 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1992 "expected aligned-adjacent pairs");
1993 }
1994
1995 if (src_lo == dst_lo && src_hi == dst_hi) {
1996 return 0; // Self copy, no move.
1997 }
1998
1999 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2000 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2001 int src_offset = ra_->reg2offset(src_lo);
2002 int dst_offset = ra_->reg2offset(dst_lo);
2003
2004 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2005 uint ireg = ideal_reg();
2006 if (ireg == Op_VecA && masm) {
2007 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2008 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2009 // stack->stack
2010 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2011 sve_vector_reg_size_in_bytes);
2012 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2013 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2017 sve_vector_reg_size_in_bytes);
2018 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2019 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2020 as_FloatRegister(Matcher::_regEncode[src_lo]),
2021 as_FloatRegister(Matcher::_regEncode[src_lo]));
2022 } else {
2023 ShouldNotReachHere();
2024 }
2025 } else if (masm) {
2026 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2027 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2028 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2029 // stack->stack
2030 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2031 if (ireg == Op_VecD) {
2032 __ unspill(rscratch1, true, src_offset);
2033 __ spill(rscratch1, true, dst_offset);
2034 } else {
2035 __ spill_copy128(src_offset, dst_offset);
2036 }
2037 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2038 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2039 ireg == Op_VecD ? __ T8B : __ T16B,
2040 as_FloatRegister(Matcher::_regEncode[src_lo]));
2041 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2042 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2043 ireg == Op_VecD ? __ D : __ Q,
2044 ra_->reg2offset(dst_lo));
2045 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2046 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2047 ireg == Op_VecD ? __ D : __ Q,
2048 ra_->reg2offset(src_lo));
2049 } else {
2050 ShouldNotReachHere();
2051 }
2052 }
2053 } else if (masm) {
2054 switch (src_lo_rc) {
2055 case rc_int:
2056 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2057 if (is64) {
2058 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2059 as_Register(Matcher::_regEncode[src_lo]));
2060 } else {
2061 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2062 as_Register(Matcher::_regEncode[src_lo]));
2063 }
2064 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2065 if (is64) {
2066 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2067 as_Register(Matcher::_regEncode[src_lo]));
2068 } else {
2069 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2070 as_Register(Matcher::_regEncode[src_lo]));
2071 }
2072 } else { // gpr --> stack spill
2073 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2074 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2075 }
2076 break;
2077 case rc_float:
2078 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2079 if (is64) {
2080 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2081 as_FloatRegister(Matcher::_regEncode[src_lo]));
2082 } else {
2083 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2084 as_FloatRegister(Matcher::_regEncode[src_lo]));
2085 }
2086 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2087 if (is64) {
2088 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2089 as_FloatRegister(Matcher::_regEncode[src_lo]));
2090 } else {
2091 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2092 as_FloatRegister(Matcher::_regEncode[src_lo]));
2093 }
2094 } else { // fpr --> stack spill
2095 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2096 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2097 is64 ? __ D : __ S, dst_offset);
2098 }
2099 break;
2100 case rc_stack:
2101 if (dst_lo_rc == rc_int) { // stack --> gpr load
2102 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2103 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2104 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2105 is64 ? __ D : __ S, src_offset);
2106 } else if (dst_lo_rc == rc_predicate) {
2107 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2108 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2109 } else { // stack --> stack copy
2110 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2111 if (ideal_reg() == Op_RegVectMask) {
2112 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2113 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2114 } else {
2115 __ unspill(rscratch1, is64, src_offset);
2116 __ spill(rscratch1, is64, dst_offset);
2117 }
2118 }
2119 break;
2120 case rc_predicate:
2121 if (dst_lo_rc == rc_predicate) {
2122 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2123 } else if (dst_lo_rc == rc_stack) {
2124 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2125 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2126 } else {
2127 assert(false, "bad src and dst rc_class combination.");
2128 ShouldNotReachHere();
2129 }
2130 break;
2131 default:
2132 assert(false, "bad rc_class for spill");
2133 ShouldNotReachHere();
2134 }
2135 }
2136
2137 if (st) {
2138 st->print("spill ");
2139 if (src_lo_rc == rc_stack) {
2140 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2141 } else {
2142 st->print("%s -> ", Matcher::regName[src_lo]);
2143 }
2144 if (dst_lo_rc == rc_stack) {
2145 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2146 } else {
2147 st->print("%s", Matcher::regName[dst_lo]);
2148 }
2149 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2150 int vsize = 0;
2151 switch (ideal_reg()) {
2152 case Op_VecD:
2153 vsize = 64;
2154 break;
2155 case Op_VecX:
2156 vsize = 128;
2157 break;
2158 case Op_VecA:
2159 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2160 break;
2161 default:
2162 assert(false, "bad register type for spill");
2163 ShouldNotReachHere();
2164 }
2165 st->print("\t# vector spill size = %d", vsize);
2166 } else if (ideal_reg() == Op_RegVectMask) {
2167 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2168 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2169 st->print("\t# predicate spill size = %d", vsize);
2170 } else {
2171 st->print("\t# spill size = %d", is64 ? 64 : 32);
2172 }
2173 }
2174
2175 return 0;
2176
2177 }
2178
2179 #ifndef PRODUCT
2180 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2181 if (!ra_)
2182 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2183 else
2184 implementation(nullptr, ra_, false, st);
2185 }
2186 #endif
2187
2188 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2189 implementation(masm, ra_, false, nullptr);
2190 }
2191
2192 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2193 return MachNode::size(ra_);
2194 }
2195
2196 //=============================================================================
2197
2198 #ifndef PRODUCT
2199 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2200 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2201 int reg = ra_->get_reg_first(this);
2202 st->print("add %s, rsp, #%d]\t# box lock",
2203 Matcher::regName[reg], offset);
2204 }
2205 #endif
2206
2207 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2208 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2209 int reg = ra_->get_encode(this);
2210
2211 // This add will handle any 24-bit signed offset. 24 bits allows an
2212 // 8 megabyte stack frame.
2213 __ add(as_Register(reg), sp, offset);
2214 }
2215
2216 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2217 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2218 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2219
2220 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2221 return NativeInstruction::instruction_size;
2222 } else {
2223 return 2 * NativeInstruction::instruction_size;
2224 }
2225 }
2226
2227 //=============================================================================
2228
2229 #ifndef PRODUCT
2230 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2231 {
2232 st->print_cr("# MachUEPNode");
2233 if (UseCompressedClassPointers) {
2234 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2235 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2236 st->print_cr("\tcmpw rscratch1, r10");
2237 } else {
2238 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2239 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2240 st->print_cr("\tcmp rscratch1, r10");
2241 }
2242 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2243 }
2244 #endif
2245
2246 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2247 {
2248 __ ic_check(InteriorEntryAlignment);
2249 }
2250
2251 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2252 {
2253 return MachNode::size(ra_);
2254 }
2255
2256 // REQUIRED EMIT CODE
2257
2258 //=============================================================================
2259
2260 // Emit deopt handler code.
2261 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2262 {
2263 // Note that the code buffer's insts_mark is always relative to insts.
2264 // That's why we must use the macroassembler to generate a handler.
2265 address base = __ start_a_stub(size_deopt_handler());
2266 if (base == nullptr) {
2267 ciEnv::current()->record_failure("CodeCache is full");
2268 return 0; // CodeBuffer::expand failed
2269 }
2270
2271 int offset = __ offset();
2272 Label start;
2273 __ bind(start);
2274 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2275
2276 int entry_offset = __ offset();
2277 __ b(start);
2278
2279 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2280 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2281 "out of bounds read in post-call NOP check");
2282 __ end_a_stub();
2283 return entry_offset;
2284 }
2285
2286 // REQUIRED MATCHER CODE
2287
2288 //=============================================================================
2289
2290 bool Matcher::match_rule_supported(int opcode) {
2291 if (!has_match_rule(opcode))
2292 return false;
2293
2294 switch (opcode) {
2295 case Op_OnSpinWait:
2296 return VM_Version::supports_on_spin_wait();
2297 case Op_CacheWB:
2298 case Op_CacheWBPreSync:
2299 case Op_CacheWBPostSync:
2300 if (!VM_Version::supports_data_cache_line_flush()) {
2301 return false;
2302 }
2303 break;
2304 case Op_ExpandBits:
2305 case Op_CompressBits:
2306 if (!VM_Version::supports_svebitperm()) {
2307 return false;
2308 }
2309 break;
2310 case Op_FmaF:
2311 case Op_FmaD:
2312 case Op_FmaVF:
2313 case Op_FmaVD:
2314 if (!UseFMA) {
2315 return false;
2316 }
2317 break;
2318 case Op_FmaHF:
2319 // UseFMA flag also needs to be checked along with FEAT_FP16
2320 if (!UseFMA || !is_feat_fp16_supported()) {
2321 return false;
2322 }
2323 break;
2324 case Op_AddHF:
2325 case Op_SubHF:
2326 case Op_MulHF:
2327 case Op_DivHF:
2328 case Op_MinHF:
2329 case Op_MaxHF:
2330 case Op_SqrtHF:
2331 // Half-precision floating point scalar operations require FEAT_FP16
2332 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2333 // features are supported.
2334 if (!is_feat_fp16_supported()) {
2335 return false;
2336 }
2337 break;
2338 }
2339
2340 return true; // Per default match rules are supported.
2341 }
2342
2343 const RegMask* Matcher::predicate_reg_mask(void) {
2344 return &_PR_REG_mask;
2345 }
2346
2347 bool Matcher::supports_vector_calling_convention(void) {
2348 return EnableVectorSupport;
2349 }
2350
2351 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2352 assert(EnableVectorSupport, "sanity");
2353 int lo = V0_num;
2354 int hi = V0_H_num;
2355 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2356 hi = V0_K_num;
2357 }
2358 return OptoRegPair(hi, lo);
2359 }
2360
2361 // Is this branch offset short enough that a short branch can be used?
2362 //
2363 // NOTE: If the platform does not provide any short branch variants, then
2364 // this method should return false for offset 0.
2365 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2366 // The passed offset is relative to address of the branch.
2367
2368 return (-32768 <= offset && offset < 32768);
2369 }
2370
2371 // Vector width in bytes.
2372 int Matcher::vector_width_in_bytes(BasicType bt) {
2373 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2374 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2375 // Minimum 2 values in vector
2376 if (size < 2*type2aelembytes(bt)) size = 0;
2377 // But never < 4
2378 if (size < 4) size = 0;
2379 return size;
2380 }
2381
2382 // Limits on vector size (number of elements) loaded into vector.
2383 int Matcher::max_vector_size(const BasicType bt) {
2384 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2385 }
2386
2387 int Matcher::min_vector_size(const BasicType bt) {
2388 // Usually, the shortest vector length supported by AArch64 ISA and
2389 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2390 // vectors in a few special cases.
2391 int size;
2392 switch(bt) {
2393 case T_BOOLEAN:
2394 // Load/store a vector mask with only 2 elements for vector types
2395 // such as "2I/2F/2L/2D".
2396 size = 2;
2397 break;
2398 case T_BYTE:
2399 // Generate a "4B" vector, to support vector cast between "8B/16B"
2400 // and "4S/4I/4L/4F/4D".
2401 size = 4;
2402 break;
2403 case T_SHORT:
2404 // Generate a "2S" vector, to support vector cast between "4S/8S"
2405 // and "2I/2L/2F/2D".
2406 size = 2;
2407 break;
2408 default:
2409 // Limit the min vector length to 64-bit.
2410 size = 8 / type2aelembytes(bt);
2411 // The number of elements in a vector should be at least 2.
2412 size = MAX2(size, 2);
2413 }
2414
2415 int max_size = max_vector_size(bt);
2416 return MIN2(size, max_size);
2417 }
2418
2419 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2420 return Matcher::max_vector_size(bt);
2421 }
2422
2423 // Actual max scalable vector register length.
2424 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2425 return Matcher::max_vector_size(bt);
2426 }
2427
2428 // Vector ideal reg.
2429 uint Matcher::vector_ideal_reg(int len) {
2430 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2431 return Op_VecA;
2432 }
2433 switch(len) {
2434 // For 16-bit/32-bit mask vector, reuse VecD.
2435 case 2:
2436 case 4:
2437 case 8: return Op_VecD;
2438 case 16: return Op_VecX;
2439 }
2440 ShouldNotReachHere();
2441 return 0;
2442 }
2443
2444 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2445 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2446 switch (ideal_reg) {
2447 case Op_VecA: return new vecAOper();
2448 case Op_VecD: return new vecDOper();
2449 case Op_VecX: return new vecXOper();
2450 }
2451 ShouldNotReachHere();
2452 return nullptr;
2453 }
2454
2455 bool Matcher::is_reg2reg_move(MachNode* m) {
2456 return false;
2457 }
2458
2459 bool Matcher::is_generic_vector(MachOper* opnd) {
2460 return opnd->opcode() == VREG;
2461 }
2462
2463 // Return whether or not this register is ever used as an argument.
2464 // This function is used on startup to build the trampoline stubs in
2465 // generateOptoStub. Registers not mentioned will be killed by the VM
2466 // call in the trampoline, and arguments in those registers not be
2467 // available to the callee.
2468 bool Matcher::can_be_java_arg(int reg)
2469 {
2470 return
2471 reg == R0_num || reg == R0_H_num ||
2472 reg == R1_num || reg == R1_H_num ||
2473 reg == R2_num || reg == R2_H_num ||
2474 reg == R3_num || reg == R3_H_num ||
2475 reg == R4_num || reg == R4_H_num ||
2476 reg == R5_num || reg == R5_H_num ||
2477 reg == R6_num || reg == R6_H_num ||
2478 reg == R7_num || reg == R7_H_num ||
2479 reg == V0_num || reg == V0_H_num ||
2480 reg == V1_num || reg == V1_H_num ||
2481 reg == V2_num || reg == V2_H_num ||
2482 reg == V3_num || reg == V3_H_num ||
2483 reg == V4_num || reg == V4_H_num ||
2484 reg == V5_num || reg == V5_H_num ||
2485 reg == V6_num || reg == V6_H_num ||
2486 reg == V7_num || reg == V7_H_num;
2487 }
2488
2489 bool Matcher::is_spillable_arg(int reg)
2490 {
2491 return can_be_java_arg(reg);
2492 }
2493
2494 uint Matcher::int_pressure_limit()
2495 {
2496 // JDK-8183543: When taking the number of available registers as int
2497 // register pressure threshold, the jtreg test:
2498 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2499 // failed due to C2 compilation failure with
2500 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2501 //
2502 // A derived pointer is live at CallNode and then is flagged by RA
2503 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2504 // derived pointers and lastly fail to spill after reaching maximum
2505 // number of iterations. Lowering the default pressure threshold to
2506 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2507 // a high register pressure area of the code so that split_DEF can
2508 // generate DefinitionSpillCopy for the derived pointer.
2509 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2510 if (!PreserveFramePointer) {
2511 // When PreserveFramePointer is off, frame pointer is allocatable,
2512 // but different from other SOC registers, it is excluded from
2513 // fatproj's mask because its save type is No-Save. Decrease 1 to
2514 // ensure high pressure at fatproj when PreserveFramePointer is off.
2515 // See check_pressure_at_fatproj().
2516 default_int_pressure_threshold--;
2517 }
2518 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2519 }
2520
2521 uint Matcher::float_pressure_limit()
2522 {
2523 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2524 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2525 }
2526
2527 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2528 return false;
2529 }
2530
2531 const RegMask& Matcher::divI_proj_mask() {
2532 ShouldNotReachHere();
2533 return RegMask::EMPTY;
2534 }
2535
2536 // Register for MODI projection of divmodI.
2537 const RegMask& Matcher::modI_proj_mask() {
2538 ShouldNotReachHere();
2539 return RegMask::EMPTY;
2540 }
2541
2542 // Register for DIVL projection of divmodL.
2543 const RegMask& Matcher::divL_proj_mask() {
2544 ShouldNotReachHere();
2545 return RegMask::EMPTY;
2546 }
2547
2548 // Register for MODL projection of divmodL.
2549 const RegMask& Matcher::modL_proj_mask() {
2550 ShouldNotReachHere();
2551 return RegMask::EMPTY;
2552 }
2553
2554 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2555 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2556 Node* u = addp->fast_out(i);
2557 if (u->is_LoadStore()) {
2558 // On AArch64, LoadStoreNodes (i.e. compare and swap
2559 // instructions) only take register indirect as an operand, so
2560 // any attempt to use an AddPNode as an input to a LoadStoreNode
2561 // must fail.
2562 return false;
2563 }
2564 if (u->is_Mem()) {
2565 int opsize = u->as_Mem()->memory_size();
2566 assert(opsize > 0, "unexpected memory operand size");
2567 if (u->as_Mem()->memory_size() != (1<<shift)) {
2568 return false;
2569 }
2570 }
2571 }
2572 return true;
2573 }
2574
2575 // Convert BootTest condition to Assembler condition.
2576 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2577 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2578 Assembler::Condition result;
2579 switch(cond) {
2580 case BoolTest::eq:
2581 result = Assembler::EQ; break;
2582 case BoolTest::ne:
2583 result = Assembler::NE; break;
2584 case BoolTest::le:
2585 result = Assembler::LE; break;
2586 case BoolTest::ge:
2587 result = Assembler::GE; break;
2588 case BoolTest::lt:
2589 result = Assembler::LT; break;
2590 case BoolTest::gt:
2591 result = Assembler::GT; break;
2592 case BoolTest::ule:
2593 result = Assembler::LS; break;
2594 case BoolTest::uge:
2595 result = Assembler::HS; break;
2596 case BoolTest::ult:
2597 result = Assembler::LO; break;
2598 case BoolTest::ugt:
2599 result = Assembler::HI; break;
2600 case BoolTest::overflow:
2601 result = Assembler::VS; break;
2602 case BoolTest::no_overflow:
2603 result = Assembler::VC; break;
2604 default:
2605 ShouldNotReachHere();
2606 return Assembler::Condition(-1);
2607 }
2608
2609 // Check conversion
2610 if (cond & BoolTest::unsigned_compare) {
2611 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2612 } else {
2613 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2614 }
2615
2616 return result;
2617 }
2618
2619 // Binary src (Replicate con)
2620 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2621 if (n == nullptr || m == nullptr) {
2622 return false;
2623 }
2624
2625 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2626 return false;
2627 }
2628
2629 Node* imm_node = m->in(1);
2630 if (!imm_node->is_Con()) {
2631 return false;
2632 }
2633
2634 const Type* t = imm_node->bottom_type();
2635 if (!(t->isa_int() || t->isa_long())) {
2636 return false;
2637 }
2638
2639 switch (n->Opcode()) {
2640 case Op_AndV:
2641 case Op_OrV:
2642 case Op_XorV: {
2643 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2644 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2645 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2646 }
2647 case Op_AddVB:
2648 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2649 case Op_AddVS:
2650 case Op_AddVI:
2651 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2652 case Op_AddVL:
2653 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2654 default:
2655 return false;
2656 }
2657 }
2658
2659 // (XorV src (Replicate m1))
2660 // (XorVMask src (MaskAll m1))
2661 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2662 if (n != nullptr && m != nullptr) {
2663 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2664 VectorNode::is_all_ones_vector(m);
2665 }
2666 return false;
2667 }
2668
2669 // Should the matcher clone input 'm' of node 'n'?
2670 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2671 if (is_vshift_con_pattern(n, m) ||
2672 is_vector_bitwise_not_pattern(n, m) ||
2673 is_valid_sve_arith_imm_pattern(n, m) ||
2674 is_encode_and_store_pattern(n, m)) {
2675 mstack.push(m, Visit);
2676 return true;
2677 }
2678 return false;
2679 }
2680
2681 // Should the Matcher clone shifts on addressing modes, expecting them
2682 // to be subsumed into complex addressing expressions or compute them
2683 // into registers?
2684 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2685
2686 // Loads and stores with indirect memory input (e.g., volatile loads and
2687 // stores) do not subsume the input into complex addressing expressions. If
2688 // the addressing expression is input to at least one such load or store, do
2689 // not clone the addressing expression. Query needs_acquiring_load and
2690 // needs_releasing_store as a proxy for indirect memory input, as it is not
2691 // possible to directly query for indirect memory input at this stage.
2692 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2693 Node* n = m->fast_out(i);
2694 if (n->is_Load() && needs_acquiring_load(n)) {
2695 return false;
2696 }
2697 if (n->is_Store() && needs_releasing_store(n)) {
2698 return false;
2699 }
2700 }
2701
2702 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2703 return true;
2704 }
2705
2706 Node *off = m->in(AddPNode::Offset);
2707 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2708 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2709 // Are there other uses besides address expressions?
2710 !is_visited(off)) {
2711 address_visited.set(off->_idx); // Flag as address_visited
2712 mstack.push(off->in(2), Visit);
2713 Node *conv = off->in(1);
2714 if (conv->Opcode() == Op_ConvI2L &&
2715 // Are there other uses besides address expressions?
2716 !is_visited(conv)) {
2717 address_visited.set(conv->_idx); // Flag as address_visited
2718 mstack.push(conv->in(1), Pre_Visit);
2719 } else {
2720 mstack.push(conv, Pre_Visit);
2721 }
2722 address_visited.test_set(m->_idx); // Flag as address_visited
2723 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2724 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2725 return true;
2726 } else if (off->Opcode() == Op_ConvI2L &&
2727 // Are there other uses besides address expressions?
2728 !is_visited(off)) {
2729 address_visited.test_set(m->_idx); // Flag as address_visited
2730 address_visited.set(off->_idx); // Flag as address_visited
2731 mstack.push(off->in(1), Pre_Visit);
2732 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2733 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2734 return true;
2735 }
2736 return false;
2737 }
2738
2739 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2740 { \
2741 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2742 guarantee(DISP == 0, "mode not permitted for volatile"); \
2743 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2744 __ INSN(REG, as_Register(BASE)); \
2745 }
2746
2747
2748 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2749 {
2750 Address::extend scale;
2751
2752 // Hooboy, this is fugly. We need a way to communicate to the
2753 // encoder that the index needs to be sign extended, so we have to
2754 // enumerate all the cases.
2755 switch (opcode) {
2756 case INDINDEXSCALEDI2L:
2757 case INDINDEXSCALEDI2LN:
2758 case INDINDEXI2L:
2759 case INDINDEXI2LN:
2760 scale = Address::sxtw(size);
2761 break;
2762 default:
2763 scale = Address::lsl(size);
2764 }
2765
2766 if (index == -1) {
2767 return Address(base, disp);
2768 } else {
2769 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2770 return Address(base, as_Register(index), scale);
2771 }
2772 }
2773
2774
2775 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2776 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2777 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2778 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2779 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2780
2781 // Used for all non-volatile memory accesses. The use of
2782 // $mem->opcode() to discover whether this pattern uses sign-extended
2783 // offsets is something of a kludge.
2784 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2785 Register reg, int opcode,
2786 Register base, int index, int scale, int disp,
2787 int size_in_memory)
2788 {
2789 Address addr = mem2address(opcode, base, index, scale, disp);
2790 if (addr.getMode() == Address::base_plus_offset) {
2791 /* Fix up any out-of-range offsets. */
2792 assert_different_registers(rscratch1, base);
2793 assert_different_registers(rscratch1, reg);
2794 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2795 }
2796 (masm->*insn)(reg, addr);
2797 }
2798
2799 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2800 FloatRegister reg, int opcode,
2801 Register base, int index, int size, int disp,
2802 int size_in_memory)
2803 {
2804 Address::extend scale;
2805
2806 switch (opcode) {
2807 case INDINDEXSCALEDI2L:
2808 case INDINDEXSCALEDI2LN:
2809 scale = Address::sxtw(size);
2810 break;
2811 default:
2812 scale = Address::lsl(size);
2813 }
2814
2815 if (index == -1) {
2816 // Fix up any out-of-range offsets.
2817 assert_different_registers(rscratch1, base);
2818 Address addr = Address(base, disp);
2819 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2820 (masm->*insn)(reg, addr);
2821 } else {
2822 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2823 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2824 }
2825 }
2826
2827 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2828 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2829 int opcode, Register base, int index, int size, int disp)
2830 {
2831 if (index == -1) {
2832 (masm->*insn)(reg, T, Address(base, disp));
2833 } else {
2834 assert(disp == 0, "unsupported address mode");
2835 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2836 }
2837 }
2838
2839 %}
2840
2841
2842
2843 //----------ENCODING BLOCK-----------------------------------------------------
2844 // This block specifies the encoding classes used by the compiler to
2845 // output byte streams. Encoding classes are parameterized macros
2846 // used by Machine Instruction Nodes in order to generate the bit
2847 // encoding of the instruction. Operands specify their base encoding
2848 // interface with the interface keyword. There are currently
2849 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2850 // COND_INTER. REG_INTER causes an operand to generate a function
2851 // which returns its register number when queried. CONST_INTER causes
2852 // an operand to generate a function which returns the value of the
2853 // constant when queried. MEMORY_INTER causes an operand to generate
2854 // four functions which return the Base Register, the Index Register,
2855 // the Scale Value, and the Offset Value of the operand when queried.
2856 // COND_INTER causes an operand to generate six functions which return
2857 // the encoding code (ie - encoding bits for the instruction)
2858 // associated with each basic boolean condition for a conditional
2859 // instruction.
2860 //
2861 // Instructions specify two basic values for encoding. Again, a
2862 // function is available to check if the constant displacement is an
2863 // oop. They use the ins_encode keyword to specify their encoding
2864 // classes (which must be a sequence of enc_class names, and their
2865 // parameters, specified in the encoding block), and they use the
2866 // opcode keyword to specify, in order, their primary, secondary, and
2867 // tertiary opcode. Only the opcode sections which a particular
2868 // instruction needs for encoding need to be specified.
2869 encode %{
2870 // Build emit functions for each basic byte or larger field in the
2871 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2872 // from C++ code in the enc_class source block. Emit functions will
2873 // live in the main source block for now. In future, we can
2874 // generalize this by adding a syntax that specifies the sizes of
2875 // fields in an order, so that the adlc can build the emit functions
2876 // automagically
2877
2878 // catch all for unimplemented encodings
2879 enc_class enc_unimplemented %{
2880 __ unimplemented("C2 catch all");
2881 %}
2882
2883 // BEGIN Non-volatile memory access
2884
2885 // This encoding class is generated automatically from ad_encode.m4.
2886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2887 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2888 Register dst_reg = as_Register($dst$$reg);
2889 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2890 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2891 %}
2892
2893 // This encoding class is generated automatically from ad_encode.m4.
2894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2895 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2896 Register dst_reg = as_Register($dst$$reg);
2897 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2898 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2899 %}
2900
2901 // This encoding class is generated automatically from ad_encode.m4.
2902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2903 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2904 Register dst_reg = as_Register($dst$$reg);
2905 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2906 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2907 %}
2908
2909 // This encoding class is generated automatically from ad_encode.m4.
2910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2911 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2912 Register dst_reg = as_Register($dst$$reg);
2913 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2914 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2915 %}
2916
2917 // This encoding class is generated automatically from ad_encode.m4.
2918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2919 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2920 Register dst_reg = as_Register($dst$$reg);
2921 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2922 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2923 %}
2924
2925 // This encoding class is generated automatically from ad_encode.m4.
2926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2927 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2928 Register dst_reg = as_Register($dst$$reg);
2929 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2931 %}
2932
2933 // This encoding class is generated automatically from ad_encode.m4.
2934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2935 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2936 Register dst_reg = as_Register($dst$$reg);
2937 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2944 Register dst_reg = as_Register($dst$$reg);
2945 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2946 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2947 %}
2948
2949 // This encoding class is generated automatically from ad_encode.m4.
2950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2951 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2952 Register dst_reg = as_Register($dst$$reg);
2953 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2954 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2955 %}
2956
2957 // This encoding class is generated automatically from ad_encode.m4.
2958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2959 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2960 Register dst_reg = as_Register($dst$$reg);
2961 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2962 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2963 %}
2964
2965 // This encoding class is generated automatically from ad_encode.m4.
2966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2967 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2968 Register dst_reg = as_Register($dst$$reg);
2969 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2970 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2971 %}
2972
2973 // This encoding class is generated automatically from ad_encode.m4.
2974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2975 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2976 Register dst_reg = as_Register($dst$$reg);
2977 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2978 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2979 %}
2980
2981 // This encoding class is generated automatically from ad_encode.m4.
2982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2983 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2984 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2985 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2986 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2987 %}
2988
2989 // This encoding class is generated automatically from ad_encode.m4.
2990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2991 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2992 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2993 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2994 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2995 %}
2996
2997 // This encoding class is generated automatically from ad_encode.m4.
2998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2999 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3000 Register src_reg = as_Register($src$$reg);
3001 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3002 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3003 %}
3004
3005 // This encoding class is generated automatically from ad_encode.m4.
3006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3007 enc_class aarch64_enc_strb0(memory1 mem) %{
3008 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3009 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3010 %}
3011
3012 // This encoding class is generated automatically from ad_encode.m4.
3013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3014 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3015 Register src_reg = as_Register($src$$reg);
3016 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3017 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3018 %}
3019
3020 // This encoding class is generated automatically from ad_encode.m4.
3021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3022 enc_class aarch64_enc_strh0(memory2 mem) %{
3023 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3024 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3025 %}
3026
3027 // This encoding class is generated automatically from ad_encode.m4.
3028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3029 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3030 Register src_reg = as_Register($src$$reg);
3031 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3032 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3033 %}
3034
3035 // This encoding class is generated automatically from ad_encode.m4.
3036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3037 enc_class aarch64_enc_strw0(memory4 mem) %{
3038 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3039 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3040 %}
3041
3042 // This encoding class is generated automatically from ad_encode.m4.
3043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3044 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3045 Register src_reg = as_Register($src$$reg);
3046 // we sometimes get asked to store the stack pointer into the
3047 // current thread -- we cannot do that directly on AArch64
3048 if (src_reg == r31_sp) {
3049 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3050 __ mov(rscratch2, sp);
3051 src_reg = rscratch2;
3052 }
3053 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3054 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3055 %}
3056
3057 // This encoding class is generated automatically from ad_encode.m4.
3058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3059 enc_class aarch64_enc_str0(memory8 mem) %{
3060 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3061 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3062 %}
3063
3064 // This encoding class is generated automatically from ad_encode.m4.
3065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3066 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3067 FloatRegister src_reg = as_FloatRegister($src$$reg);
3068 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3069 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3070 %}
3071
3072 // This encoding class is generated automatically from ad_encode.m4.
3073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3074 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3075 FloatRegister src_reg = as_FloatRegister($src$$reg);
3076 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3077 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3078 %}
3079
3080 // This encoding class is generated automatically from ad_encode.m4.
3081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3082 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3083 __ membar(Assembler::StoreStore);
3084 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3085 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3086 %}
3087
3088 // END Non-volatile memory access
3089
3090 // Vector loads and stores
3091 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3092 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3093 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3094 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3095 %}
3096
3097 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3098 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3099 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3100 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3101 %}
3102
3103 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3104 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3105 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3106 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3107 %}
3108
3109 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3110 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3111 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3112 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3113 %}
3114
3115 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3116 FloatRegister src_reg = as_FloatRegister($src$$reg);
3117 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3118 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3119 %}
3120
3121 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3122 FloatRegister src_reg = as_FloatRegister($src$$reg);
3123 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3124 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3125 %}
3126
3127 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3128 FloatRegister src_reg = as_FloatRegister($src$$reg);
3129 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3130 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3131 %}
3132
3133 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3134 FloatRegister src_reg = as_FloatRegister($src$$reg);
3135 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3136 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3137 %}
3138
3139 // volatile loads and stores
3140
3141 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3142 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3143 rscratch1, stlrb);
3144 %}
3145
3146 enc_class aarch64_enc_stlrb0(memory mem) %{
3147 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3148 rscratch1, stlrb);
3149 %}
3150
3151 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3152 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, stlrh);
3154 %}
3155
3156 enc_class aarch64_enc_stlrh0(memory mem) %{
3157 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3158 rscratch1, stlrh);
3159 %}
3160
3161 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3162 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3163 rscratch1, stlrw);
3164 %}
3165
3166 enc_class aarch64_enc_stlrw0(memory mem) %{
3167 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3168 rscratch1, stlrw);
3169 %}
3170
3171 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3172 Register dst_reg = as_Register($dst$$reg);
3173 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3174 rscratch1, ldarb);
3175 __ sxtbw(dst_reg, dst_reg);
3176 %}
3177
3178 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3179 Register dst_reg = as_Register($dst$$reg);
3180 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, ldarb);
3182 __ sxtb(dst_reg, dst_reg);
3183 %}
3184
3185 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3186 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3187 rscratch1, ldarb);
3188 %}
3189
3190 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3191 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3192 rscratch1, ldarb);
3193 %}
3194
3195 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3196 Register dst_reg = as_Register($dst$$reg);
3197 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3198 rscratch1, ldarh);
3199 __ sxthw(dst_reg, dst_reg);
3200 %}
3201
3202 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3203 Register dst_reg = as_Register($dst$$reg);
3204 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3205 rscratch1, ldarh);
3206 __ sxth(dst_reg, dst_reg);
3207 %}
3208
3209 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3210 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3211 rscratch1, ldarh);
3212 %}
3213
3214 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3215 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3216 rscratch1, ldarh);
3217 %}
3218
3219 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3220 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3221 rscratch1, ldarw);
3222 %}
3223
3224 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3225 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, ldarw);
3227 %}
3228
3229 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3230 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3231 rscratch1, ldar);
3232 %}
3233
3234 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3235 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3236 rscratch1, ldarw);
3237 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3238 %}
3239
3240 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3241 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3242 rscratch1, ldar);
3243 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3244 %}
3245
3246 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3247 Register src_reg = as_Register($src$$reg);
3248 // we sometimes get asked to store the stack pointer into the
3249 // current thread -- we cannot do that directly on AArch64
3250 if (src_reg == r31_sp) {
3251 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3252 __ mov(rscratch2, sp);
3253 src_reg = rscratch2;
3254 }
3255 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3256 rscratch1, stlr);
3257 %}
3258
3259 enc_class aarch64_enc_stlr0(memory mem) %{
3260 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3261 rscratch1, stlr);
3262 %}
3263
3264 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3265 {
3266 FloatRegister src_reg = as_FloatRegister($src$$reg);
3267 __ fmovs(rscratch2, src_reg);
3268 }
3269 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3270 rscratch1, stlrw);
3271 %}
3272
3273 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3274 {
3275 FloatRegister src_reg = as_FloatRegister($src$$reg);
3276 __ fmovd(rscratch2, src_reg);
3277 }
3278 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3279 rscratch1, stlr);
3280 %}
3281
3282 // synchronized read/update encodings
3283
3284 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3285 Register dst_reg = as_Register($dst$$reg);
3286 Register base = as_Register($mem$$base);
3287 int index = $mem$$index;
3288 int scale = $mem$$scale;
3289 int disp = $mem$$disp;
3290 if (index == -1) {
3291 if (disp != 0) {
3292 __ lea(rscratch1, Address(base, disp));
3293 __ ldaxr(dst_reg, rscratch1);
3294 } else {
3295 // TODO
3296 // should we ever get anything other than this case?
3297 __ ldaxr(dst_reg, base);
3298 }
3299 } else {
3300 Register index_reg = as_Register(index);
3301 if (disp == 0) {
3302 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3303 __ ldaxr(dst_reg, rscratch1);
3304 } else {
3305 __ lea(rscratch1, Address(base, disp));
3306 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3307 __ ldaxr(dst_reg, rscratch1);
3308 }
3309 }
3310 %}
3311
3312 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3313 Register src_reg = as_Register($src$$reg);
3314 Register base = as_Register($mem$$base);
3315 int index = $mem$$index;
3316 int scale = $mem$$scale;
3317 int disp = $mem$$disp;
3318 if (index == -1) {
3319 if (disp != 0) {
3320 __ lea(rscratch2, Address(base, disp));
3321 __ stlxr(rscratch1, src_reg, rscratch2);
3322 } else {
3323 // TODO
3324 // should we ever get anything other than this case?
3325 __ stlxr(rscratch1, src_reg, base);
3326 }
3327 } else {
3328 Register index_reg = as_Register(index);
3329 if (disp == 0) {
3330 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3331 __ stlxr(rscratch1, src_reg, rscratch2);
3332 } else {
3333 __ lea(rscratch2, Address(base, disp));
3334 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3335 __ stlxr(rscratch1, src_reg, rscratch2);
3336 }
3337 }
3338 __ cmpw(rscratch1, zr);
3339 %}
3340
3341 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3342 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3343 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3344 Assembler::xword, /*acquire*/ false, /*release*/ true,
3345 /*weak*/ false, noreg);
3346 %}
3347
3348 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3349 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3350 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3351 Assembler::word, /*acquire*/ false, /*release*/ true,
3352 /*weak*/ false, noreg);
3353 %}
3354
3355 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3356 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3357 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3358 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3359 /*weak*/ false, noreg);
3360 %}
3361
3362 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3363 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3364 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3365 Assembler::byte, /*acquire*/ false, /*release*/ true,
3366 /*weak*/ false, noreg);
3367 %}
3368
3369
3370 // The only difference between aarch64_enc_cmpxchg and
3371 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3372 // CompareAndSwap sequence to serve as a barrier on acquiring a
3373 // lock.
3374 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegL oldval, iRegL newval) %{
3375 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3376 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3377 Assembler::xword, /*acquire*/ true, /*release*/ true,
3378 /*weak*/ false, noreg);
3379 %}
3380
3381 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegI oldval, iRegI newval) %{
3382 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3383 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3384 Assembler::word, /*acquire*/ true, /*release*/ true,
3385 /*weak*/ false, noreg);
3386 %}
3387
3388 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegI oldval, iRegI newval) %{
3389 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3390 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3391 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3392 /*weak*/ false, noreg);
3393 %}
3394
3395 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegI oldval, iRegI newval) %{
3396 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3397 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3398 Assembler::byte, /*acquire*/ true, /*release*/ true,
3399 /*weak*/ false, noreg);
3400 %}
3401
3402 // auxiliary used for CompareAndSwapX to set result register
3403 enc_class aarch64_enc_cset_eq(iRegI res) %{
3404 Register res_reg = as_Register($res$$reg);
3405 __ cset(res_reg, Assembler::EQ);
3406 %}
3407
3408 // prefetch encodings
3409
3410 enc_class aarch64_enc_prefetchw(memory mem) %{
3411 Register base = as_Register($mem$$base);
3412 int index = $mem$$index;
3413 int scale = $mem$$scale;
3414 int disp = $mem$$disp;
3415 if (index == -1) {
3416 // Fix up any out-of-range offsets.
3417 assert_different_registers(rscratch1, base);
3418 Address addr = Address(base, disp);
3419 addr = __ legitimize_address(addr, 8, rscratch1);
3420 __ prfm(addr, PSTL1KEEP);
3421 } else {
3422 Register index_reg = as_Register(index);
3423 if (disp == 0) {
3424 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3425 } else {
3426 __ lea(rscratch1, Address(base, disp));
3427 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3428 }
3429 }
3430 %}
3431
3432 // mov encodings
3433
3434 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3435 uint32_t con = (uint32_t)$src$$constant;
3436 Register dst_reg = as_Register($dst$$reg);
3437 if (con == 0) {
3438 __ movw(dst_reg, zr);
3439 } else {
3440 __ movw(dst_reg, con);
3441 }
3442 %}
3443
3444 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3445 Register dst_reg = as_Register($dst$$reg);
3446 uint64_t con = (uint64_t)$src$$constant;
3447 if (con == 0) {
3448 __ mov(dst_reg, zr);
3449 } else {
3450 __ mov(dst_reg, con);
3451 }
3452 %}
3453
3454 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3455 Register dst_reg = as_Register($dst$$reg);
3456 address con = (address)$src$$constant;
3457 if (con == nullptr || con == (address)1) {
3458 ShouldNotReachHere();
3459 } else {
3460 relocInfo::relocType rtype = $src->constant_reloc();
3461 if (rtype == relocInfo::oop_type) {
3462 __ movoop(dst_reg, (jobject)con);
3463 } else if (rtype == relocInfo::metadata_type) {
3464 __ mov_metadata(dst_reg, (Metadata*)con);
3465 } else {
3466 assert(rtype == relocInfo::none, "unexpected reloc type");
3467 if (! __ is_valid_AArch64_address(con) ||
3468 con < (address)(uintptr_t)os::vm_page_size()) {
3469 __ mov(dst_reg, con);
3470 } else {
3471 uint64_t offset;
3472 __ adrp(dst_reg, con, offset);
3473 __ add(dst_reg, dst_reg, offset);
3474 }
3475 }
3476 }
3477 %}
3478
3479 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3480 Register dst_reg = as_Register($dst$$reg);
3481 __ mov(dst_reg, zr);
3482 %}
3483
3484 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3485 Register dst_reg = as_Register($dst$$reg);
3486 __ mov(dst_reg, (uint64_t)1);
3487 %}
3488
3489 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3490 Register dst_reg = as_Register($dst$$reg);
3491 address con = (address)$src$$constant;
3492 if (con == nullptr) {
3493 ShouldNotReachHere();
3494 } else {
3495 relocInfo::relocType rtype = $src->constant_reloc();
3496 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3497 __ set_narrow_oop(dst_reg, (jobject)con);
3498 }
3499 %}
3500
3501 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3502 Register dst_reg = as_Register($dst$$reg);
3503 __ mov(dst_reg, zr);
3504 %}
3505
3506 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3507 Register dst_reg = as_Register($dst$$reg);
3508 address con = (address)$src$$constant;
3509 if (con == nullptr) {
3510 ShouldNotReachHere();
3511 } else {
3512 relocInfo::relocType rtype = $src->constant_reloc();
3513 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3514 __ set_narrow_klass(dst_reg, (Klass *)con);
3515 }
3516 %}
3517
3518 // arithmetic encodings
3519
3520 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3521 Register dst_reg = as_Register($dst$$reg);
3522 Register src_reg = as_Register($src1$$reg);
3523 int32_t con = (int32_t)$src2$$constant;
3524 // add has primary == 0, subtract has primary == 1
3525 if ($primary) { con = -con; }
3526 if (con < 0) {
3527 __ subw(dst_reg, src_reg, -con);
3528 } else {
3529 __ addw(dst_reg, src_reg, con);
3530 }
3531 %}
3532
3533 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3534 Register dst_reg = as_Register($dst$$reg);
3535 Register src_reg = as_Register($src1$$reg);
3536 int32_t con = (int32_t)$src2$$constant;
3537 // add has primary == 0, subtract has primary == 1
3538 if ($primary) { con = -con; }
3539 if (con < 0) {
3540 __ sub(dst_reg, src_reg, -con);
3541 } else {
3542 __ add(dst_reg, src_reg, con);
3543 }
3544 %}
3545
3546 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3547 Register dst_reg = as_Register($dst$$reg);
3548 Register src1_reg = as_Register($src1$$reg);
3549 Register src2_reg = as_Register($src2$$reg);
3550 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3551 %}
3552
3553 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3554 Register dst_reg = as_Register($dst$$reg);
3555 Register src1_reg = as_Register($src1$$reg);
3556 Register src2_reg = as_Register($src2$$reg);
3557 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3558 %}
3559
3560 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3561 Register dst_reg = as_Register($dst$$reg);
3562 Register src1_reg = as_Register($src1$$reg);
3563 Register src2_reg = as_Register($src2$$reg);
3564 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3565 %}
3566
3567 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3568 Register dst_reg = as_Register($dst$$reg);
3569 Register src1_reg = as_Register($src1$$reg);
3570 Register src2_reg = as_Register($src2$$reg);
3571 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3572 %}
3573
3574 // compare instruction encodings
3575
3576 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3577 Register reg1 = as_Register($src1$$reg);
3578 Register reg2 = as_Register($src2$$reg);
3579 __ cmpw(reg1, reg2);
3580 %}
3581
3582 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3583 Register reg = as_Register($src1$$reg);
3584 int32_t val = $src2$$constant;
3585 if (val >= 0) {
3586 __ subsw(zr, reg, val);
3587 } else {
3588 __ addsw(zr, reg, -val);
3589 }
3590 %}
3591
3592 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3593 Register reg1 = as_Register($src1$$reg);
3594 uint32_t val = (uint32_t)$src2$$constant;
3595 __ movw(rscratch1, val);
3596 __ cmpw(reg1, rscratch1);
3597 %}
3598
3599 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3600 Register reg1 = as_Register($src1$$reg);
3601 Register reg2 = as_Register($src2$$reg);
3602 __ cmp(reg1, reg2);
3603 %}
3604
3605 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3606 Register reg = as_Register($src1$$reg);
3607 int64_t val = $src2$$constant;
3608 if (val >= 0) {
3609 __ subs(zr, reg, val);
3610 } else if (val != -val) {
3611 __ adds(zr, reg, -val);
3612 } else {
3613 // aargh, Long.MIN_VALUE is a special case
3614 __ orr(rscratch1, zr, (uint64_t)val);
3615 __ subs(zr, reg, rscratch1);
3616 }
3617 %}
3618
3619 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3620 Register reg1 = as_Register($src1$$reg);
3621 uint64_t val = (uint64_t)$src2$$constant;
3622 __ mov(rscratch1, val);
3623 __ cmp(reg1, rscratch1);
3624 %}
3625
3626 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3627 Register reg1 = as_Register($src1$$reg);
3628 Register reg2 = as_Register($src2$$reg);
3629 __ cmp(reg1, reg2);
3630 %}
3631
3632 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3633 Register reg1 = as_Register($src1$$reg);
3634 Register reg2 = as_Register($src2$$reg);
3635 __ cmpw(reg1, reg2);
3636 %}
3637
3638 enc_class aarch64_enc_testp(iRegP src) %{
3639 Register reg = as_Register($src$$reg);
3640 __ cmp(reg, zr);
3641 %}
3642
3643 enc_class aarch64_enc_testn(iRegN src) %{
3644 Register reg = as_Register($src$$reg);
3645 __ cmpw(reg, zr);
3646 %}
3647
3648 enc_class aarch64_enc_b(label lbl) %{
3649 Label *L = $lbl$$label;
3650 __ b(*L);
3651 %}
3652
3653 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3654 Label *L = $lbl$$label;
3655 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3656 %}
3657
3658 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3659 Label *L = $lbl$$label;
3660 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3661 %}
3662
3663 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3664 %{
3665 Register sub_reg = as_Register($sub$$reg);
3666 Register super_reg = as_Register($super$$reg);
3667 Register temp_reg = as_Register($temp$$reg);
3668 Register result_reg = as_Register($result$$reg);
3669
3670 Label miss;
3671 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3672 nullptr, &miss,
3673 /*set_cond_codes:*/ true);
3674 if ($primary) {
3675 __ mov(result_reg, zr);
3676 }
3677 __ bind(miss);
3678 %}
3679
3680 enc_class aarch64_enc_java_static_call(method meth) %{
3681 address addr = (address)$meth$$method;
3682 address call;
3683 if (!_method) {
3684 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3685 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3686 if (call == nullptr) {
3687 ciEnv::current()->record_failure("CodeCache is full");
3688 return;
3689 }
3690 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3691 // The NOP here is purely to ensure that eliding a call to
3692 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3693 __ nop();
3694 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3695 } else {
3696 int method_index = resolved_method_index(masm);
3697 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3698 : static_call_Relocation::spec(method_index);
3699 call = __ trampoline_call(Address(addr, rspec));
3700 if (call == nullptr) {
3701 ciEnv::current()->record_failure("CodeCache is full");
3702 return;
3703 }
3704 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3705 // Calls of the same statically bound method can share
3706 // a stub to the interpreter.
3707 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3708 } else {
3709 // Emit stub for static call
3710 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3711 if (stub == nullptr) {
3712 ciEnv::current()->record_failure("CodeCache is full");
3713 return;
3714 }
3715 }
3716 }
3717
3718 __ post_call_nop();
3719
3720 // Only non uncommon_trap calls need to reinitialize ptrue.
3721 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3722 __ reinitialize_ptrue();
3723 }
3724 %}
3725
3726 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3727 int method_index = resolved_method_index(masm);
3728 address call = __ ic_call((address)$meth$$method, method_index);
3729 if (call == nullptr) {
3730 ciEnv::current()->record_failure("CodeCache is full");
3731 return;
3732 }
3733 __ post_call_nop();
3734 if (Compile::current()->max_vector_size() > 0) {
3735 __ reinitialize_ptrue();
3736 }
3737 %}
3738
3739 enc_class aarch64_enc_call_epilog() %{
3740 if (VerifyStackAtCalls) {
3741 // Check that stack depth is unchanged: find majik cookie on stack
3742 __ call_Unimplemented();
3743 }
3744 %}
3745
3746 enc_class aarch64_enc_java_to_runtime(method meth) %{
3747 // some calls to generated routines (arraycopy code) are scheduled
3748 // by C2 as runtime calls. if so we can call them using a br (they
3749 // will be in a reachable segment) otherwise we have to use a blr
3750 // which loads the absolute address into a register.
3751 address entry = (address)$meth$$method;
3752 CodeBlob *cb = CodeCache::find_blob(entry);
3753 if (cb) {
3754 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3755 if (call == nullptr) {
3756 ciEnv::current()->record_failure("CodeCache is full");
3757 return;
3758 }
3759 __ post_call_nop();
3760 } else {
3761 Label retaddr;
3762 // Make the anchor frame walkable
3763 __ adr(rscratch2, retaddr);
3764 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3765 __ lea(rscratch1, RuntimeAddress(entry));
3766 __ blr(rscratch1);
3767 __ bind(retaddr);
3768 __ post_call_nop();
3769 }
3770 if (Compile::current()->max_vector_size() > 0) {
3771 __ reinitialize_ptrue();
3772 }
3773 %}
3774
3775 enc_class aarch64_enc_rethrow() %{
3776 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3777 %}
3778
3779 enc_class aarch64_enc_ret() %{
3780 #ifdef ASSERT
3781 if (Compile::current()->max_vector_size() > 0) {
3782 __ verify_ptrue();
3783 }
3784 #endif
3785 __ ret(lr);
3786 %}
3787
3788 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3789 Register target_reg = as_Register($jump_target$$reg);
3790 __ br(target_reg);
3791 %}
3792
3793 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3794 Register target_reg = as_Register($jump_target$$reg);
3795 // exception oop should be in r0
3796 // ret addr has been popped into lr
3797 // callee expects it in r3
3798 __ mov(r3, lr);
3799 __ br(target_reg);
3800 %}
3801
3802 %}
3803
3804 //----------FRAME--------------------------------------------------------------
3805 // Definition of frame structure and management information.
3806 //
3807 // S T A C K L A Y O U T Allocators stack-slot number
3808 // | (to get allocators register number
3809 // G Owned by | | v add OptoReg::stack0())
3810 // r CALLER | |
3811 // o | +--------+ pad to even-align allocators stack-slot
3812 // w V | pad0 | numbers; owned by CALLER
3813 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3814 // h ^ | in | 5
3815 // | | args | 4 Holes in incoming args owned by SELF
3816 // | | | | 3
3817 // | | +--------+
3818 // V | | old out| Empty on Intel, window on Sparc
3819 // | old |preserve| Must be even aligned.
3820 // | SP-+--------+----> Matcher::_old_SP, even aligned
3821 // | | in | 3 area for Intel ret address
3822 // Owned by |preserve| Empty on Sparc.
3823 // SELF +--------+
3824 // | | pad2 | 2 pad to align old SP
3825 // | +--------+ 1
3826 // | | locks | 0
3827 // | +--------+----> OptoReg::stack0(), even aligned
3828 // | | pad1 | 11 pad to align new SP
3829 // | +--------+
3830 // | | | 10
3831 // | | spills | 9 spills
3832 // V | | 8 (pad0 slot for callee)
3833 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3834 // ^ | out | 7
3835 // | | args | 6 Holes in outgoing args owned by CALLEE
3836 // Owned by +--------+
3837 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3838 // | new |preserve| Must be even-aligned.
3839 // | SP-+--------+----> Matcher::_new_SP, even aligned
3840 // | | |
3841 //
3842 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3843 // known from SELF's arguments and the Java calling convention.
3844 // Region 6-7 is determined per call site.
3845 // Note 2: If the calling convention leaves holes in the incoming argument
3846 // area, those holes are owned by SELF. Holes in the outgoing area
3847 // are owned by the CALLEE. Holes should not be necessary in the
3848 // incoming area, as the Java calling convention is completely under
3849 // the control of the AD file. Doubles can be sorted and packed to
3850 // avoid holes. Holes in the outgoing arguments may be necessary for
3851 // varargs C calling conventions.
3852 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3853 // even aligned with pad0 as needed.
3854 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3855 // (the latter is true on Intel but is it false on AArch64?)
3856 // region 6-11 is even aligned; it may be padded out more so that
3857 // the region from SP to FP meets the minimum stack alignment.
3858 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3859 // alignment. Region 11, pad1, may be dynamically extended so that
3860 // SP meets the minimum alignment.
3861
3862 frame %{
3863 // These three registers define part of the calling convention
3864 // between compiled code and the interpreter.
3865
3866 // Inline Cache Register or Method for I2C.
3867 inline_cache_reg(R12);
3868
3869 // Number of stack slots consumed by locking an object
3870 sync_stack_slots(2);
3871
3872 // Compiled code's Frame Pointer
3873 frame_pointer(R31);
3874
3875 // Interpreter stores its frame pointer in a register which is
3876 // stored to the stack by I2CAdaptors.
3877 // I2CAdaptors convert from interpreted java to compiled java.
3878 interpreter_frame_pointer(R29);
3879
3880 // Stack alignment requirement
3881 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3882
3883 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3884 // for calls to C. Supports the var-args backing area for register parms.
3885 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3886
3887 // The after-PROLOG location of the return address. Location of
3888 // return address specifies a type (REG or STACK) and a number
3889 // representing the register number (i.e. - use a register name) or
3890 // stack slot.
3891 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3892 // Otherwise, it is above the locks and verification slot and alignment word
3893 // TODO this may well be correct but need to check why that - 2 is there
3894 // ppc port uses 0 but we definitely need to allow for fixed_slots
3895 // which folds in the space used for monitors
3896 return_addr(STACK - 2 +
3897 align_up((Compile::current()->in_preserve_stack_slots() +
3898 Compile::current()->fixed_slots()),
3899 stack_alignment_in_slots()));
3900
3901 // Location of compiled Java return values. Same as C for now.
3902 return_value
3903 %{
3904 // TODO do we allow ideal_reg == Op_RegN???
3905 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3906 "only return normal values");
3907
3908 static const int lo[Op_RegL + 1] = { // enum name
3909 0, // Op_Node
3910 0, // Op_Set
3911 R0_num, // Op_RegN
3912 R0_num, // Op_RegI
3913 R0_num, // Op_RegP
3914 V0_num, // Op_RegF
3915 V0_num, // Op_RegD
3916 R0_num // Op_RegL
3917 };
3918
3919 static const int hi[Op_RegL + 1] = { // enum name
3920 0, // Op_Node
3921 0, // Op_Set
3922 OptoReg::Bad, // Op_RegN
3923 OptoReg::Bad, // Op_RegI
3924 R0_H_num, // Op_RegP
3925 OptoReg::Bad, // Op_RegF
3926 V0_H_num, // Op_RegD
3927 R0_H_num // Op_RegL
3928 };
3929
3930 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3931 %}
3932 %}
3933
3934 //----------ATTRIBUTES---------------------------------------------------------
3935 //----------Operand Attributes-------------------------------------------------
3936 op_attrib op_cost(1); // Required cost attribute
3937
3938 //----------Instruction Attributes---------------------------------------------
3939 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3940 ins_attrib ins_size(32); // Required size attribute (in bits)
3941 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3942 // a non-matching short branch variant
3943 // of some long branch?
3944 ins_attrib ins_alignment(4); // Required alignment attribute (must
3945 // be a power of 2) specifies the
3946 // alignment that some part of the
3947 // instruction (not necessarily the
3948 // start) requires. If > 1, a
3949 // compute_padding() function must be
3950 // provided for the instruction
3951
3952 // Whether this node is expanded during code emission into a sequence of
3953 // instructions and the first instruction can perform an implicit null check.
3954 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3955
3956 //----------OPERANDS-----------------------------------------------------------
3957 // Operand definitions must precede instruction definitions for correct parsing
3958 // in the ADLC because operands constitute user defined types which are used in
3959 // instruction definitions.
3960
3961 //----------Simple Operands----------------------------------------------------
3962
3963 // Integer operands 32 bit
3964 // 32 bit immediate
3965 operand immI()
3966 %{
3967 match(ConI);
3968
3969 op_cost(0);
3970 format %{ %}
3971 interface(CONST_INTER);
3972 %}
3973
3974 // 32 bit zero
3975 operand immI0()
3976 %{
3977 predicate(n->get_int() == 0);
3978 match(ConI);
3979
3980 op_cost(0);
3981 format %{ %}
3982 interface(CONST_INTER);
3983 %}
3984
3985 // 32 bit unit increment
3986 operand immI_1()
3987 %{
3988 predicate(n->get_int() == 1);
3989 match(ConI);
3990
3991 op_cost(0);
3992 format %{ %}
3993 interface(CONST_INTER);
3994 %}
3995
3996 // 32 bit unit decrement
3997 operand immI_M1()
3998 %{
3999 predicate(n->get_int() == -1);
4000 match(ConI);
4001
4002 op_cost(0);
4003 format %{ %}
4004 interface(CONST_INTER);
4005 %}
4006
4007 // Shift values for add/sub extension shift
4008 operand immIExt()
4009 %{
4010 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4011 match(ConI);
4012
4013 op_cost(0);
4014 format %{ %}
4015 interface(CONST_INTER);
4016 %}
4017
4018 operand immI_gt_1()
4019 %{
4020 predicate(n->get_int() > 1);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 operand immI_le_4()
4029 %{
4030 predicate(n->get_int() <= 4);
4031 match(ConI);
4032
4033 op_cost(0);
4034 format %{ %}
4035 interface(CONST_INTER);
4036 %}
4037
4038 operand immI_16()
4039 %{
4040 predicate(n->get_int() == 16);
4041 match(ConI);
4042
4043 op_cost(0);
4044 format %{ %}
4045 interface(CONST_INTER);
4046 %}
4047
4048 operand immI_24()
4049 %{
4050 predicate(n->get_int() == 24);
4051 match(ConI);
4052
4053 op_cost(0);
4054 format %{ %}
4055 interface(CONST_INTER);
4056 %}
4057
4058 operand immI_32()
4059 %{
4060 predicate(n->get_int() == 32);
4061 match(ConI);
4062
4063 op_cost(0);
4064 format %{ %}
4065 interface(CONST_INTER);
4066 %}
4067
4068 operand immI_48()
4069 %{
4070 predicate(n->get_int() == 48);
4071 match(ConI);
4072
4073 op_cost(0);
4074 format %{ %}
4075 interface(CONST_INTER);
4076 %}
4077
4078 operand immI_56()
4079 %{
4080 predicate(n->get_int() == 56);
4081 match(ConI);
4082
4083 op_cost(0);
4084 format %{ %}
4085 interface(CONST_INTER);
4086 %}
4087
4088 operand immI_255()
4089 %{
4090 predicate(n->get_int() == 255);
4091 match(ConI);
4092
4093 op_cost(0);
4094 format %{ %}
4095 interface(CONST_INTER);
4096 %}
4097
4098 operand immI_65535()
4099 %{
4100 predicate(n->get_int() == 65535);
4101 match(ConI);
4102
4103 op_cost(0);
4104 format %{ %}
4105 interface(CONST_INTER);
4106 %}
4107
4108 operand immI_positive()
4109 %{
4110 predicate(n->get_int() > 0);
4111 match(ConI);
4112
4113 op_cost(0);
4114 format %{ %}
4115 interface(CONST_INTER);
4116 %}
4117
4118 // BoolTest condition for signed compare
4119 operand immI_cmp_cond()
4120 %{
4121 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4122 match(ConI);
4123
4124 op_cost(0);
4125 format %{ %}
4126 interface(CONST_INTER);
4127 %}
4128
4129 // BoolTest condition for unsigned compare
4130 operand immI_cmpU_cond()
4131 %{
4132 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4133 match(ConI);
4134
4135 op_cost(0);
4136 format %{ %}
4137 interface(CONST_INTER);
4138 %}
4139
4140 operand immL_255()
4141 %{
4142 predicate(n->get_long() == 255L);
4143 match(ConL);
4144
4145 op_cost(0);
4146 format %{ %}
4147 interface(CONST_INTER);
4148 %}
4149
4150 operand immL_65535()
4151 %{
4152 predicate(n->get_long() == 65535L);
4153 match(ConL);
4154
4155 op_cost(0);
4156 format %{ %}
4157 interface(CONST_INTER);
4158 %}
4159
4160 operand immL_4294967295()
4161 %{
4162 predicate(n->get_long() == 4294967295L);
4163 match(ConL);
4164
4165 op_cost(0);
4166 format %{ %}
4167 interface(CONST_INTER);
4168 %}
4169
4170 operand immL_bitmask()
4171 %{
4172 predicate((n->get_long() != 0)
4173 && ((n->get_long() & 0xc000000000000000l) == 0)
4174 && is_power_of_2(n->get_long() + 1));
4175 match(ConL);
4176
4177 op_cost(0);
4178 format %{ %}
4179 interface(CONST_INTER);
4180 %}
4181
4182 operand immI_bitmask()
4183 %{
4184 predicate((n->get_int() != 0)
4185 && ((n->get_int() & 0xc0000000) == 0)
4186 && is_power_of_2(n->get_int() + 1));
4187 match(ConI);
4188
4189 op_cost(0);
4190 format %{ %}
4191 interface(CONST_INTER);
4192 %}
4193
4194 operand immL_positive_bitmaskI()
4195 %{
4196 predicate((n->get_long() != 0)
4197 && ((julong)n->get_long() < 0x80000000ULL)
4198 && is_power_of_2(n->get_long() + 1));
4199 match(ConL);
4200
4201 op_cost(0);
4202 format %{ %}
4203 interface(CONST_INTER);
4204 %}
4205
4206 // Scale values for scaled offset addressing modes (up to long but not quad)
4207 operand immIScale()
4208 %{
4209 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4210 match(ConI);
4211
4212 op_cost(0);
4213 format %{ %}
4214 interface(CONST_INTER);
4215 %}
4216
4217 // 5 bit signed integer
4218 operand immI5()
4219 %{
4220 predicate(Assembler::is_simm(n->get_int(), 5));
4221 match(ConI);
4222
4223 op_cost(0);
4224 format %{ %}
4225 interface(CONST_INTER);
4226 %}
4227
4228 // 7 bit unsigned integer
4229 operand immIU7()
4230 %{
4231 predicate(Assembler::is_uimm(n->get_int(), 7));
4232 match(ConI);
4233
4234 op_cost(0);
4235 format %{ %}
4236 interface(CONST_INTER);
4237 %}
4238
4239 // Offset for scaled or unscaled immediate loads and stores
4240 operand immIOffset()
4241 %{
4242 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4243 match(ConI);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immIOffset1()
4251 %{
4252 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4253 match(ConI);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immIOffset2()
4261 %{
4262 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4263 match(ConI);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immIOffset4()
4271 %{
4272 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4273 match(ConI);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immIOffset8()
4281 %{
4282 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4283 match(ConI);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 operand immIOffset16()
4291 %{
4292 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4293 match(ConI);
4294
4295 op_cost(0);
4296 format %{ %}
4297 interface(CONST_INTER);
4298 %}
4299
4300 operand immLOffset()
4301 %{
4302 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4303 match(ConL);
4304
4305 op_cost(0);
4306 format %{ %}
4307 interface(CONST_INTER);
4308 %}
4309
4310 operand immLoffset1()
4311 %{
4312 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4313 match(ConL);
4314
4315 op_cost(0);
4316 format %{ %}
4317 interface(CONST_INTER);
4318 %}
4319
4320 operand immLoffset2()
4321 %{
4322 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4323 match(ConL);
4324
4325 op_cost(0);
4326 format %{ %}
4327 interface(CONST_INTER);
4328 %}
4329
4330 operand immLoffset4()
4331 %{
4332 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4333 match(ConL);
4334
4335 op_cost(0);
4336 format %{ %}
4337 interface(CONST_INTER);
4338 %}
4339
4340 operand immLoffset8()
4341 %{
4342 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4343 match(ConL);
4344
4345 op_cost(0);
4346 format %{ %}
4347 interface(CONST_INTER);
4348 %}
4349
4350 operand immLoffset16()
4351 %{
4352 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4353 match(ConL);
4354
4355 op_cost(0);
4356 format %{ %}
4357 interface(CONST_INTER);
4358 %}
4359
4360 // 5 bit signed long integer
4361 operand immL5()
4362 %{
4363 predicate(Assembler::is_simm(n->get_long(), 5));
4364 match(ConL);
4365
4366 op_cost(0);
4367 format %{ %}
4368 interface(CONST_INTER);
4369 %}
4370
4371 // 7 bit unsigned long integer
4372 operand immLU7()
4373 %{
4374 predicate(Assembler::is_uimm(n->get_long(), 7));
4375 match(ConL);
4376
4377 op_cost(0);
4378 format %{ %}
4379 interface(CONST_INTER);
4380 %}
4381
4382 // 8 bit signed value.
4383 operand immI8()
4384 %{
4385 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4386 match(ConI);
4387
4388 op_cost(0);
4389 format %{ %}
4390 interface(CONST_INTER);
4391 %}
4392
4393 // 8 bit signed value (simm8), or #simm8 LSL 8.
4394 operand immIDupV()
4395 %{
4396 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4397 match(ConI);
4398
4399 op_cost(0);
4400 format %{ %}
4401 interface(CONST_INTER);
4402 %}
4403
4404 // 8 bit signed value (simm8), or #simm8 LSL 8.
4405 operand immLDupV()
4406 %{
4407 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4408 match(ConL);
4409
4410 op_cost(0);
4411 format %{ %}
4412 interface(CONST_INTER);
4413 %}
4414
4415 // 8 bit signed value (simm8), or #simm8 LSL 8.
4416 operand immHDupV()
4417 %{
4418 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4419 match(ConH);
4420
4421 op_cost(0);
4422 format %{ %}
4423 interface(CONST_INTER);
4424 %}
4425
4426 // 8 bit integer valid for vector add sub immediate
4427 operand immBAddSubV()
4428 %{
4429 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4430 match(ConI);
4431
4432 op_cost(0);
4433 format %{ %}
4434 interface(CONST_INTER);
4435 %}
4436
4437 // 32 bit integer valid for add sub immediate
4438 operand immIAddSub()
4439 %{
4440 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4441 match(ConI);
4442 op_cost(0);
4443 format %{ %}
4444 interface(CONST_INTER);
4445 %}
4446
4447 // 32 bit integer valid for vector add sub immediate
4448 operand immIAddSubV()
4449 %{
4450 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4451 match(ConI);
4452
4453 op_cost(0);
4454 format %{ %}
4455 interface(CONST_INTER);
4456 %}
4457
4458 // 32 bit unsigned integer valid for logical immediate
4459
4460 operand immBLog()
4461 %{
4462 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4463 match(ConI);
4464
4465 op_cost(0);
4466 format %{ %}
4467 interface(CONST_INTER);
4468 %}
4469
4470 operand immSLog()
4471 %{
4472 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4473 match(ConI);
4474
4475 op_cost(0);
4476 format %{ %}
4477 interface(CONST_INTER);
4478 %}
4479
4480 operand immILog()
4481 %{
4482 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4483 match(ConI);
4484
4485 op_cost(0);
4486 format %{ %}
4487 interface(CONST_INTER);
4488 %}
4489
4490 // Integer operands 64 bit
4491 // 64 bit immediate
4492 operand immL()
4493 %{
4494 match(ConL);
4495
4496 op_cost(0);
4497 format %{ %}
4498 interface(CONST_INTER);
4499 %}
4500
4501 // 64 bit zero
4502 operand immL0()
4503 %{
4504 predicate(n->get_long() == 0);
4505 match(ConL);
4506
4507 op_cost(0);
4508 format %{ %}
4509 interface(CONST_INTER);
4510 %}
4511
4512 // 64 bit unit decrement
4513 operand immL_M1()
4514 %{
4515 predicate(n->get_long() == -1);
4516 match(ConL);
4517
4518 op_cost(0);
4519 format %{ %}
4520 interface(CONST_INTER);
4521 %}
4522
4523 // 64 bit integer valid for add sub immediate
4524 operand immLAddSub()
4525 %{
4526 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4527 match(ConL);
4528 op_cost(0);
4529 format %{ %}
4530 interface(CONST_INTER);
4531 %}
4532
4533 // 64 bit integer valid for addv subv immediate
4534 operand immLAddSubV()
4535 %{
4536 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4537 match(ConL);
4538
4539 op_cost(0);
4540 format %{ %}
4541 interface(CONST_INTER);
4542 %}
4543
4544 // 64 bit integer valid for logical immediate
4545 operand immLLog()
4546 %{
4547 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4548 match(ConL);
4549 op_cost(0);
4550 format %{ %}
4551 interface(CONST_INTER);
4552 %}
4553
4554 // Long Immediate: low 32-bit mask
4555 operand immL_32bits()
4556 %{
4557 predicate(n->get_long() == 0xFFFFFFFFL);
4558 match(ConL);
4559 op_cost(0);
4560 format %{ %}
4561 interface(CONST_INTER);
4562 %}
4563
4564 // Pointer operands
4565 // Pointer Immediate
4566 operand immP()
4567 %{
4568 match(ConP);
4569
4570 op_cost(0);
4571 format %{ %}
4572 interface(CONST_INTER);
4573 %}
4574
4575 // nullptr Pointer Immediate
4576 operand immP0()
4577 %{
4578 predicate(n->get_ptr() == 0);
4579 match(ConP);
4580
4581 op_cost(0);
4582 format %{ %}
4583 interface(CONST_INTER);
4584 %}
4585
4586 // Pointer Immediate One
4587 // this is used in object initialization (initial object header)
4588 operand immP_1()
4589 %{
4590 predicate(n->get_ptr() == 1);
4591 match(ConP);
4592
4593 op_cost(0);
4594 format %{ %}
4595 interface(CONST_INTER);
4596 %}
4597
4598 // Float and Double operands
4599 // Double Immediate
4600 operand immD()
4601 %{
4602 match(ConD);
4603 op_cost(0);
4604 format %{ %}
4605 interface(CONST_INTER);
4606 %}
4607
4608 // Double Immediate: +0.0d
4609 operand immD0()
4610 %{
4611 predicate(jlong_cast(n->getd()) == 0);
4612 match(ConD);
4613
4614 op_cost(0);
4615 format %{ %}
4616 interface(CONST_INTER);
4617 %}
4618
4619 // constant 'double +0.0'.
4620 operand immDPacked()
4621 %{
4622 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4623 match(ConD);
4624 op_cost(0);
4625 format %{ %}
4626 interface(CONST_INTER);
4627 %}
4628
4629 // Float Immediate
4630 operand immF()
4631 %{
4632 match(ConF);
4633 op_cost(0);
4634 format %{ %}
4635 interface(CONST_INTER);
4636 %}
4637
4638 // Float Immediate: +0.0f.
4639 operand immF0()
4640 %{
4641 predicate(jint_cast(n->getf()) == 0);
4642 match(ConF);
4643
4644 op_cost(0);
4645 format %{ %}
4646 interface(CONST_INTER);
4647 %}
4648
4649 // Half Float (FP16) Immediate
4650 operand immH()
4651 %{
4652 match(ConH);
4653 op_cost(0);
4654 format %{ %}
4655 interface(CONST_INTER);
4656 %}
4657
4658 //
4659 operand immFPacked()
4660 %{
4661 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4662 match(ConF);
4663 op_cost(0);
4664 format %{ %}
4665 interface(CONST_INTER);
4666 %}
4667
4668 // Narrow pointer operands
4669 // Narrow Pointer Immediate
4670 operand immN()
4671 %{
4672 match(ConN);
4673
4674 op_cost(0);
4675 format %{ %}
4676 interface(CONST_INTER);
4677 %}
4678
4679 // Narrow nullptr Pointer Immediate
4680 operand immN0()
4681 %{
4682 predicate(n->get_narrowcon() == 0);
4683 match(ConN);
4684
4685 op_cost(0);
4686 format %{ %}
4687 interface(CONST_INTER);
4688 %}
4689
4690 operand immNKlass()
4691 %{
4692 match(ConNKlass);
4693
4694 op_cost(0);
4695 format %{ %}
4696 interface(CONST_INTER);
4697 %}
4698
4699 // Integer 32 bit Register Operands
4700 // Integer 32 bitRegister (excludes SP)
4701 operand iRegI()
4702 %{
4703 constraint(ALLOC_IN_RC(any_reg32));
4704 match(RegI);
4705 match(iRegINoSp);
4706 op_cost(0);
4707 format %{ %}
4708 interface(REG_INTER);
4709 %}
4710
4711 // Integer 32 bit Register not Special
4712 operand iRegINoSp()
4713 %{
4714 constraint(ALLOC_IN_RC(no_special_reg32));
4715 match(RegI);
4716 op_cost(0);
4717 format %{ %}
4718 interface(REG_INTER);
4719 %}
4720
4721 // Integer 64 bit Register Operands
4722 // Integer 64 bit Register (includes SP)
4723 operand iRegL()
4724 %{
4725 constraint(ALLOC_IN_RC(any_reg));
4726 match(RegL);
4727 match(iRegLNoSp);
4728 op_cost(0);
4729 format %{ %}
4730 interface(REG_INTER);
4731 %}
4732
4733 // Integer 64 bit Register not Special
4734 operand iRegLNoSp()
4735 %{
4736 constraint(ALLOC_IN_RC(no_special_reg));
4737 match(RegL);
4738 match(iRegL_R0);
4739 format %{ %}
4740 interface(REG_INTER);
4741 %}
4742
4743 // Pointer Register Operands
4744 // Pointer Register
4745 operand iRegP()
4746 %{
4747 constraint(ALLOC_IN_RC(ptr_reg));
4748 match(RegP);
4749 match(iRegPNoSp);
4750 match(iRegP_R0);
4751 //match(iRegP_R2);
4752 //match(iRegP_R4);
4753 match(iRegP_R5);
4754 match(thread_RegP);
4755 op_cost(0);
4756 format %{ %}
4757 interface(REG_INTER);
4758 %}
4759
4760 // Pointer 64 bit Register not Special
4761 operand iRegPNoSp()
4762 %{
4763 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4764 match(RegP);
4765 // match(iRegP);
4766 // match(iRegP_R0);
4767 // match(iRegP_R2);
4768 // match(iRegP_R4);
4769 // match(iRegP_R5);
4770 // match(thread_RegP);
4771 op_cost(0);
4772 format %{ %}
4773 interface(REG_INTER);
4774 %}
4775
4776 // This operand is not allowed to use rfp even if
4777 // rfp is not used to hold the frame pointer.
4778 operand iRegPNoSpNoRfp()
4779 %{
4780 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4781 match(RegP);
4782 match(iRegPNoSp);
4783 op_cost(0);
4784 format %{ %}
4785 interface(REG_INTER);
4786 %}
4787
4788 // Pointer 64 bit Register R0 only
4789 operand iRegP_R0()
4790 %{
4791 constraint(ALLOC_IN_RC(r0_reg));
4792 match(RegP);
4793 // match(iRegP);
4794 match(iRegPNoSp);
4795 op_cost(0);
4796 format %{ %}
4797 interface(REG_INTER);
4798 %}
4799
4800 // Pointer 64 bit Register R1 only
4801 operand iRegP_R1()
4802 %{
4803 constraint(ALLOC_IN_RC(r1_reg));
4804 match(RegP);
4805 // match(iRegP);
4806 match(iRegPNoSp);
4807 op_cost(0);
4808 format %{ %}
4809 interface(REG_INTER);
4810 %}
4811
4812 // Pointer 64 bit Register R2 only
4813 operand iRegP_R2()
4814 %{
4815 constraint(ALLOC_IN_RC(r2_reg));
4816 match(RegP);
4817 // match(iRegP);
4818 match(iRegPNoSp);
4819 op_cost(0);
4820 format %{ %}
4821 interface(REG_INTER);
4822 %}
4823
4824 // Pointer 64 bit Register R3 only
4825 operand iRegP_R3()
4826 %{
4827 constraint(ALLOC_IN_RC(r3_reg));
4828 match(RegP);
4829 // match(iRegP);
4830 match(iRegPNoSp);
4831 op_cost(0);
4832 format %{ %}
4833 interface(REG_INTER);
4834 %}
4835
4836 // Pointer 64 bit Register R4 only
4837 operand iRegP_R4()
4838 %{
4839 constraint(ALLOC_IN_RC(r4_reg));
4840 match(RegP);
4841 // match(iRegP);
4842 match(iRegPNoSp);
4843 op_cost(0);
4844 format %{ %}
4845 interface(REG_INTER);
4846 %}
4847
4848 // Pointer 64 bit Register R5 only
4849 operand iRegP_R5()
4850 %{
4851 constraint(ALLOC_IN_RC(r5_reg));
4852 match(RegP);
4853 // match(iRegP);
4854 match(iRegPNoSp);
4855 op_cost(0);
4856 format %{ %}
4857 interface(REG_INTER);
4858 %}
4859
4860 // Pointer 64 bit Register R10 only
4861 operand iRegP_R10()
4862 %{
4863 constraint(ALLOC_IN_RC(r10_reg));
4864 match(RegP);
4865 // match(iRegP);
4866 match(iRegPNoSp);
4867 op_cost(0);
4868 format %{ %}
4869 interface(REG_INTER);
4870 %}
4871
4872 // Long 64 bit Register R0 only
4873 operand iRegL_R0()
4874 %{
4875 constraint(ALLOC_IN_RC(r0_reg));
4876 match(RegL);
4877 match(iRegLNoSp);
4878 op_cost(0);
4879 format %{ %}
4880 interface(REG_INTER);
4881 %}
4882
4883 // Long 64 bit Register R11 only
4884 operand iRegL_R11()
4885 %{
4886 constraint(ALLOC_IN_RC(r11_reg));
4887 match(RegL);
4888 match(iRegLNoSp);
4889 op_cost(0);
4890 format %{ %}
4891 interface(REG_INTER);
4892 %}
4893
4894 // Register R0 only
4895 operand iRegI_R0()
4896 %{
4897 constraint(ALLOC_IN_RC(int_r0_reg));
4898 match(RegI);
4899 match(iRegINoSp);
4900 op_cost(0);
4901 format %{ %}
4902 interface(REG_INTER);
4903 %}
4904
4905 // Register R2 only
4906 operand iRegI_R2()
4907 %{
4908 constraint(ALLOC_IN_RC(int_r2_reg));
4909 match(RegI);
4910 match(iRegINoSp);
4911 op_cost(0);
4912 format %{ %}
4913 interface(REG_INTER);
4914 %}
4915
4916 // Register R3 only
4917 operand iRegI_R3()
4918 %{
4919 constraint(ALLOC_IN_RC(int_r3_reg));
4920 match(RegI);
4921 match(iRegINoSp);
4922 op_cost(0);
4923 format %{ %}
4924 interface(REG_INTER);
4925 %}
4926
4927
4928 // Register R4 only
4929 operand iRegI_R4()
4930 %{
4931 constraint(ALLOC_IN_RC(int_r4_reg));
4932 match(RegI);
4933 match(iRegINoSp);
4934 op_cost(0);
4935 format %{ %}
4936 interface(REG_INTER);
4937 %}
4938
4939
4940 // Pointer Register Operands
4941 // Narrow Pointer Register
4942 operand iRegN()
4943 %{
4944 constraint(ALLOC_IN_RC(any_reg32));
4945 match(RegN);
4946 match(iRegNNoSp);
4947 op_cost(0);
4948 format %{ %}
4949 interface(REG_INTER);
4950 %}
4951
4952 // Integer 64 bit Register not Special
4953 operand iRegNNoSp()
4954 %{
4955 constraint(ALLOC_IN_RC(no_special_reg32));
4956 match(RegN);
4957 op_cost(0);
4958 format %{ %}
4959 interface(REG_INTER);
4960 %}
4961
4962 // Float Register
4963 // Float register operands
4964 operand vRegF()
4965 %{
4966 constraint(ALLOC_IN_RC(float_reg));
4967 match(RegF);
4968
4969 op_cost(0);
4970 format %{ %}
4971 interface(REG_INTER);
4972 %}
4973
4974 // Double Register
4975 // Double register operands
4976 operand vRegD()
4977 %{
4978 constraint(ALLOC_IN_RC(double_reg));
4979 match(RegD);
4980
4981 op_cost(0);
4982 format %{ %}
4983 interface(REG_INTER);
4984 %}
4985
4986 // Generic vector class. This will be used for
4987 // all vector operands, including NEON and SVE.
4988 operand vReg()
4989 %{
4990 constraint(ALLOC_IN_RC(dynamic));
4991 match(VecA);
4992 match(VecD);
4993 match(VecX);
4994
4995 op_cost(0);
4996 format %{ %}
4997 interface(REG_INTER);
4998 %}
4999
5000 operand vReg_V10()
5001 %{
5002 constraint(ALLOC_IN_RC(v10_veca_reg));
5003 match(vReg);
5004
5005 op_cost(0);
5006 format %{ %}
5007 interface(REG_INTER);
5008 %}
5009
5010 operand vReg_V11()
5011 %{
5012 constraint(ALLOC_IN_RC(v11_veca_reg));
5013 match(vReg);
5014
5015 op_cost(0);
5016 format %{ %}
5017 interface(REG_INTER);
5018 %}
5019
5020 operand vReg_V12()
5021 %{
5022 constraint(ALLOC_IN_RC(v12_veca_reg));
5023 match(vReg);
5024
5025 op_cost(0);
5026 format %{ %}
5027 interface(REG_INTER);
5028 %}
5029
5030 operand vReg_V13()
5031 %{
5032 constraint(ALLOC_IN_RC(v13_veca_reg));
5033 match(vReg);
5034
5035 op_cost(0);
5036 format %{ %}
5037 interface(REG_INTER);
5038 %}
5039
5040 operand vReg_V17()
5041 %{
5042 constraint(ALLOC_IN_RC(v17_veca_reg));
5043 match(vReg);
5044
5045 op_cost(0);
5046 format %{ %}
5047 interface(REG_INTER);
5048 %}
5049
5050 operand vReg_V18()
5051 %{
5052 constraint(ALLOC_IN_RC(v18_veca_reg));
5053 match(vReg);
5054
5055 op_cost(0);
5056 format %{ %}
5057 interface(REG_INTER);
5058 %}
5059
5060 operand vReg_V23()
5061 %{
5062 constraint(ALLOC_IN_RC(v23_veca_reg));
5063 match(vReg);
5064
5065 op_cost(0);
5066 format %{ %}
5067 interface(REG_INTER);
5068 %}
5069
5070 operand vReg_V24()
5071 %{
5072 constraint(ALLOC_IN_RC(v24_veca_reg));
5073 match(vReg);
5074
5075 op_cost(0);
5076 format %{ %}
5077 interface(REG_INTER);
5078 %}
5079
5080 operand vecA()
5081 %{
5082 constraint(ALLOC_IN_RC(vectora_reg));
5083 match(VecA);
5084
5085 op_cost(0);
5086 format %{ %}
5087 interface(REG_INTER);
5088 %}
5089
5090 operand vecD()
5091 %{
5092 constraint(ALLOC_IN_RC(vectord_reg));
5093 match(VecD);
5094
5095 op_cost(0);
5096 format %{ %}
5097 interface(REG_INTER);
5098 %}
5099
5100 operand vecX()
5101 %{
5102 constraint(ALLOC_IN_RC(vectorx_reg));
5103 match(VecX);
5104
5105 op_cost(0);
5106 format %{ %}
5107 interface(REG_INTER);
5108 %}
5109
5110 operand vRegD_V0()
5111 %{
5112 constraint(ALLOC_IN_RC(v0_reg));
5113 match(RegD);
5114 op_cost(0);
5115 format %{ %}
5116 interface(REG_INTER);
5117 %}
5118
5119 operand vRegD_V1()
5120 %{
5121 constraint(ALLOC_IN_RC(v1_reg));
5122 match(RegD);
5123 op_cost(0);
5124 format %{ %}
5125 interface(REG_INTER);
5126 %}
5127
5128 operand vRegD_V2()
5129 %{
5130 constraint(ALLOC_IN_RC(v2_reg));
5131 match(RegD);
5132 op_cost(0);
5133 format %{ %}
5134 interface(REG_INTER);
5135 %}
5136
5137 operand vRegD_V3()
5138 %{
5139 constraint(ALLOC_IN_RC(v3_reg));
5140 match(RegD);
5141 op_cost(0);
5142 format %{ %}
5143 interface(REG_INTER);
5144 %}
5145
5146 operand vRegD_V4()
5147 %{
5148 constraint(ALLOC_IN_RC(v4_reg));
5149 match(RegD);
5150 op_cost(0);
5151 format %{ %}
5152 interface(REG_INTER);
5153 %}
5154
5155 operand vRegD_V5()
5156 %{
5157 constraint(ALLOC_IN_RC(v5_reg));
5158 match(RegD);
5159 op_cost(0);
5160 format %{ %}
5161 interface(REG_INTER);
5162 %}
5163
5164 operand vRegD_V6()
5165 %{
5166 constraint(ALLOC_IN_RC(v6_reg));
5167 match(RegD);
5168 op_cost(0);
5169 format %{ %}
5170 interface(REG_INTER);
5171 %}
5172
5173 operand vRegD_V7()
5174 %{
5175 constraint(ALLOC_IN_RC(v7_reg));
5176 match(RegD);
5177 op_cost(0);
5178 format %{ %}
5179 interface(REG_INTER);
5180 %}
5181
5182 operand vRegD_V12()
5183 %{
5184 constraint(ALLOC_IN_RC(v12_reg));
5185 match(RegD);
5186 op_cost(0);
5187 format %{ %}
5188 interface(REG_INTER);
5189 %}
5190
5191 operand vRegD_V13()
5192 %{
5193 constraint(ALLOC_IN_RC(v13_reg));
5194 match(RegD);
5195 op_cost(0);
5196 format %{ %}
5197 interface(REG_INTER);
5198 %}
5199
5200 operand pReg()
5201 %{
5202 constraint(ALLOC_IN_RC(pr_reg));
5203 match(RegVectMask);
5204 match(pRegGov);
5205 op_cost(0);
5206 format %{ %}
5207 interface(REG_INTER);
5208 %}
5209
5210 operand pRegGov()
5211 %{
5212 constraint(ALLOC_IN_RC(gov_pr));
5213 match(RegVectMask);
5214 match(pReg);
5215 op_cost(0);
5216 format %{ %}
5217 interface(REG_INTER);
5218 %}
5219
5220 operand pRegGov_P0()
5221 %{
5222 constraint(ALLOC_IN_RC(p0_reg));
5223 match(RegVectMask);
5224 op_cost(0);
5225 format %{ %}
5226 interface(REG_INTER);
5227 %}
5228
5229 operand pRegGov_P1()
5230 %{
5231 constraint(ALLOC_IN_RC(p1_reg));
5232 match(RegVectMask);
5233 op_cost(0);
5234 format %{ %}
5235 interface(REG_INTER);
5236 %}
5237
5238 // Flags register, used as output of signed compare instructions
5239
5240 // note that on AArch64 we also use this register as the output for
5241 // for floating point compare instructions (CmpF CmpD). this ensures
5242 // that ordered inequality tests use GT, GE, LT or LE none of which
5243 // pass through cases where the result is unordered i.e. one or both
5244 // inputs to the compare is a NaN. this means that the ideal code can
5245 // replace e.g. a GT with an LE and not end up capturing the NaN case
5246 // (where the comparison should always fail). EQ and NE tests are
5247 // always generated in ideal code so that unordered folds into the NE
5248 // case, matching the behaviour of AArch64 NE.
5249 //
5250 // This differs from x86 where the outputs of FP compares use a
5251 // special FP flags registers and where compares based on this
5252 // register are distinguished into ordered inequalities (cmpOpUCF) and
5253 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5254 // to explicitly handle the unordered case in branches. x86 also has
5255 // to include extra CMoveX rules to accept a cmpOpUCF input.
5256
5257 operand rFlagsReg()
5258 %{
5259 constraint(ALLOC_IN_RC(int_flags));
5260 match(RegFlags);
5261
5262 op_cost(0);
5263 format %{ "RFLAGS" %}
5264 interface(REG_INTER);
5265 %}
5266
5267 // Flags register, used as output of unsigned compare instructions
5268 operand rFlagsRegU()
5269 %{
5270 constraint(ALLOC_IN_RC(int_flags));
5271 match(RegFlags);
5272
5273 op_cost(0);
5274 format %{ "RFLAGSU" %}
5275 interface(REG_INTER);
5276 %}
5277
5278 // Special Registers
5279
5280 // Method Register
5281 operand inline_cache_RegP(iRegP reg)
5282 %{
5283 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5284 match(reg);
5285 match(iRegPNoSp);
5286 op_cost(0);
5287 format %{ %}
5288 interface(REG_INTER);
5289 %}
5290
5291 // Thread Register
5292 operand thread_RegP(iRegP reg)
5293 %{
5294 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5295 match(reg);
5296 op_cost(0);
5297 format %{ %}
5298 interface(REG_INTER);
5299 %}
5300
5301 //----------Memory Operands----------------------------------------------------
5302
5303 operand indirect(iRegP reg)
5304 %{
5305 constraint(ALLOC_IN_RC(ptr_reg));
5306 match(reg);
5307 op_cost(0);
5308 format %{ "[$reg]" %}
5309 interface(MEMORY_INTER) %{
5310 base($reg);
5311 index(0xffffffff);
5312 scale(0x0);
5313 disp(0x0);
5314 %}
5315 %}
5316
5317 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5318 %{
5319 constraint(ALLOC_IN_RC(ptr_reg));
5320 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5321 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5322 op_cost(0);
5323 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5324 interface(MEMORY_INTER) %{
5325 base($reg);
5326 index($ireg);
5327 scale($scale);
5328 disp(0x0);
5329 %}
5330 %}
5331
5332 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5333 %{
5334 constraint(ALLOC_IN_RC(ptr_reg));
5335 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5336 match(AddP reg (LShiftL lreg scale));
5337 op_cost(0);
5338 format %{ "$reg, $lreg lsl($scale)" %}
5339 interface(MEMORY_INTER) %{
5340 base($reg);
5341 index($lreg);
5342 scale($scale);
5343 disp(0x0);
5344 %}
5345 %}
5346
5347 operand indIndexI2L(iRegP reg, iRegI ireg)
5348 %{
5349 constraint(ALLOC_IN_RC(ptr_reg));
5350 match(AddP reg (ConvI2L ireg));
5351 op_cost(0);
5352 format %{ "$reg, $ireg, 0, I2L" %}
5353 interface(MEMORY_INTER) %{
5354 base($reg);
5355 index($ireg);
5356 scale(0x0);
5357 disp(0x0);
5358 %}
5359 %}
5360
5361 operand indIndex(iRegP reg, iRegL lreg)
5362 %{
5363 constraint(ALLOC_IN_RC(ptr_reg));
5364 match(AddP reg lreg);
5365 op_cost(0);
5366 format %{ "$reg, $lreg" %}
5367 interface(MEMORY_INTER) %{
5368 base($reg);
5369 index($lreg);
5370 scale(0x0);
5371 disp(0x0);
5372 %}
5373 %}
5374
5375 operand indOffI1(iRegP reg, immIOffset1 off)
5376 %{
5377 constraint(ALLOC_IN_RC(ptr_reg));
5378 match(AddP reg off);
5379 op_cost(0);
5380 format %{ "[$reg, $off]" %}
5381 interface(MEMORY_INTER) %{
5382 base($reg);
5383 index(0xffffffff);
5384 scale(0x0);
5385 disp($off);
5386 %}
5387 %}
5388
5389 operand indOffI2(iRegP reg, immIOffset2 off)
5390 %{
5391 constraint(ALLOC_IN_RC(ptr_reg));
5392 match(AddP reg off);
5393 op_cost(0);
5394 format %{ "[$reg, $off]" %}
5395 interface(MEMORY_INTER) %{
5396 base($reg);
5397 index(0xffffffff);
5398 scale(0x0);
5399 disp($off);
5400 %}
5401 %}
5402
5403 operand indOffI4(iRegP reg, immIOffset4 off)
5404 %{
5405 constraint(ALLOC_IN_RC(ptr_reg));
5406 match(AddP reg off);
5407 op_cost(0);
5408 format %{ "[$reg, $off]" %}
5409 interface(MEMORY_INTER) %{
5410 base($reg);
5411 index(0xffffffff);
5412 scale(0x0);
5413 disp($off);
5414 %}
5415 %}
5416
5417 operand indOffI8(iRegP reg, immIOffset8 off)
5418 %{
5419 constraint(ALLOC_IN_RC(ptr_reg));
5420 match(AddP reg off);
5421 op_cost(0);
5422 format %{ "[$reg, $off]" %}
5423 interface(MEMORY_INTER) %{
5424 base($reg);
5425 index(0xffffffff);
5426 scale(0x0);
5427 disp($off);
5428 %}
5429 %}
5430
5431 operand indOffI16(iRegP reg, immIOffset16 off)
5432 %{
5433 constraint(ALLOC_IN_RC(ptr_reg));
5434 match(AddP reg off);
5435 op_cost(0);
5436 format %{ "[$reg, $off]" %}
5437 interface(MEMORY_INTER) %{
5438 base($reg);
5439 index(0xffffffff);
5440 scale(0x0);
5441 disp($off);
5442 %}
5443 %}
5444
5445 operand indOffL1(iRegP reg, immLoffset1 off)
5446 %{
5447 constraint(ALLOC_IN_RC(ptr_reg));
5448 match(AddP reg off);
5449 op_cost(0);
5450 format %{ "[$reg, $off]" %}
5451 interface(MEMORY_INTER) %{
5452 base($reg);
5453 index(0xffffffff);
5454 scale(0x0);
5455 disp($off);
5456 %}
5457 %}
5458
5459 operand indOffL2(iRegP reg, immLoffset2 off)
5460 %{
5461 constraint(ALLOC_IN_RC(ptr_reg));
5462 match(AddP reg off);
5463 op_cost(0);
5464 format %{ "[$reg, $off]" %}
5465 interface(MEMORY_INTER) %{
5466 base($reg);
5467 index(0xffffffff);
5468 scale(0x0);
5469 disp($off);
5470 %}
5471 %}
5472
5473 operand indOffL4(iRegP reg, immLoffset4 off)
5474 %{
5475 constraint(ALLOC_IN_RC(ptr_reg));
5476 match(AddP reg off);
5477 op_cost(0);
5478 format %{ "[$reg, $off]" %}
5479 interface(MEMORY_INTER) %{
5480 base($reg);
5481 index(0xffffffff);
5482 scale(0x0);
5483 disp($off);
5484 %}
5485 %}
5486
5487 operand indOffL8(iRegP reg, immLoffset8 off)
5488 %{
5489 constraint(ALLOC_IN_RC(ptr_reg));
5490 match(AddP reg off);
5491 op_cost(0);
5492 format %{ "[$reg, $off]" %}
5493 interface(MEMORY_INTER) %{
5494 base($reg);
5495 index(0xffffffff);
5496 scale(0x0);
5497 disp($off);
5498 %}
5499 %}
5500
5501 operand indOffL16(iRegP reg, immLoffset16 off)
5502 %{
5503 constraint(ALLOC_IN_RC(ptr_reg));
5504 match(AddP reg off);
5505 op_cost(0);
5506 format %{ "[$reg, $off]" %}
5507 interface(MEMORY_INTER) %{
5508 base($reg);
5509 index(0xffffffff);
5510 scale(0x0);
5511 disp($off);
5512 %}
5513 %}
5514
5515 operand indirectX2P(iRegL reg)
5516 %{
5517 constraint(ALLOC_IN_RC(ptr_reg));
5518 match(CastX2P reg);
5519 op_cost(0);
5520 format %{ "[$reg]\t# long -> ptr" %}
5521 interface(MEMORY_INTER) %{
5522 base($reg);
5523 index(0xffffffff);
5524 scale(0x0);
5525 disp(0x0);
5526 %}
5527 %}
5528
5529 operand indOffX2P(iRegL reg, immLOffset off)
5530 %{
5531 constraint(ALLOC_IN_RC(ptr_reg));
5532 match(AddP (CastX2P reg) off);
5533 op_cost(0);
5534 format %{ "[$reg, $off]\t# long -> ptr" %}
5535 interface(MEMORY_INTER) %{
5536 base($reg);
5537 index(0xffffffff);
5538 scale(0x0);
5539 disp($off);
5540 %}
5541 %}
5542
5543 operand indirectN(iRegN reg)
5544 %{
5545 predicate(CompressedOops::shift() == 0);
5546 constraint(ALLOC_IN_RC(ptr_reg));
5547 match(DecodeN reg);
5548 op_cost(0);
5549 format %{ "[$reg]\t# narrow" %}
5550 interface(MEMORY_INTER) %{
5551 base($reg);
5552 index(0xffffffff);
5553 scale(0x0);
5554 disp(0x0);
5555 %}
5556 %}
5557
5558 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5559 %{
5560 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5561 constraint(ALLOC_IN_RC(ptr_reg));
5562 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5563 op_cost(0);
5564 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5565 interface(MEMORY_INTER) %{
5566 base($reg);
5567 index($ireg);
5568 scale($scale);
5569 disp(0x0);
5570 %}
5571 %}
5572
5573 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5574 %{
5575 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5576 constraint(ALLOC_IN_RC(ptr_reg));
5577 match(AddP (DecodeN reg) (LShiftL lreg scale));
5578 op_cost(0);
5579 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5580 interface(MEMORY_INTER) %{
5581 base($reg);
5582 index($lreg);
5583 scale($scale);
5584 disp(0x0);
5585 %}
5586 %}
5587
5588 operand indIndexI2LN(iRegN reg, iRegI ireg)
5589 %{
5590 predicate(CompressedOops::shift() == 0);
5591 constraint(ALLOC_IN_RC(ptr_reg));
5592 match(AddP (DecodeN reg) (ConvI2L ireg));
5593 op_cost(0);
5594 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5595 interface(MEMORY_INTER) %{
5596 base($reg);
5597 index($ireg);
5598 scale(0x0);
5599 disp(0x0);
5600 %}
5601 %}
5602
5603 operand indIndexN(iRegN reg, iRegL lreg)
5604 %{
5605 predicate(CompressedOops::shift() == 0);
5606 constraint(ALLOC_IN_RC(ptr_reg));
5607 match(AddP (DecodeN reg) lreg);
5608 op_cost(0);
5609 format %{ "$reg, $lreg\t# narrow" %}
5610 interface(MEMORY_INTER) %{
5611 base($reg);
5612 index($lreg);
5613 scale(0x0);
5614 disp(0x0);
5615 %}
5616 %}
5617
5618 operand indOffIN(iRegN reg, immIOffset off)
5619 %{
5620 predicate(CompressedOops::shift() == 0);
5621 constraint(ALLOC_IN_RC(ptr_reg));
5622 match(AddP (DecodeN reg) off);
5623 op_cost(0);
5624 format %{ "[$reg, $off]\t# narrow" %}
5625 interface(MEMORY_INTER) %{
5626 base($reg);
5627 index(0xffffffff);
5628 scale(0x0);
5629 disp($off);
5630 %}
5631 %}
5632
5633 operand indOffLN(iRegN reg, immLOffset off)
5634 %{
5635 predicate(CompressedOops::shift() == 0);
5636 constraint(ALLOC_IN_RC(ptr_reg));
5637 match(AddP (DecodeN reg) off);
5638 op_cost(0);
5639 format %{ "[$reg, $off]\t# narrow" %}
5640 interface(MEMORY_INTER) %{
5641 base($reg);
5642 index(0xffffffff);
5643 scale(0x0);
5644 disp($off);
5645 %}
5646 %}
5647
5648
5649 //----------Special Memory Operands--------------------------------------------
5650 // Stack Slot Operand - This operand is used for loading and storing temporary
5651 // values on the stack where a match requires a value to
5652 // flow through memory.
5653 operand stackSlotP(sRegP reg)
5654 %{
5655 constraint(ALLOC_IN_RC(stack_slots));
5656 op_cost(100);
5657 // No match rule because this operand is only generated in matching
5658 // match(RegP);
5659 format %{ "[$reg]" %}
5660 interface(MEMORY_INTER) %{
5661 base(0x1e); // RSP
5662 index(0x0); // No Index
5663 scale(0x0); // No Scale
5664 disp($reg); // Stack Offset
5665 %}
5666 %}
5667
5668 operand stackSlotI(sRegI reg)
5669 %{
5670 constraint(ALLOC_IN_RC(stack_slots));
5671 // No match rule because this operand is only generated in matching
5672 // match(RegI);
5673 format %{ "[$reg]" %}
5674 interface(MEMORY_INTER) %{
5675 base(0x1e); // RSP
5676 index(0x0); // No Index
5677 scale(0x0); // No Scale
5678 disp($reg); // Stack Offset
5679 %}
5680 %}
5681
5682 operand stackSlotF(sRegF reg)
5683 %{
5684 constraint(ALLOC_IN_RC(stack_slots));
5685 // No match rule because this operand is only generated in matching
5686 // match(RegF);
5687 format %{ "[$reg]" %}
5688 interface(MEMORY_INTER) %{
5689 base(0x1e); // RSP
5690 index(0x0); // No Index
5691 scale(0x0); // No Scale
5692 disp($reg); // Stack Offset
5693 %}
5694 %}
5695
5696 operand stackSlotD(sRegD reg)
5697 %{
5698 constraint(ALLOC_IN_RC(stack_slots));
5699 // No match rule because this operand is only generated in matching
5700 // match(RegD);
5701 format %{ "[$reg]" %}
5702 interface(MEMORY_INTER) %{
5703 base(0x1e); // RSP
5704 index(0x0); // No Index
5705 scale(0x0); // No Scale
5706 disp($reg); // Stack Offset
5707 %}
5708 %}
5709
5710 operand stackSlotL(sRegL reg)
5711 %{
5712 constraint(ALLOC_IN_RC(stack_slots));
5713 // No match rule because this operand is only generated in matching
5714 // match(RegL);
5715 format %{ "[$reg]" %}
5716 interface(MEMORY_INTER) %{
5717 base(0x1e); // RSP
5718 index(0x0); // No Index
5719 scale(0x0); // No Scale
5720 disp($reg); // Stack Offset
5721 %}
5722 %}
5723
5724 // Operands for expressing Control Flow
5725 // NOTE: Label is a predefined operand which should not be redefined in
5726 // the AD file. It is generically handled within the ADLC.
5727
5728 //----------Conditional Branch Operands----------------------------------------
5729 // Comparison Op - This is the operation of the comparison, and is limited to
5730 // the following set of codes:
5731 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5732 //
5733 // Other attributes of the comparison, such as unsignedness, are specified
5734 // by the comparison instruction that sets a condition code flags register.
5735 // That result is represented by a flags operand whose subtype is appropriate
5736 // to the unsignedness (etc.) of the comparison.
5737 //
5738 // Later, the instruction which matches both the Comparison Op (a Bool) and
5739 // the flags (produced by the Cmp) specifies the coding of the comparison op
5740 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5741
5742 // used for signed integral comparisons and fp comparisons
5743
5744 operand cmpOp()
5745 %{
5746 match(Bool);
5747
5748 format %{ "" %}
5749 interface(COND_INTER) %{
5750 equal(0x0, "eq");
5751 not_equal(0x1, "ne");
5752 less(0xb, "lt");
5753 greater_equal(0xa, "ge");
5754 less_equal(0xd, "le");
5755 greater(0xc, "gt");
5756 overflow(0x6, "vs");
5757 no_overflow(0x7, "vc");
5758 %}
5759 %}
5760
5761 // used for unsigned integral comparisons
5762
5763 operand cmpOpU()
5764 %{
5765 match(Bool);
5766
5767 format %{ "" %}
5768 interface(COND_INTER) %{
5769 equal(0x0, "eq");
5770 not_equal(0x1, "ne");
5771 less(0x3, "lo");
5772 greater_equal(0x2, "hs");
5773 less_equal(0x9, "ls");
5774 greater(0x8, "hi");
5775 overflow(0x6, "vs");
5776 no_overflow(0x7, "vc");
5777 %}
5778 %}
5779
5780 // used for certain integral comparisons which can be
5781 // converted to cbxx or tbxx instructions
5782
5783 operand cmpOpEqNe()
5784 %{
5785 match(Bool);
5786 op_cost(0);
5787 predicate(n->as_Bool()->_test._test == BoolTest::ne
5788 || n->as_Bool()->_test._test == BoolTest::eq);
5789
5790 format %{ "" %}
5791 interface(COND_INTER) %{
5792 equal(0x0, "eq");
5793 not_equal(0x1, "ne");
5794 less(0xb, "lt");
5795 greater_equal(0xa, "ge");
5796 less_equal(0xd, "le");
5797 greater(0xc, "gt");
5798 overflow(0x6, "vs");
5799 no_overflow(0x7, "vc");
5800 %}
5801 %}
5802
5803 // used for certain integral comparisons which can be
5804 // converted to cbxx or tbxx instructions
5805
5806 operand cmpOpLtGe()
5807 %{
5808 match(Bool);
5809 op_cost(0);
5810
5811 predicate(n->as_Bool()->_test._test == BoolTest::lt
5812 || n->as_Bool()->_test._test == BoolTest::ge);
5813
5814 format %{ "" %}
5815 interface(COND_INTER) %{
5816 equal(0x0, "eq");
5817 not_equal(0x1, "ne");
5818 less(0xb, "lt");
5819 greater_equal(0xa, "ge");
5820 less_equal(0xd, "le");
5821 greater(0xc, "gt");
5822 overflow(0x6, "vs");
5823 no_overflow(0x7, "vc");
5824 %}
5825 %}
5826
5827 // used for certain unsigned integral comparisons which can be
5828 // converted to cbxx or tbxx instructions
5829
5830 operand cmpOpUEqNeLeGt()
5831 %{
5832 match(Bool);
5833 op_cost(0);
5834
5835 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5836 n->as_Bool()->_test._test == BoolTest::ne ||
5837 n->as_Bool()->_test._test == BoolTest::le ||
5838 n->as_Bool()->_test._test == BoolTest::gt);
5839
5840 format %{ "" %}
5841 interface(COND_INTER) %{
5842 equal(0x0, "eq");
5843 not_equal(0x1, "ne");
5844 less(0x3, "lo");
5845 greater_equal(0x2, "hs");
5846 less_equal(0x9, "ls");
5847 greater(0x8, "hi");
5848 overflow(0x6, "vs");
5849 no_overflow(0x7, "vc");
5850 %}
5851 %}
5852
5853 // Special operand allowing long args to int ops to be truncated for free
5854
5855 operand iRegL2I(iRegL reg) %{
5856
5857 op_cost(0);
5858
5859 match(ConvL2I reg);
5860
5861 format %{ "l2i($reg)" %}
5862
5863 interface(REG_INTER)
5864 %}
5865
5866 operand iRegL2P(iRegL reg) %{
5867
5868 op_cost(0);
5869
5870 match(CastX2P reg);
5871
5872 format %{ "l2p($reg)" %}
5873
5874 interface(REG_INTER)
5875 %}
5876
5877 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5878 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5879 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5880 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5881
5882 //----------OPERAND CLASSES----------------------------------------------------
5883 // Operand Classes are groups of operands that are used as to simplify
5884 // instruction definitions by not requiring the AD writer to specify
5885 // separate instructions for every form of operand when the
5886 // instruction accepts multiple operand types with the same basic
5887 // encoding and format. The classic case of this is memory operands.
5888
5889 // memory is used to define read/write location for load/store
5890 // instruction defs. we can turn a memory op into an Address
5891
5892 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5893 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5894
5895 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5896 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5897
5898 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5899 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5900
5901 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5902 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5903
5904 // All of the memory operands. For the pipeline description.
5905 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5906 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5907 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5908
5909 opclass memory_noindex(indirect,
5910 indOffI1, indOffL1,indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5911 indirectN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5912
5913 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5914 // operations. it allows the src to be either an iRegI or a (ConvL2I
5915 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5916 // can be elided because the 32-bit instruction will just employ the
5917 // lower 32 bits anyway.
5918 //
5919 // n.b. this does not elide all L2I conversions. if the truncated
5920 // value is consumed by more than one operation then the ConvL2I
5921 // cannot be bundled into the consuming nodes so an l2i gets planted
5922 // (actually a movw $dst $src) and the downstream instructions consume
5923 // the result of the l2i as an iRegI input. That's a shame since the
5924 // movw is actually redundant but its not too costly.
5925
5926 opclass iRegIorL2I(iRegI, iRegL2I);
5927 opclass iRegPorL2P(iRegP, iRegL2P);
5928
5929 //----------PIPELINE-----------------------------------------------------------
5930 // Rules which define the behavior of the target architectures pipeline.
5931
5932 // For specific pipelines, eg A53, define the stages of that pipeline
5933 //pipe_desc(ISS, EX1, EX2, WR);
5934 #define ISS S0
5935 #define EX1 S1
5936 #define EX2 S2
5937 #define WR S3
5938
5939 // Integer ALU reg operation
5940 pipeline %{
5941
5942 attributes %{
5943 // ARM instructions are of fixed length
5944 fixed_size_instructions; // Fixed size instructions TODO does
5945 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5946 // ARM instructions come in 32-bit word units
5947 instruction_unit_size = 4; // An instruction is 4 bytes long
5948 instruction_fetch_unit_size = 64; // The processor fetches one line
5949 instruction_fetch_units = 1; // of 64 bytes
5950 %}
5951
5952 // We don't use an actual pipeline model so don't care about resources
5953 // or description. we do use pipeline classes to introduce fixed
5954 // latencies
5955
5956 //----------RESOURCES----------------------------------------------------------
5957 // Resources are the functional units available to the machine
5958
5959 resources( INS0, INS1, INS01 = INS0 | INS1,
5960 ALU0, ALU1, ALU = ALU0 | ALU1,
5961 MAC,
5962 DIV,
5963 BRANCH,
5964 LDST,
5965 NEON_FP);
5966
5967 //----------PIPELINE DESCRIPTION-----------------------------------------------
5968 // Pipeline Description specifies the stages in the machine's pipeline
5969
5970 // Define the pipeline as a generic 6 stage pipeline
5971 pipe_desc(S0, S1, S2, S3, S4, S5);
5972
5973 //----------PIPELINE CLASSES---------------------------------------------------
5974 // Pipeline Classes describe the stages in which input and output are
5975 // referenced by the hardware pipeline.
5976
5977 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5978 %{
5979 single_instruction;
5980 src1 : S1(read);
5981 src2 : S2(read);
5982 dst : S5(write);
5983 INS01 : ISS;
5984 NEON_FP : S5;
5985 %}
5986
5987 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5988 %{
5989 single_instruction;
5990 src1 : S1(read);
5991 src2 : S2(read);
5992 dst : S5(write);
5993 INS01 : ISS;
5994 NEON_FP : S5;
5995 %}
5996
5997 pipe_class fp_uop_s(vRegF dst, vRegF src)
5998 %{
5999 single_instruction;
6000 src : S1(read);
6001 dst : S5(write);
6002 INS01 : ISS;
6003 NEON_FP : S5;
6004 %}
6005
6006 pipe_class fp_uop_d(vRegD dst, vRegD src)
6007 %{
6008 single_instruction;
6009 src : S1(read);
6010 dst : S5(write);
6011 INS01 : ISS;
6012 NEON_FP : S5;
6013 %}
6014
6015 pipe_class fp_d2f(vRegF dst, vRegD src)
6016 %{
6017 single_instruction;
6018 src : S1(read);
6019 dst : S5(write);
6020 INS01 : ISS;
6021 NEON_FP : S5;
6022 %}
6023
6024 pipe_class fp_f2d(vRegD dst, vRegF src)
6025 %{
6026 single_instruction;
6027 src : S1(read);
6028 dst : S5(write);
6029 INS01 : ISS;
6030 NEON_FP : S5;
6031 %}
6032
6033 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6034 %{
6035 single_instruction;
6036 src : S1(read);
6037 dst : S5(write);
6038 INS01 : ISS;
6039 NEON_FP : S5;
6040 %}
6041
6042 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6043 %{
6044 single_instruction;
6045 src : S1(read);
6046 dst : S5(write);
6047 INS01 : ISS;
6048 NEON_FP : S5;
6049 %}
6050
6051 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6052 %{
6053 single_instruction;
6054 src : S1(read);
6055 dst : S5(write);
6056 INS01 : ISS;
6057 NEON_FP : S5;
6058 %}
6059
6060 pipe_class fp_l2f(vRegF dst, iRegL src)
6061 %{
6062 single_instruction;
6063 src : S1(read);
6064 dst : S5(write);
6065 INS01 : ISS;
6066 NEON_FP : S5;
6067 %}
6068
6069 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6070 %{
6071 single_instruction;
6072 src : S1(read);
6073 dst : S5(write);
6074 INS01 : ISS;
6075 NEON_FP : S5;
6076 %}
6077
6078 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6079 %{
6080 single_instruction;
6081 src : S1(read);
6082 dst : S5(write);
6083 INS01 : ISS;
6084 NEON_FP : S5;
6085 %}
6086
6087 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6088 %{
6089 single_instruction;
6090 src : S1(read);
6091 dst : S5(write);
6092 INS01 : ISS;
6093 NEON_FP : S5;
6094 %}
6095
6096 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6097 %{
6098 single_instruction;
6099 src : S1(read);
6100 dst : S5(write);
6101 INS01 : ISS;
6102 NEON_FP : S5;
6103 %}
6104
6105 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6106 %{
6107 single_instruction;
6108 src1 : S1(read);
6109 src2 : S2(read);
6110 dst : S5(write);
6111 INS0 : ISS;
6112 NEON_FP : S5;
6113 %}
6114
6115 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6116 %{
6117 single_instruction;
6118 src1 : S1(read);
6119 src2 : S2(read);
6120 dst : S5(write);
6121 INS0 : ISS;
6122 NEON_FP : S5;
6123 %}
6124
6125 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6126 %{
6127 single_instruction;
6128 cr : S1(read);
6129 src1 : S1(read);
6130 src2 : S1(read);
6131 dst : S3(write);
6132 INS01 : ISS;
6133 NEON_FP : S3;
6134 %}
6135
6136 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6137 %{
6138 single_instruction;
6139 cr : S1(read);
6140 src1 : S1(read);
6141 src2 : S1(read);
6142 dst : S3(write);
6143 INS01 : ISS;
6144 NEON_FP : S3;
6145 %}
6146
6147 pipe_class fp_imm_s(vRegF dst)
6148 %{
6149 single_instruction;
6150 dst : S3(write);
6151 INS01 : ISS;
6152 NEON_FP : S3;
6153 %}
6154
6155 pipe_class fp_imm_d(vRegD dst)
6156 %{
6157 single_instruction;
6158 dst : S3(write);
6159 INS01 : ISS;
6160 NEON_FP : S3;
6161 %}
6162
6163 pipe_class fp_load_constant_s(vRegF dst)
6164 %{
6165 single_instruction;
6166 dst : S4(write);
6167 INS01 : ISS;
6168 NEON_FP : S4;
6169 %}
6170
6171 pipe_class fp_load_constant_d(vRegD dst)
6172 %{
6173 single_instruction;
6174 dst : S4(write);
6175 INS01 : ISS;
6176 NEON_FP : S4;
6177 %}
6178
6179 //------- Integer ALU operations --------------------------
6180
6181 // Integer ALU reg-reg operation
6182 // Operands needed in EX1, result generated in EX2
6183 // Eg. ADD x0, x1, x2
6184 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6185 %{
6186 single_instruction;
6187 dst : EX2(write);
6188 src1 : EX1(read);
6189 src2 : EX1(read);
6190 INS01 : ISS; // Dual issue as instruction 0 or 1
6191 ALU : EX2;
6192 %}
6193
6194 // Integer ALU reg-reg operation with constant shift
6195 // Shifted register must be available in LATE_ISS instead of EX1
6196 // Eg. ADD x0, x1, x2, LSL #2
6197 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6198 %{
6199 single_instruction;
6200 dst : EX2(write);
6201 src1 : EX1(read);
6202 src2 : ISS(read);
6203 INS01 : ISS;
6204 ALU : EX2;
6205 %}
6206
6207 // Integer ALU reg operation with constant shift
6208 // Eg. LSL x0, x1, #shift
6209 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6210 %{
6211 single_instruction;
6212 dst : EX2(write);
6213 src1 : ISS(read);
6214 INS01 : ISS;
6215 ALU : EX2;
6216 %}
6217
6218 // Integer ALU reg-reg operation with variable shift
6219 // Both operands must be available in LATE_ISS instead of EX1
6220 // Result is available in EX1 instead of EX2
6221 // Eg. LSLV x0, x1, x2
6222 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6223 %{
6224 single_instruction;
6225 dst : EX1(write);
6226 src1 : ISS(read);
6227 src2 : ISS(read);
6228 INS01 : ISS;
6229 ALU : EX1;
6230 %}
6231
6232 // Integer ALU reg-reg operation with extract
6233 // As for _vshift above, but result generated in EX2
6234 // Eg. EXTR x0, x1, x2, #N
6235 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6236 %{
6237 single_instruction;
6238 dst : EX2(write);
6239 src1 : ISS(read);
6240 src2 : ISS(read);
6241 INS1 : ISS; // Can only dual issue as Instruction 1
6242 ALU : EX1;
6243 %}
6244
6245 // Integer ALU reg operation
6246 // Eg. NEG x0, x1
6247 pipe_class ialu_reg(iRegI dst, iRegI src)
6248 %{
6249 single_instruction;
6250 dst : EX2(write);
6251 src : EX1(read);
6252 INS01 : ISS;
6253 ALU : EX2;
6254 %}
6255
6256 // Integer ALU reg mmediate operation
6257 // Eg. ADD x0, x1, #N
6258 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6259 %{
6260 single_instruction;
6261 dst : EX2(write);
6262 src1 : EX1(read);
6263 INS01 : ISS;
6264 ALU : EX2;
6265 %}
6266
6267 // Integer ALU immediate operation (no source operands)
6268 // Eg. MOV x0, #N
6269 pipe_class ialu_imm(iRegI dst)
6270 %{
6271 single_instruction;
6272 dst : EX1(write);
6273 INS01 : ISS;
6274 ALU : EX1;
6275 %}
6276
6277 //------- Compare operation -------------------------------
6278
6279 // Compare reg-reg
6280 // Eg. CMP x0, x1
6281 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6282 %{
6283 single_instruction;
6284 // fixed_latency(16);
6285 cr : EX2(write);
6286 op1 : EX1(read);
6287 op2 : EX1(read);
6288 INS01 : ISS;
6289 ALU : EX2;
6290 %}
6291
6292 // Compare reg-reg
6293 // Eg. CMP x0, #N
6294 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6295 %{
6296 single_instruction;
6297 // fixed_latency(16);
6298 cr : EX2(write);
6299 op1 : EX1(read);
6300 INS01 : ISS;
6301 ALU : EX2;
6302 %}
6303
6304 //------- Conditional instructions ------------------------
6305
6306 // Conditional no operands
6307 // Eg. CSINC x0, zr, zr, <cond>
6308 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6309 %{
6310 single_instruction;
6311 cr : EX1(read);
6312 dst : EX2(write);
6313 INS01 : ISS;
6314 ALU : EX2;
6315 %}
6316
6317 // Conditional 2 operand
6318 // EG. CSEL X0, X1, X2, <cond>
6319 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6320 %{
6321 single_instruction;
6322 cr : EX1(read);
6323 src1 : EX1(read);
6324 src2 : EX1(read);
6325 dst : EX2(write);
6326 INS01 : ISS;
6327 ALU : EX2;
6328 %}
6329
6330 // Conditional 2 operand
6331 // EG. CSEL X0, X1, X2, <cond>
6332 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6333 %{
6334 single_instruction;
6335 cr : EX1(read);
6336 src : EX1(read);
6337 dst : EX2(write);
6338 INS01 : ISS;
6339 ALU : EX2;
6340 %}
6341
6342 //------- Multiply pipeline operations --------------------
6343
6344 // Multiply reg-reg
6345 // Eg. MUL w0, w1, w2
6346 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6347 %{
6348 single_instruction;
6349 dst : WR(write);
6350 src1 : ISS(read);
6351 src2 : ISS(read);
6352 INS01 : ISS;
6353 MAC : WR;
6354 %}
6355
6356 // Multiply accumulate
6357 // Eg. MADD w0, w1, w2, w3
6358 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6359 %{
6360 single_instruction;
6361 dst : WR(write);
6362 src1 : ISS(read);
6363 src2 : ISS(read);
6364 src3 : ISS(read);
6365 INS01 : ISS;
6366 MAC : WR;
6367 %}
6368
6369 // Eg. MUL w0, w1, w2
6370 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6371 %{
6372 single_instruction;
6373 fixed_latency(3); // Maximum latency for 64 bit mul
6374 dst : WR(write);
6375 src1 : ISS(read);
6376 src2 : ISS(read);
6377 INS01 : ISS;
6378 MAC : WR;
6379 %}
6380
6381 // Multiply accumulate
6382 // Eg. MADD w0, w1, w2, w3
6383 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6384 %{
6385 single_instruction;
6386 fixed_latency(3); // Maximum latency for 64 bit mul
6387 dst : WR(write);
6388 src1 : ISS(read);
6389 src2 : ISS(read);
6390 src3 : ISS(read);
6391 INS01 : ISS;
6392 MAC : WR;
6393 %}
6394
6395 //------- Divide pipeline operations --------------------
6396
6397 // Eg. SDIV w0, w1, w2
6398 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6399 %{
6400 single_instruction;
6401 fixed_latency(8); // Maximum latency for 32 bit divide
6402 dst : WR(write);
6403 src1 : ISS(read);
6404 src2 : ISS(read);
6405 INS0 : ISS; // Can only dual issue as instruction 0
6406 DIV : WR;
6407 %}
6408
6409 // Eg. SDIV x0, x1, x2
6410 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6411 %{
6412 single_instruction;
6413 fixed_latency(16); // Maximum latency for 64 bit divide
6414 dst : WR(write);
6415 src1 : ISS(read);
6416 src2 : ISS(read);
6417 INS0 : ISS; // Can only dual issue as instruction 0
6418 DIV : WR;
6419 %}
6420
6421 //------- Load pipeline operations ------------------------
6422
6423 // Load - prefetch
6424 // Eg. PFRM <mem>
6425 pipe_class iload_prefetch(memory mem)
6426 %{
6427 single_instruction;
6428 mem : ISS(read);
6429 INS01 : ISS;
6430 LDST : WR;
6431 %}
6432
6433 // Load - reg, mem
6434 // Eg. LDR x0, <mem>
6435 pipe_class iload_reg_mem(iRegI dst, memory mem)
6436 %{
6437 single_instruction;
6438 dst : WR(write);
6439 mem : ISS(read);
6440 INS01 : ISS;
6441 LDST : WR;
6442 %}
6443
6444 // Load - reg, reg
6445 // Eg. LDR x0, [sp, x1]
6446 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6447 %{
6448 single_instruction;
6449 dst : WR(write);
6450 src : ISS(read);
6451 INS01 : ISS;
6452 LDST : WR;
6453 %}
6454
6455 //------- Store pipeline operations -----------------------
6456
6457 // Store - zr, mem
6458 // Eg. STR zr, <mem>
6459 pipe_class istore_mem(memory mem)
6460 %{
6461 single_instruction;
6462 mem : ISS(read);
6463 INS01 : ISS;
6464 LDST : WR;
6465 %}
6466
6467 // Store - reg, mem
6468 // Eg. STR x0, <mem>
6469 pipe_class istore_reg_mem(iRegI src, memory mem)
6470 %{
6471 single_instruction;
6472 mem : ISS(read);
6473 src : EX2(read);
6474 INS01 : ISS;
6475 LDST : WR;
6476 %}
6477
6478 // Store - reg, reg
6479 // Eg. STR x0, [sp, x1]
6480 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6481 %{
6482 single_instruction;
6483 dst : ISS(read);
6484 src : EX2(read);
6485 INS01 : ISS;
6486 LDST : WR;
6487 %}
6488
6489 //------- Store pipeline operations -----------------------
6490
6491 // Branch
6492 pipe_class pipe_branch()
6493 %{
6494 single_instruction;
6495 INS01 : ISS;
6496 BRANCH : EX1;
6497 %}
6498
6499 // Conditional branch
6500 pipe_class pipe_branch_cond(rFlagsReg cr)
6501 %{
6502 single_instruction;
6503 cr : EX1(read);
6504 INS01 : ISS;
6505 BRANCH : EX1;
6506 %}
6507
6508 // Compare & Branch
6509 // EG. CBZ/CBNZ
6510 pipe_class pipe_cmp_branch(iRegI op1)
6511 %{
6512 single_instruction;
6513 op1 : EX1(read);
6514 INS01 : ISS;
6515 BRANCH : EX1;
6516 %}
6517
6518 //------- Synchronisation operations ----------------------
6519
6520 // Any operation requiring serialization.
6521 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6522 pipe_class pipe_serial()
6523 %{
6524 single_instruction;
6525 force_serialization;
6526 fixed_latency(16);
6527 INS01 : ISS(2); // Cannot dual issue with any other instruction
6528 LDST : WR;
6529 %}
6530
6531 // Generic big/slow expanded idiom - also serialized
6532 pipe_class pipe_slow()
6533 %{
6534 instruction_count(10);
6535 multiple_bundles;
6536 force_serialization;
6537 fixed_latency(16);
6538 INS01 : ISS(2); // Cannot dual issue with any other instruction
6539 LDST : WR;
6540 %}
6541
6542 // Empty pipeline class
6543 pipe_class pipe_class_empty()
6544 %{
6545 single_instruction;
6546 fixed_latency(0);
6547 %}
6548
6549 // Default pipeline class.
6550 pipe_class pipe_class_default()
6551 %{
6552 single_instruction;
6553 fixed_latency(2);
6554 %}
6555
6556 // Pipeline class for compares.
6557 pipe_class pipe_class_compare()
6558 %{
6559 single_instruction;
6560 fixed_latency(16);
6561 %}
6562
6563 // Pipeline class for memory operations.
6564 pipe_class pipe_class_memory()
6565 %{
6566 single_instruction;
6567 fixed_latency(16);
6568 %}
6569
6570 // Pipeline class for call.
6571 pipe_class pipe_class_call()
6572 %{
6573 single_instruction;
6574 fixed_latency(100);
6575 %}
6576
6577 // Define the class for the Nop node.
6578 define %{
6579 MachNop = pipe_class_empty;
6580 %}
6581
6582 %}
6583 //----------INSTRUCTIONS-------------------------------------------------------
6584 //
6585 // match -- States which machine-independent subtree may be replaced
6586 // by this instruction.
6587 // ins_cost -- The estimated cost of this instruction is used by instruction
6588 // selection to identify a minimum cost tree of machine
6589 // instructions that matches a tree of machine-independent
6590 // instructions.
6591 // format -- A string providing the disassembly for this instruction.
6592 // The value of an instruction's operand may be inserted
6593 // by referring to it with a '$' prefix.
6594 // opcode -- Three instruction opcodes may be provided. These are referred
6595 // to within an encode class as $primary, $secondary, and $tertiary
6596 // rrspectively. The primary opcode is commonly used to
6597 // indicate the type of machine instruction, while secondary
6598 // and tertiary are often used for prefix options or addressing
6599 // modes.
6600 // ins_encode -- A list of encode classes with parameters. The encode class
6601 // name must have been defined in an 'enc_class' specification
6602 // in the encode section of the architecture description.
6603
6604 // ============================================================================
6605 // Memory (Load/Store) Instructions
6606
6607 // Load Instructions
6608
6609 // Load Byte (8 bit signed)
6610 instruct loadB(iRegINoSp dst, memory1 mem)
6611 %{
6612 match(Set dst (LoadB mem));
6613 predicate(!needs_acquiring_load(n));
6614
6615 ins_cost(4 * INSN_COST);
6616 format %{ "ldrsbw $dst, $mem\t# byte" %}
6617
6618 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6619
6620 ins_pipe(iload_reg_mem);
6621 %}
6622
6623 // Load Byte (8 bit signed) into long
6624 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6625 %{
6626 match(Set dst (ConvI2L (LoadB mem)));
6627 predicate(!needs_acquiring_load(n->in(1)));
6628
6629 ins_cost(4 * INSN_COST);
6630 format %{ "ldrsb $dst, $mem\t# byte" %}
6631
6632 ins_encode(aarch64_enc_ldrsb(dst, mem));
6633
6634 ins_pipe(iload_reg_mem);
6635 %}
6636
6637 // Load Byte (8 bit unsigned)
6638 instruct loadUB(iRegINoSp dst, memory1 mem)
6639 %{
6640 match(Set dst (LoadUB mem));
6641 predicate(!needs_acquiring_load(n));
6642
6643 ins_cost(4 * INSN_COST);
6644 format %{ "ldrbw $dst, $mem\t# byte" %}
6645
6646 ins_encode(aarch64_enc_ldrb(dst, mem));
6647
6648 ins_pipe(iload_reg_mem);
6649 %}
6650
6651 // Load Byte (8 bit unsigned) into long
6652 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6653 %{
6654 match(Set dst (ConvI2L (LoadUB mem)));
6655 predicate(!needs_acquiring_load(n->in(1)));
6656
6657 ins_cost(4 * INSN_COST);
6658 format %{ "ldrb $dst, $mem\t# byte" %}
6659
6660 ins_encode(aarch64_enc_ldrb(dst, mem));
6661
6662 ins_pipe(iload_reg_mem);
6663 %}
6664
6665 // Load Short (16 bit signed)
6666 instruct loadS(iRegINoSp dst, memory2 mem)
6667 %{
6668 match(Set dst (LoadS mem));
6669 predicate(!needs_acquiring_load(n));
6670
6671 ins_cost(4 * INSN_COST);
6672 format %{ "ldrshw $dst, $mem\t# short" %}
6673
6674 ins_encode(aarch64_enc_ldrshw(dst, mem));
6675
6676 ins_pipe(iload_reg_mem);
6677 %}
6678
6679 // Load Short (16 bit signed) into long
6680 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6681 %{
6682 match(Set dst (ConvI2L (LoadS mem)));
6683 predicate(!needs_acquiring_load(n->in(1)));
6684
6685 ins_cost(4 * INSN_COST);
6686 format %{ "ldrsh $dst, $mem\t# short" %}
6687
6688 ins_encode(aarch64_enc_ldrsh(dst, mem));
6689
6690 ins_pipe(iload_reg_mem);
6691 %}
6692
6693 // Load Char (16 bit unsigned)
6694 instruct loadUS(iRegINoSp dst, memory2 mem)
6695 %{
6696 match(Set dst (LoadUS mem));
6697 predicate(!needs_acquiring_load(n));
6698
6699 ins_cost(4 * INSN_COST);
6700 format %{ "ldrh $dst, $mem\t# short" %}
6701
6702 ins_encode(aarch64_enc_ldrh(dst, mem));
6703
6704 ins_pipe(iload_reg_mem);
6705 %}
6706
6707 // Load Short/Char (16 bit unsigned) into long
6708 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6709 %{
6710 match(Set dst (ConvI2L (LoadUS mem)));
6711 predicate(!needs_acquiring_load(n->in(1)));
6712
6713 ins_cost(4 * INSN_COST);
6714 format %{ "ldrh $dst, $mem\t# short" %}
6715
6716 ins_encode(aarch64_enc_ldrh(dst, mem));
6717
6718 ins_pipe(iload_reg_mem);
6719 %}
6720
6721 // Load Integer (32 bit signed)
6722 instruct loadI(iRegINoSp dst, memory4 mem)
6723 %{
6724 match(Set dst (LoadI mem));
6725 predicate(!needs_acquiring_load(n));
6726
6727 ins_cost(4 * INSN_COST);
6728 format %{ "ldrw $dst, $mem\t# int" %}
6729
6730 ins_encode(aarch64_enc_ldrw(dst, mem));
6731
6732 ins_pipe(iload_reg_mem);
6733 %}
6734
6735 // Load Integer (32 bit signed) into long
6736 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6737 %{
6738 match(Set dst (ConvI2L (LoadI mem)));
6739 predicate(!needs_acquiring_load(n->in(1)));
6740
6741 ins_cost(4 * INSN_COST);
6742 format %{ "ldrsw $dst, $mem\t# int" %}
6743
6744 ins_encode(aarch64_enc_ldrsw(dst, mem));
6745
6746 ins_pipe(iload_reg_mem);
6747 %}
6748
6749 // Load Integer (32 bit unsigned) into long
6750 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6751 %{
6752 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6753 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6754
6755 ins_cost(4 * INSN_COST);
6756 format %{ "ldrw $dst, $mem\t# int" %}
6757
6758 ins_encode(aarch64_enc_ldrw(dst, mem));
6759
6760 ins_pipe(iload_reg_mem);
6761 %}
6762
6763 // Load Long (64 bit signed)
6764 instruct loadL(iRegLNoSp dst, memory8 mem)
6765 %{
6766 match(Set dst (LoadL mem));
6767 predicate(!needs_acquiring_load(n));
6768
6769 ins_cost(4 * INSN_COST);
6770 format %{ "ldr $dst, $mem\t# int" %}
6771
6772 ins_encode(aarch64_enc_ldr(dst, mem));
6773
6774 ins_pipe(iload_reg_mem);
6775 %}
6776
6777 // Load Range
6778 instruct loadRange(iRegINoSp dst, memory4 mem)
6779 %{
6780 match(Set dst (LoadRange mem));
6781
6782 ins_cost(4 * INSN_COST);
6783 format %{ "ldrw $dst, $mem\t# range" %}
6784
6785 ins_encode(aarch64_enc_ldrw(dst, mem));
6786
6787 ins_pipe(iload_reg_mem);
6788 %}
6789
6790 // Load Pointer
6791 instruct loadP(iRegPNoSp dst, memory8 mem)
6792 %{
6793 match(Set dst (LoadP mem));
6794 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6795
6796 ins_cost(4 * INSN_COST);
6797 format %{ "ldr $dst, $mem\t# ptr" %}
6798
6799 ins_encode(aarch64_enc_ldr(dst, mem));
6800
6801 ins_pipe(iload_reg_mem);
6802 %}
6803
6804 // Load Compressed Pointer
6805 instruct loadN(iRegNNoSp dst, memory4 mem)
6806 %{
6807 match(Set dst (LoadN mem));
6808 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6809
6810 ins_cost(4 * INSN_COST);
6811 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6812
6813 ins_encode(aarch64_enc_ldrw(dst, mem));
6814
6815 ins_pipe(iload_reg_mem);
6816 %}
6817
6818 // Load Klass Pointer
6819 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6820 %{
6821 match(Set dst (LoadKlass mem));
6822 predicate(!needs_acquiring_load(n));
6823
6824 ins_cost(4 * INSN_COST);
6825 format %{ "ldr $dst, $mem\t# class" %}
6826
6827 ins_encode(aarch64_enc_ldr(dst, mem));
6828
6829 ins_pipe(iload_reg_mem);
6830 %}
6831
6832 // Load Narrow Klass Pointer
6833 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6834 %{
6835 match(Set dst (LoadNKlass mem));
6836 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6837
6838 ins_cost(4 * INSN_COST);
6839 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6840
6841 ins_encode(aarch64_enc_ldrw(dst, mem));
6842
6843 ins_pipe(iload_reg_mem);
6844 %}
6845
6846 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory_noindex mem)
6847 %{
6848 match(Set dst (LoadNKlass mem));
6849 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6850
6851 ins_cost(4 * INSN_COST);
6852 format %{
6853 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6854 "lsrw $dst, $dst, markWord::klass_shift"
6855 %}
6856 ins_encode %{
6857 assert($mem$$index$$Register == noreg, "must not have indexed address");
6858 // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
6859 // obj-start, so that we can load from the object's mark-word instead.
6860 __ ldrw($dst$$Register, Address($mem$$base$$Register, $mem$$disp - Type::klass_offset()));
6861 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift);
6862 %}
6863 ins_pipe(iload_reg_mem);
6864 %}
6865
6866 // Load Float
6867 instruct loadF(vRegF dst, memory4 mem)
6868 %{
6869 match(Set dst (LoadF mem));
6870 predicate(!needs_acquiring_load(n));
6871
6872 ins_cost(4 * INSN_COST);
6873 format %{ "ldrs $dst, $mem\t# float" %}
6874
6875 ins_encode( aarch64_enc_ldrs(dst, mem) );
6876
6877 ins_pipe(pipe_class_memory);
6878 %}
6879
6880 // Load Double
6881 instruct loadD(vRegD dst, memory8 mem)
6882 %{
6883 match(Set dst (LoadD mem));
6884 predicate(!needs_acquiring_load(n));
6885
6886 ins_cost(4 * INSN_COST);
6887 format %{ "ldrd $dst, $mem\t# double" %}
6888
6889 ins_encode( aarch64_enc_ldrd(dst, mem) );
6890
6891 ins_pipe(pipe_class_memory);
6892 %}
6893
6894
6895 // Load Int Constant
6896 instruct loadConI(iRegINoSp dst, immI src)
6897 %{
6898 match(Set dst src);
6899
6900 ins_cost(INSN_COST);
6901 format %{ "mov $dst, $src\t# int" %}
6902
6903 ins_encode( aarch64_enc_movw_imm(dst, src) );
6904
6905 ins_pipe(ialu_imm);
6906 %}
6907
6908 // Load Long Constant
6909 instruct loadConL(iRegLNoSp dst, immL src)
6910 %{
6911 match(Set dst src);
6912
6913 ins_cost(INSN_COST);
6914 format %{ "mov $dst, $src\t# long" %}
6915
6916 ins_encode( aarch64_enc_mov_imm(dst, src) );
6917
6918 ins_pipe(ialu_imm);
6919 %}
6920
6921 // Load Pointer Constant
6922
6923 instruct loadConP(iRegPNoSp dst, immP con)
6924 %{
6925 match(Set dst con);
6926
6927 ins_cost(INSN_COST * 4);
6928 format %{
6929 "mov $dst, $con\t# ptr\n\t"
6930 %}
6931
6932 ins_encode(aarch64_enc_mov_p(dst, con));
6933
6934 ins_pipe(ialu_imm);
6935 %}
6936
6937 // Load Null Pointer Constant
6938
6939 instruct loadConP0(iRegPNoSp dst, immP0 con)
6940 %{
6941 match(Set dst con);
6942
6943 ins_cost(INSN_COST);
6944 format %{ "mov $dst, $con\t# nullptr ptr" %}
6945
6946 ins_encode(aarch64_enc_mov_p0(dst, con));
6947
6948 ins_pipe(ialu_imm);
6949 %}
6950
6951 // Load Pointer Constant One
6952
6953 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6954 %{
6955 match(Set dst con);
6956
6957 ins_cost(INSN_COST);
6958 format %{ "mov $dst, $con\t# nullptr ptr" %}
6959
6960 ins_encode(aarch64_enc_mov_p1(dst, con));
6961
6962 ins_pipe(ialu_imm);
6963 %}
6964
6965 // Load Narrow Pointer Constant
6966
6967 instruct loadConN(iRegNNoSp dst, immN con)
6968 %{
6969 match(Set dst con);
6970
6971 ins_cost(INSN_COST * 4);
6972 format %{ "mov $dst, $con\t# compressed ptr" %}
6973
6974 ins_encode(aarch64_enc_mov_n(dst, con));
6975
6976 ins_pipe(ialu_imm);
6977 %}
6978
6979 // Load Narrow Null Pointer Constant
6980
6981 instruct loadConN0(iRegNNoSp dst, immN0 con)
6982 %{
6983 match(Set dst con);
6984
6985 ins_cost(INSN_COST);
6986 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6987
6988 ins_encode(aarch64_enc_mov_n0(dst, con));
6989
6990 ins_pipe(ialu_imm);
6991 %}
6992
6993 // Load Narrow Klass Constant
6994
6995 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6996 %{
6997 match(Set dst con);
6998
6999 ins_cost(INSN_COST);
7000 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7001
7002 ins_encode(aarch64_enc_mov_nk(dst, con));
7003
7004 ins_pipe(ialu_imm);
7005 %}
7006
7007 // Load Packed Float Constant
7008
7009 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7010 match(Set dst con);
7011 ins_cost(INSN_COST * 4);
7012 format %{ "fmovs $dst, $con"%}
7013 ins_encode %{
7014 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7015 %}
7016
7017 ins_pipe(fp_imm_s);
7018 %}
7019
7020 // Load Float Constant
7021
7022 instruct loadConF(vRegF dst, immF con) %{
7023 match(Set dst con);
7024
7025 ins_cost(INSN_COST * 4);
7026
7027 format %{
7028 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7029 %}
7030
7031 ins_encode %{
7032 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7033 %}
7034
7035 ins_pipe(fp_load_constant_s);
7036 %}
7037
7038 // Load Packed Double Constant
7039
7040 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7041 match(Set dst con);
7042 ins_cost(INSN_COST);
7043 format %{ "fmovd $dst, $con"%}
7044 ins_encode %{
7045 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7046 %}
7047
7048 ins_pipe(fp_imm_d);
7049 %}
7050
7051 // Load Double Constant
7052
7053 instruct loadConD(vRegD dst, immD con) %{
7054 match(Set dst con);
7055
7056 ins_cost(INSN_COST * 5);
7057 format %{
7058 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7059 %}
7060
7061 ins_encode %{
7062 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7063 %}
7064
7065 ins_pipe(fp_load_constant_d);
7066 %}
7067
7068 // Load Half Float Constant
7069 instruct loadConH(vRegF dst, immH con) %{
7070 match(Set dst con);
7071 format %{ "mov rscratch1, $con\n\t"
7072 "fmov $dst, rscratch1"
7073 %}
7074 ins_encode %{
7075 __ movw(rscratch1, (uint32_t)$con$$constant);
7076 __ fmovs($dst$$FloatRegister, rscratch1);
7077 %}
7078 ins_pipe(pipe_class_default);
7079 %}
7080
7081 // Store Instructions
7082
7083 // Store Byte
7084 instruct storeB(iRegIorL2I src, memory1 mem)
7085 %{
7086 match(Set mem (StoreB mem src));
7087 predicate(!needs_releasing_store(n));
7088
7089 ins_cost(INSN_COST);
7090 format %{ "strb $src, $mem\t# byte" %}
7091
7092 ins_encode(aarch64_enc_strb(src, mem));
7093
7094 ins_pipe(istore_reg_mem);
7095 %}
7096
7097
7098 instruct storeimmB0(immI0 zero, memory1 mem)
7099 %{
7100 match(Set mem (StoreB mem zero));
7101 predicate(!needs_releasing_store(n));
7102
7103 ins_cost(INSN_COST);
7104 format %{ "strb rscractch2, $mem\t# byte" %}
7105
7106 ins_encode(aarch64_enc_strb0(mem));
7107
7108 ins_pipe(istore_mem);
7109 %}
7110
7111 // Store Char/Short
7112 instruct storeC(iRegIorL2I src, memory2 mem)
7113 %{
7114 match(Set mem (StoreC mem src));
7115 predicate(!needs_releasing_store(n));
7116
7117 ins_cost(INSN_COST);
7118 format %{ "strh $src, $mem\t# short" %}
7119
7120 ins_encode(aarch64_enc_strh(src, mem));
7121
7122 ins_pipe(istore_reg_mem);
7123 %}
7124
7125 instruct storeimmC0(immI0 zero, memory2 mem)
7126 %{
7127 match(Set mem (StoreC mem zero));
7128 predicate(!needs_releasing_store(n));
7129
7130 ins_cost(INSN_COST);
7131 format %{ "strh zr, $mem\t# short" %}
7132
7133 ins_encode(aarch64_enc_strh0(mem));
7134
7135 ins_pipe(istore_mem);
7136 %}
7137
7138 // Store Integer
7139
7140 instruct storeI(iRegIorL2I src, memory4 mem)
7141 %{
7142 match(Set mem(StoreI mem src));
7143 predicate(!needs_releasing_store(n));
7144
7145 ins_cost(INSN_COST);
7146 format %{ "strw $src, $mem\t# int" %}
7147
7148 ins_encode(aarch64_enc_strw(src, mem));
7149
7150 ins_pipe(istore_reg_mem);
7151 %}
7152
7153 instruct storeimmI0(immI0 zero, memory4 mem)
7154 %{
7155 match(Set mem(StoreI mem zero));
7156 predicate(!needs_releasing_store(n));
7157
7158 ins_cost(INSN_COST);
7159 format %{ "strw zr, $mem\t# int" %}
7160
7161 ins_encode(aarch64_enc_strw0(mem));
7162
7163 ins_pipe(istore_mem);
7164 %}
7165
7166 // Store Long (64 bit signed)
7167 instruct storeL(iRegL src, memory8 mem)
7168 %{
7169 match(Set mem (StoreL mem src));
7170 predicate(!needs_releasing_store(n));
7171
7172 ins_cost(INSN_COST);
7173 format %{ "str $src, $mem\t# int" %}
7174
7175 ins_encode(aarch64_enc_str(src, mem));
7176
7177 ins_pipe(istore_reg_mem);
7178 %}
7179
7180 // Store Long (64 bit signed)
7181 instruct storeimmL0(immL0 zero, memory8 mem)
7182 %{
7183 match(Set mem (StoreL mem zero));
7184 predicate(!needs_releasing_store(n));
7185
7186 ins_cost(INSN_COST);
7187 format %{ "str zr, $mem\t# int" %}
7188
7189 ins_encode(aarch64_enc_str0(mem));
7190
7191 ins_pipe(istore_mem);
7192 %}
7193
7194 // Store Pointer
7195 instruct storeP(iRegP src, memory8 mem)
7196 %{
7197 match(Set mem (StoreP mem src));
7198 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7199
7200 ins_cost(INSN_COST);
7201 format %{ "str $src, $mem\t# ptr" %}
7202
7203 ins_encode(aarch64_enc_str(src, mem));
7204
7205 ins_pipe(istore_reg_mem);
7206 %}
7207
7208 // Store Pointer
7209 instruct storeimmP0(immP0 zero, memory8 mem)
7210 %{
7211 match(Set mem (StoreP mem zero));
7212 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7213
7214 ins_cost(INSN_COST);
7215 format %{ "str zr, $mem\t# ptr" %}
7216
7217 ins_encode(aarch64_enc_str0(mem));
7218
7219 ins_pipe(istore_mem);
7220 %}
7221
7222 // Store Compressed Pointer
7223 instruct storeN(iRegN src, memory4 mem)
7224 %{
7225 match(Set mem (StoreN mem src));
7226 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7227
7228 ins_cost(INSN_COST);
7229 format %{ "strw $src, $mem\t# compressed ptr" %}
7230
7231 ins_encode(aarch64_enc_strw(src, mem));
7232
7233 ins_pipe(istore_reg_mem);
7234 %}
7235
7236 instruct storeImmN0(immN0 zero, memory4 mem)
7237 %{
7238 match(Set mem (StoreN mem zero));
7239 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7240
7241 ins_cost(INSN_COST);
7242 format %{ "strw zr, $mem\t# compressed ptr" %}
7243
7244 ins_encode(aarch64_enc_strw0(mem));
7245
7246 ins_pipe(istore_mem);
7247 %}
7248
7249 // Store Float
7250 instruct storeF(vRegF src, memory4 mem)
7251 %{
7252 match(Set mem (StoreF mem src));
7253 predicate(!needs_releasing_store(n));
7254
7255 ins_cost(INSN_COST);
7256 format %{ "strs $src, $mem\t# float" %}
7257
7258 ins_encode( aarch64_enc_strs(src, mem) );
7259
7260 ins_pipe(pipe_class_memory);
7261 %}
7262
7263 // TODO
7264 // implement storeImmF0 and storeFImmPacked
7265
7266 // Store Double
7267 instruct storeD(vRegD src, memory8 mem)
7268 %{
7269 match(Set mem (StoreD mem src));
7270 predicate(!needs_releasing_store(n));
7271
7272 ins_cost(INSN_COST);
7273 format %{ "strd $src, $mem\t# double" %}
7274
7275 ins_encode( aarch64_enc_strd(src, mem) );
7276
7277 ins_pipe(pipe_class_memory);
7278 %}
7279
7280 // Store Compressed Klass Pointer
7281 instruct storeNKlass(iRegN src, memory4 mem)
7282 %{
7283 predicate(!needs_releasing_store(n));
7284 match(Set mem (StoreNKlass mem src));
7285
7286 ins_cost(INSN_COST);
7287 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7288
7289 ins_encode(aarch64_enc_strw(src, mem));
7290
7291 ins_pipe(istore_reg_mem);
7292 %}
7293
7294 // TODO
7295 // implement storeImmD0 and storeDImmPacked
7296
7297 // prefetch instructions
7298 // Must be safe to execute with invalid address (cannot fault).
7299
7300 instruct prefetchalloc( memory8 mem ) %{
7301 match(PrefetchAllocation mem);
7302
7303 ins_cost(INSN_COST);
7304 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7305
7306 ins_encode( aarch64_enc_prefetchw(mem) );
7307
7308 ins_pipe(iload_prefetch);
7309 %}
7310
7311 // ---------------- volatile loads and stores ----------------
7312
7313 // Load Byte (8 bit signed)
7314 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7315 %{
7316 match(Set dst (LoadB mem));
7317
7318 ins_cost(VOLATILE_REF_COST);
7319 format %{ "ldarsb $dst, $mem\t# byte" %}
7320
7321 ins_encode(aarch64_enc_ldarsb(dst, mem));
7322
7323 ins_pipe(pipe_serial);
7324 %}
7325
7326 // Load Byte (8 bit signed) into long
7327 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7328 %{
7329 match(Set dst (ConvI2L (LoadB mem)));
7330
7331 ins_cost(VOLATILE_REF_COST);
7332 format %{ "ldarsb $dst, $mem\t# byte" %}
7333
7334 ins_encode(aarch64_enc_ldarsb(dst, mem));
7335
7336 ins_pipe(pipe_serial);
7337 %}
7338
7339 // Load Byte (8 bit unsigned)
7340 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7341 %{
7342 match(Set dst (LoadUB mem));
7343
7344 ins_cost(VOLATILE_REF_COST);
7345 format %{ "ldarb $dst, $mem\t# byte" %}
7346
7347 ins_encode(aarch64_enc_ldarb(dst, mem));
7348
7349 ins_pipe(pipe_serial);
7350 %}
7351
7352 // Load Byte (8 bit unsigned) into long
7353 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7354 %{
7355 match(Set dst (ConvI2L (LoadUB mem)));
7356
7357 ins_cost(VOLATILE_REF_COST);
7358 format %{ "ldarb $dst, $mem\t# byte" %}
7359
7360 ins_encode(aarch64_enc_ldarb(dst, mem));
7361
7362 ins_pipe(pipe_serial);
7363 %}
7364
7365 // Load Short (16 bit signed)
7366 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7367 %{
7368 match(Set dst (LoadS mem));
7369
7370 ins_cost(VOLATILE_REF_COST);
7371 format %{ "ldarshw $dst, $mem\t# short" %}
7372
7373 ins_encode(aarch64_enc_ldarshw(dst, mem));
7374
7375 ins_pipe(pipe_serial);
7376 %}
7377
7378 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7379 %{
7380 match(Set dst (LoadUS mem));
7381
7382 ins_cost(VOLATILE_REF_COST);
7383 format %{ "ldarhw $dst, $mem\t# short" %}
7384
7385 ins_encode(aarch64_enc_ldarhw(dst, mem));
7386
7387 ins_pipe(pipe_serial);
7388 %}
7389
7390 // Load Short/Char (16 bit unsigned) into long
7391 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7392 %{
7393 match(Set dst (ConvI2L (LoadUS mem)));
7394
7395 ins_cost(VOLATILE_REF_COST);
7396 format %{ "ldarh $dst, $mem\t# short" %}
7397
7398 ins_encode(aarch64_enc_ldarh(dst, mem));
7399
7400 ins_pipe(pipe_serial);
7401 %}
7402
7403 // Load Short/Char (16 bit signed) into long
7404 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7405 %{
7406 match(Set dst (ConvI2L (LoadS mem)));
7407
7408 ins_cost(VOLATILE_REF_COST);
7409 format %{ "ldarh $dst, $mem\t# short" %}
7410
7411 ins_encode(aarch64_enc_ldarsh(dst, mem));
7412
7413 ins_pipe(pipe_serial);
7414 %}
7415
7416 // Load Integer (32 bit signed)
7417 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7418 %{
7419 match(Set dst (LoadI mem));
7420
7421 ins_cost(VOLATILE_REF_COST);
7422 format %{ "ldarw $dst, $mem\t# int" %}
7423
7424 ins_encode(aarch64_enc_ldarw(dst, mem));
7425
7426 ins_pipe(pipe_serial);
7427 %}
7428
7429 // Load Integer (32 bit unsigned) into long
7430 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7431 %{
7432 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7433
7434 ins_cost(VOLATILE_REF_COST);
7435 format %{ "ldarw $dst, $mem\t# int" %}
7436
7437 ins_encode(aarch64_enc_ldarw(dst, mem));
7438
7439 ins_pipe(pipe_serial);
7440 %}
7441
7442 // Load Long (64 bit signed)
7443 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7444 %{
7445 match(Set dst (LoadL mem));
7446
7447 ins_cost(VOLATILE_REF_COST);
7448 format %{ "ldar $dst, $mem\t# int" %}
7449
7450 ins_encode(aarch64_enc_ldar(dst, mem));
7451
7452 ins_pipe(pipe_serial);
7453 %}
7454
7455 // Load Pointer
7456 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7457 %{
7458 match(Set dst (LoadP mem));
7459 predicate(n->as_Load()->barrier_data() == 0);
7460
7461 ins_cost(VOLATILE_REF_COST);
7462 format %{ "ldar $dst, $mem\t# ptr" %}
7463
7464 ins_encode(aarch64_enc_ldar(dst, mem));
7465
7466 ins_pipe(pipe_serial);
7467 %}
7468
7469 // Load Compressed Pointer
7470 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7471 %{
7472 match(Set dst (LoadN mem));
7473 predicate(n->as_Load()->barrier_data() == 0);
7474
7475 ins_cost(VOLATILE_REF_COST);
7476 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7477
7478 ins_encode(aarch64_enc_ldarw(dst, mem));
7479
7480 ins_pipe(pipe_serial);
7481 %}
7482
7483 // Load Float
7484 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7485 %{
7486 match(Set dst (LoadF mem));
7487
7488 ins_cost(VOLATILE_REF_COST);
7489 format %{ "ldars $dst, $mem\t# float" %}
7490
7491 ins_encode( aarch64_enc_fldars(dst, mem) );
7492
7493 ins_pipe(pipe_serial);
7494 %}
7495
7496 // Load Double
7497 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7498 %{
7499 match(Set dst (LoadD mem));
7500
7501 ins_cost(VOLATILE_REF_COST);
7502 format %{ "ldard $dst, $mem\t# double" %}
7503
7504 ins_encode( aarch64_enc_fldard(dst, mem) );
7505
7506 ins_pipe(pipe_serial);
7507 %}
7508
7509 // Store Byte
7510 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7511 %{
7512 match(Set mem (StoreB mem src));
7513
7514 ins_cost(VOLATILE_REF_COST);
7515 format %{ "stlrb $src, $mem\t# byte" %}
7516
7517 ins_encode(aarch64_enc_stlrb(src, mem));
7518
7519 ins_pipe(pipe_class_memory);
7520 %}
7521
7522 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7523 %{
7524 match(Set mem (StoreB mem zero));
7525
7526 ins_cost(VOLATILE_REF_COST);
7527 format %{ "stlrb zr, $mem\t# byte" %}
7528
7529 ins_encode(aarch64_enc_stlrb0(mem));
7530
7531 ins_pipe(pipe_class_memory);
7532 %}
7533
7534 // Store Char/Short
7535 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7536 %{
7537 match(Set mem (StoreC mem src));
7538
7539 ins_cost(VOLATILE_REF_COST);
7540 format %{ "stlrh $src, $mem\t# short" %}
7541
7542 ins_encode(aarch64_enc_stlrh(src, mem));
7543
7544 ins_pipe(pipe_class_memory);
7545 %}
7546
7547 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7548 %{
7549 match(Set mem (StoreC mem zero));
7550
7551 ins_cost(VOLATILE_REF_COST);
7552 format %{ "stlrh zr, $mem\t# short" %}
7553
7554 ins_encode(aarch64_enc_stlrh0(mem));
7555
7556 ins_pipe(pipe_class_memory);
7557 %}
7558
7559 // Store Integer
7560
7561 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7562 %{
7563 match(Set mem(StoreI mem src));
7564
7565 ins_cost(VOLATILE_REF_COST);
7566 format %{ "stlrw $src, $mem\t# int" %}
7567
7568 ins_encode(aarch64_enc_stlrw(src, mem));
7569
7570 ins_pipe(pipe_class_memory);
7571 %}
7572
7573 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7574 %{
7575 match(Set mem(StoreI mem zero));
7576
7577 ins_cost(VOLATILE_REF_COST);
7578 format %{ "stlrw zr, $mem\t# int" %}
7579
7580 ins_encode(aarch64_enc_stlrw0(mem));
7581
7582 ins_pipe(pipe_class_memory);
7583 %}
7584
7585 // Store Long (64 bit signed)
7586 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7587 %{
7588 match(Set mem (StoreL mem src));
7589
7590 ins_cost(VOLATILE_REF_COST);
7591 format %{ "stlr $src, $mem\t# int" %}
7592
7593 ins_encode(aarch64_enc_stlr(src, mem));
7594
7595 ins_pipe(pipe_class_memory);
7596 %}
7597
7598 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7599 %{
7600 match(Set mem (StoreL mem zero));
7601
7602 ins_cost(VOLATILE_REF_COST);
7603 format %{ "stlr zr, $mem\t# int" %}
7604
7605 ins_encode(aarch64_enc_stlr0(mem));
7606
7607 ins_pipe(pipe_class_memory);
7608 %}
7609
7610 // Store Pointer
7611 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7612 %{
7613 match(Set mem (StoreP mem src));
7614 predicate(n->as_Store()->barrier_data() == 0);
7615
7616 ins_cost(VOLATILE_REF_COST);
7617 format %{ "stlr $src, $mem\t# ptr" %}
7618
7619 ins_encode(aarch64_enc_stlr(src, mem));
7620
7621 ins_pipe(pipe_class_memory);
7622 %}
7623
7624 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7625 %{
7626 match(Set mem (StoreP mem zero));
7627 predicate(n->as_Store()->barrier_data() == 0);
7628
7629 ins_cost(VOLATILE_REF_COST);
7630 format %{ "stlr zr, $mem\t# ptr" %}
7631
7632 ins_encode(aarch64_enc_stlr0(mem));
7633
7634 ins_pipe(pipe_class_memory);
7635 %}
7636
7637 // Store Compressed Pointer
7638 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7639 %{
7640 match(Set mem (StoreN mem src));
7641 predicate(n->as_Store()->barrier_data() == 0);
7642
7643 ins_cost(VOLATILE_REF_COST);
7644 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7645
7646 ins_encode(aarch64_enc_stlrw(src, mem));
7647
7648 ins_pipe(pipe_class_memory);
7649 %}
7650
7651 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7652 %{
7653 match(Set mem (StoreN mem zero));
7654 predicate(n->as_Store()->barrier_data() == 0);
7655
7656 ins_cost(VOLATILE_REF_COST);
7657 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7658
7659 ins_encode(aarch64_enc_stlrw0(mem));
7660
7661 ins_pipe(pipe_class_memory);
7662 %}
7663
7664 // Store Float
7665 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7666 %{
7667 match(Set mem (StoreF mem src));
7668
7669 ins_cost(VOLATILE_REF_COST);
7670 format %{ "stlrs $src, $mem\t# float" %}
7671
7672 ins_encode( aarch64_enc_fstlrs(src, mem) );
7673
7674 ins_pipe(pipe_class_memory);
7675 %}
7676
7677 // TODO
7678 // implement storeImmF0 and storeFImmPacked
7679
7680 // Store Double
7681 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7682 %{
7683 match(Set mem (StoreD mem src));
7684
7685 ins_cost(VOLATILE_REF_COST);
7686 format %{ "stlrd $src, $mem\t# double" %}
7687
7688 ins_encode( aarch64_enc_fstlrd(src, mem) );
7689
7690 ins_pipe(pipe_class_memory);
7691 %}
7692
7693 // ---------------- end of volatile loads and stores ----------------
7694
7695 instruct cacheWB(indirect addr)
7696 %{
7697 predicate(VM_Version::supports_data_cache_line_flush());
7698 match(CacheWB addr);
7699
7700 ins_cost(100);
7701 format %{"cache wb $addr" %}
7702 ins_encode %{
7703 assert($addr->index_position() < 0, "should be");
7704 assert($addr$$disp == 0, "should be");
7705 __ cache_wb(Address($addr$$base$$Register, 0));
7706 %}
7707 ins_pipe(pipe_slow); // XXX
7708 %}
7709
7710 instruct cacheWBPreSync()
7711 %{
7712 predicate(VM_Version::supports_data_cache_line_flush());
7713 match(CacheWBPreSync);
7714
7715 ins_cost(100);
7716 format %{"cache wb presync" %}
7717 ins_encode %{
7718 __ cache_wbsync(true);
7719 %}
7720 ins_pipe(pipe_slow); // XXX
7721 %}
7722
7723 instruct cacheWBPostSync()
7724 %{
7725 predicate(VM_Version::supports_data_cache_line_flush());
7726 match(CacheWBPostSync);
7727
7728 ins_cost(100);
7729 format %{"cache wb postsync" %}
7730 ins_encode %{
7731 __ cache_wbsync(false);
7732 %}
7733 ins_pipe(pipe_slow); // XXX
7734 %}
7735
7736 // ============================================================================
7737 // BSWAP Instructions
7738
7739 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7740 match(Set dst (ReverseBytesI src));
7741
7742 ins_cost(INSN_COST);
7743 format %{ "revw $dst, $src" %}
7744
7745 ins_encode %{
7746 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7747 %}
7748
7749 ins_pipe(ialu_reg);
7750 %}
7751
7752 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7753 match(Set dst (ReverseBytesL src));
7754
7755 ins_cost(INSN_COST);
7756 format %{ "rev $dst, $src" %}
7757
7758 ins_encode %{
7759 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7760 %}
7761
7762 ins_pipe(ialu_reg);
7763 %}
7764
7765 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7766 match(Set dst (ReverseBytesUS src));
7767
7768 ins_cost(INSN_COST);
7769 format %{ "rev16w $dst, $src" %}
7770
7771 ins_encode %{
7772 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7773 %}
7774
7775 ins_pipe(ialu_reg);
7776 %}
7777
7778 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7779 match(Set dst (ReverseBytesS src));
7780
7781 ins_cost(INSN_COST);
7782 format %{ "rev16w $dst, $src\n\t"
7783 "sbfmw $dst, $dst, #0, #15" %}
7784
7785 ins_encode %{
7786 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7787 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7788 %}
7789
7790 ins_pipe(ialu_reg);
7791 %}
7792
7793 // ============================================================================
7794 // Zero Count Instructions
7795
7796 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7797 match(Set dst (CountLeadingZerosI src));
7798
7799 ins_cost(INSN_COST);
7800 format %{ "clzw $dst, $src" %}
7801 ins_encode %{
7802 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7803 %}
7804
7805 ins_pipe(ialu_reg);
7806 %}
7807
7808 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7809 match(Set dst (CountLeadingZerosL src));
7810
7811 ins_cost(INSN_COST);
7812 format %{ "clz $dst, $src" %}
7813 ins_encode %{
7814 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7815 %}
7816
7817 ins_pipe(ialu_reg);
7818 %}
7819
7820 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7821 match(Set dst (CountTrailingZerosI src));
7822
7823 ins_cost(INSN_COST * 2);
7824 format %{ "rbitw $dst, $src\n\t"
7825 "clzw $dst, $dst" %}
7826 ins_encode %{
7827 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7828 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7829 %}
7830
7831 ins_pipe(ialu_reg);
7832 %}
7833
7834 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7835 match(Set dst (CountTrailingZerosL src));
7836
7837 ins_cost(INSN_COST * 2);
7838 format %{ "rbit $dst, $src\n\t"
7839 "clz $dst, $dst" %}
7840 ins_encode %{
7841 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7842 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7843 %}
7844
7845 ins_pipe(ialu_reg);
7846 %}
7847
7848 //---------- Population Count Instructions -------------------------------------
7849 //
7850
7851 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7852 match(Set dst (PopCountI src));
7853 effect(TEMP tmp);
7854 ins_cost(INSN_COST * 13);
7855
7856 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7857 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7858 "addv $tmp, $tmp\t# vector (8B)\n\t"
7859 "mov $dst, $tmp\t# vector (1D)" %}
7860 ins_encode %{
7861 __ fmovs($tmp$$FloatRegister, $src$$Register);
7862 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7863 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7864 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7865 %}
7866
7867 ins_pipe(pipe_class_default);
7868 %}
7869
7870 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7871 match(Set dst (PopCountI (LoadI mem)));
7872 effect(TEMP tmp);
7873 ins_cost(INSN_COST * 13);
7874
7875 format %{ "ldrs $tmp, $mem\n\t"
7876 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7877 "addv $tmp, $tmp\t# vector (8B)\n\t"
7878 "mov $dst, $tmp\t# vector (1D)" %}
7879 ins_encode %{
7880 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7881 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7882 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7883 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7884 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7885 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7886 %}
7887
7888 ins_pipe(pipe_class_default);
7889 %}
7890
7891 // Note: Long.bitCount(long) returns an int.
7892 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7893 match(Set dst (PopCountL src));
7894 effect(TEMP tmp);
7895 ins_cost(INSN_COST * 13);
7896
7897 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7898 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7899 "addv $tmp, $tmp\t# vector (8B)\n\t"
7900 "mov $dst, $tmp\t# vector (1D)" %}
7901 ins_encode %{
7902 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7903 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7904 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7905 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7906 %}
7907
7908 ins_pipe(pipe_class_default);
7909 %}
7910
7911 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7912 match(Set dst (PopCountL (LoadL mem)));
7913 effect(TEMP tmp);
7914 ins_cost(INSN_COST * 13);
7915
7916 format %{ "ldrd $tmp, $mem\n\t"
7917 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7918 "addv $tmp, $tmp\t# vector (8B)\n\t"
7919 "mov $dst, $tmp\t# vector (1D)" %}
7920 ins_encode %{
7921 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7922 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7923 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7924 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7925 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7926 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7927 %}
7928
7929 ins_pipe(pipe_class_default);
7930 %}
7931
7932 // ============================================================================
7933 // VerifyVectorAlignment Instruction
7934
7935 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7936 match(Set addr (VerifyVectorAlignment addr mask));
7937 effect(KILL cr);
7938 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7939 ins_encode %{
7940 Label Lskip;
7941 // check if masked bits of addr are zero
7942 __ tst($addr$$Register, $mask$$constant);
7943 __ br(Assembler::EQ, Lskip);
7944 __ stop("verify_vector_alignment found a misaligned vector memory access");
7945 __ bind(Lskip);
7946 %}
7947 ins_pipe(pipe_slow);
7948 %}
7949
7950 // ============================================================================
7951 // MemBar Instruction
7952
7953 instruct load_fence() %{
7954 match(LoadFence);
7955 ins_cost(VOLATILE_REF_COST);
7956
7957 format %{ "load_fence" %}
7958
7959 ins_encode %{
7960 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7961 %}
7962 ins_pipe(pipe_serial);
7963 %}
7964
7965 instruct unnecessary_membar_acquire() %{
7966 predicate(unnecessary_acquire(n));
7967 match(MemBarAcquire);
7968 ins_cost(0);
7969
7970 format %{ "membar_acquire (elided)" %}
7971
7972 ins_encode %{
7973 __ block_comment("membar_acquire (elided)");
7974 %}
7975
7976 ins_pipe(pipe_class_empty);
7977 %}
7978
7979 instruct membar_acquire() %{
7980 match(MemBarAcquire);
7981 ins_cost(VOLATILE_REF_COST);
7982
7983 format %{ "membar_acquire\n\t"
7984 "dmb ishld" %}
7985
7986 ins_encode %{
7987 __ block_comment("membar_acquire");
7988 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7989 %}
7990
7991 ins_pipe(pipe_serial);
7992 %}
7993
7994
7995 instruct membar_acquire_lock() %{
7996 match(MemBarAcquireLock);
7997 ins_cost(VOLATILE_REF_COST);
7998
7999 format %{ "membar_acquire_lock (elided)" %}
8000
8001 ins_encode %{
8002 __ block_comment("membar_acquire_lock (elided)");
8003 %}
8004
8005 ins_pipe(pipe_serial);
8006 %}
8007
8008 instruct store_fence() %{
8009 match(StoreFence);
8010 ins_cost(VOLATILE_REF_COST);
8011
8012 format %{ "store_fence" %}
8013
8014 ins_encode %{
8015 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8016 %}
8017 ins_pipe(pipe_serial);
8018 %}
8019
8020 instruct unnecessary_membar_release() %{
8021 predicate(unnecessary_release(n));
8022 match(MemBarRelease);
8023 ins_cost(0);
8024
8025 format %{ "membar_release (elided)" %}
8026
8027 ins_encode %{
8028 __ block_comment("membar_release (elided)");
8029 %}
8030 ins_pipe(pipe_serial);
8031 %}
8032
8033 instruct membar_release() %{
8034 match(MemBarRelease);
8035 ins_cost(VOLATILE_REF_COST);
8036
8037 format %{ "membar_release\n\t"
8038 "dmb ishst\n\tdmb ishld" %}
8039
8040 ins_encode %{
8041 __ block_comment("membar_release");
8042 // These will be merged if AlwaysMergeDMB is enabled.
8043 __ membar(Assembler::StoreStore);
8044 __ membar(Assembler::LoadStore);
8045 %}
8046 ins_pipe(pipe_serial);
8047 %}
8048
8049 instruct membar_storestore() %{
8050 match(MemBarStoreStore);
8051 match(StoreStoreFence);
8052 ins_cost(VOLATILE_REF_COST);
8053
8054 format %{ "MEMBAR-store-store" %}
8055
8056 ins_encode %{
8057 __ membar(Assembler::StoreStore);
8058 %}
8059 ins_pipe(pipe_serial);
8060 %}
8061
8062 instruct membar_release_lock() %{
8063 match(MemBarReleaseLock);
8064 ins_cost(VOLATILE_REF_COST);
8065
8066 format %{ "membar_release_lock (elided)" %}
8067
8068 ins_encode %{
8069 __ block_comment("membar_release_lock (elided)");
8070 %}
8071
8072 ins_pipe(pipe_serial);
8073 %}
8074
8075 instruct unnecessary_membar_volatile() %{
8076 predicate(unnecessary_volatile(n));
8077 match(MemBarVolatile);
8078 ins_cost(0);
8079
8080 format %{ "membar_volatile (elided)" %}
8081
8082 ins_encode %{
8083 __ block_comment("membar_volatile (elided)");
8084 %}
8085
8086 ins_pipe(pipe_serial);
8087 %}
8088
8089 instruct membar_volatile() %{
8090 match(MemBarVolatile);
8091 ins_cost(VOLATILE_REF_COST*100);
8092
8093 format %{ "membar_volatile\n\t"
8094 "dmb ish"%}
8095
8096 ins_encode %{
8097 __ block_comment("membar_volatile");
8098 __ membar(Assembler::StoreLoad);
8099 %}
8100
8101 ins_pipe(pipe_serial);
8102 %}
8103
8104 // ============================================================================
8105 // Cast/Convert Instructions
8106
8107 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8108 match(Set dst (CastX2P src));
8109
8110 ins_cost(INSN_COST);
8111 format %{ "mov $dst, $src\t# long -> ptr" %}
8112
8113 ins_encode %{
8114 if ($dst$$reg != $src$$reg) {
8115 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8116 }
8117 %}
8118
8119 ins_pipe(ialu_reg);
8120 %}
8121
8122 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8123 match(Set dst (CastP2X src));
8124
8125 ins_cost(INSN_COST);
8126 format %{ "mov $dst, $src\t# ptr -> long" %}
8127
8128 ins_encode %{
8129 if ($dst$$reg != $src$$reg) {
8130 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8131 }
8132 %}
8133
8134 ins_pipe(ialu_reg);
8135 %}
8136
8137 // Convert oop into int for vectors alignment masking
8138 instruct convP2I(iRegINoSp dst, iRegP src) %{
8139 match(Set dst (ConvL2I (CastP2X src)));
8140
8141 ins_cost(INSN_COST);
8142 format %{ "movw $dst, $src\t# ptr -> int" %}
8143 ins_encode %{
8144 __ movw($dst$$Register, $src$$Register);
8145 %}
8146
8147 ins_pipe(ialu_reg);
8148 %}
8149
8150 // Convert compressed oop into int for vectors alignment masking
8151 // in case of 32bit oops (heap < 4Gb).
8152 instruct convN2I(iRegINoSp dst, iRegN src)
8153 %{
8154 predicate(CompressedOops::shift() == 0);
8155 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8156
8157 ins_cost(INSN_COST);
8158 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8159 ins_encode %{
8160 __ movw($dst$$Register, $src$$Register);
8161 %}
8162
8163 ins_pipe(ialu_reg);
8164 %}
8165
8166
8167 // Convert oop pointer into compressed form
8168 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8169 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8170 match(Set dst (EncodeP src));
8171 effect(KILL cr);
8172 ins_cost(INSN_COST * 3);
8173 format %{ "encode_heap_oop $dst, $src" %}
8174 ins_encode %{
8175 Register s = $src$$Register;
8176 Register d = $dst$$Register;
8177 __ encode_heap_oop(d, s);
8178 %}
8179 ins_pipe(ialu_reg);
8180 %}
8181
8182 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8183 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8184 match(Set dst (EncodeP src));
8185 ins_cost(INSN_COST * 3);
8186 format %{ "encode_heap_oop_not_null $dst, $src" %}
8187 ins_encode %{
8188 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8189 %}
8190 ins_pipe(ialu_reg);
8191 %}
8192
8193 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8194 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8195 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8196 match(Set dst (DecodeN src));
8197 ins_cost(INSN_COST * 3);
8198 format %{ "decode_heap_oop $dst, $src" %}
8199 ins_encode %{
8200 Register s = $src$$Register;
8201 Register d = $dst$$Register;
8202 __ decode_heap_oop(d, s);
8203 %}
8204 ins_pipe(ialu_reg);
8205 %}
8206
8207 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8208 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8209 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8210 match(Set dst (DecodeN src));
8211 ins_cost(INSN_COST * 3);
8212 format %{ "decode_heap_oop_not_null $dst, $src" %}
8213 ins_encode %{
8214 Register s = $src$$Register;
8215 Register d = $dst$$Register;
8216 __ decode_heap_oop_not_null(d, s);
8217 %}
8218 ins_pipe(ialu_reg);
8219 %}
8220
8221 // n.b. AArch64 implementations of encode_klass_not_null and
8222 // decode_klass_not_null do not modify the flags register so, unlike
8223 // Intel, we don't kill CR as a side effect here
8224
8225 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8226 match(Set dst (EncodePKlass src));
8227
8228 ins_cost(INSN_COST * 3);
8229 format %{ "encode_klass_not_null $dst,$src" %}
8230
8231 ins_encode %{
8232 Register src_reg = as_Register($src$$reg);
8233 Register dst_reg = as_Register($dst$$reg);
8234 __ encode_klass_not_null(dst_reg, src_reg);
8235 %}
8236
8237 ins_pipe(ialu_reg);
8238 %}
8239
8240 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8241 match(Set dst (DecodeNKlass src));
8242
8243 ins_cost(INSN_COST * 3);
8244 format %{ "decode_klass_not_null $dst,$src" %}
8245
8246 ins_encode %{
8247 Register src_reg = as_Register($src$$reg);
8248 Register dst_reg = as_Register($dst$$reg);
8249 if (dst_reg != src_reg) {
8250 __ decode_klass_not_null(dst_reg, src_reg);
8251 } else {
8252 __ decode_klass_not_null(dst_reg);
8253 }
8254 %}
8255
8256 ins_pipe(ialu_reg);
8257 %}
8258
8259 instruct checkCastPP(iRegPNoSp dst)
8260 %{
8261 match(Set dst (CheckCastPP dst));
8262
8263 size(0);
8264 format %{ "# checkcastPP of $dst" %}
8265 ins_encode(/* empty encoding */);
8266 ins_pipe(pipe_class_empty);
8267 %}
8268
8269 instruct castPP(iRegPNoSp dst)
8270 %{
8271 match(Set dst (CastPP dst));
8272
8273 size(0);
8274 format %{ "# castPP of $dst" %}
8275 ins_encode(/* empty encoding */);
8276 ins_pipe(pipe_class_empty);
8277 %}
8278
8279 instruct castII(iRegI dst)
8280 %{
8281 predicate(VerifyConstraintCasts == 0);
8282 match(Set dst (CastII dst));
8283
8284 size(0);
8285 format %{ "# castII of $dst" %}
8286 ins_encode(/* empty encoding */);
8287 ins_cost(0);
8288 ins_pipe(pipe_class_empty);
8289 %}
8290
8291 instruct castII_checked(iRegI dst, rFlagsReg cr)
8292 %{
8293 predicate(VerifyConstraintCasts > 0);
8294 match(Set dst (CastII dst));
8295 effect(KILL cr);
8296
8297 format %{ "# castII_checked of $dst" %}
8298 ins_encode %{
8299 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8300 %}
8301 ins_pipe(pipe_slow);
8302 %}
8303
8304 instruct castLL(iRegL dst)
8305 %{
8306 predicate(VerifyConstraintCasts == 0);
8307 match(Set dst (CastLL dst));
8308
8309 size(0);
8310 format %{ "# castLL of $dst" %}
8311 ins_encode(/* empty encoding */);
8312 ins_cost(0);
8313 ins_pipe(pipe_class_empty);
8314 %}
8315
8316 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8317 %{
8318 predicate(VerifyConstraintCasts > 0);
8319 match(Set dst (CastLL dst));
8320 effect(KILL cr);
8321
8322 format %{ "# castLL_checked of $dst" %}
8323 ins_encode %{
8324 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8325 %}
8326 ins_pipe(pipe_slow);
8327 %}
8328
8329 instruct castHH(vRegF dst)
8330 %{
8331 match(Set dst (CastHH dst));
8332 size(0);
8333 format %{ "# castHH of $dst" %}
8334 ins_encode(/* empty encoding */);
8335 ins_cost(0);
8336 ins_pipe(pipe_class_empty);
8337 %}
8338
8339 instruct castFF(vRegF dst)
8340 %{
8341 match(Set dst (CastFF dst));
8342
8343 size(0);
8344 format %{ "# castFF of $dst" %}
8345 ins_encode(/* empty encoding */);
8346 ins_cost(0);
8347 ins_pipe(pipe_class_empty);
8348 %}
8349
8350 instruct castDD(vRegD dst)
8351 %{
8352 match(Set dst (CastDD dst));
8353
8354 size(0);
8355 format %{ "# castDD of $dst" %}
8356 ins_encode(/* empty encoding */);
8357 ins_cost(0);
8358 ins_pipe(pipe_class_empty);
8359 %}
8360
8361 instruct castVV(vReg dst)
8362 %{
8363 match(Set dst (CastVV dst));
8364
8365 size(0);
8366 format %{ "# castVV of $dst" %}
8367 ins_encode(/* empty encoding */);
8368 ins_cost(0);
8369 ins_pipe(pipe_class_empty);
8370 %}
8371
8372 instruct castVVMask(pRegGov dst)
8373 %{
8374 match(Set dst (CastVV dst));
8375
8376 size(0);
8377 format %{ "# castVV of $dst" %}
8378 ins_encode(/* empty encoding */);
8379 ins_cost(0);
8380 ins_pipe(pipe_class_empty);
8381 %}
8382
8383 // ============================================================================
8384 // Atomic operation instructions
8385 //
8386
8387 // standard CompareAndSwapX when we are using barriers
8388 // these have higher priority than the rules selected by a predicate
8389
8390 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8391 // can't match them
8392
8393 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8394
8395 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8396 ins_cost(2 * VOLATILE_REF_COST);
8397
8398 effect(KILL cr);
8399
8400 format %{
8401 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8402 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8403 %}
8404
8405 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8406 aarch64_enc_cset_eq(res));
8407
8408 ins_pipe(pipe_slow);
8409 %}
8410
8411 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8412
8413 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8414 ins_cost(2 * VOLATILE_REF_COST);
8415
8416 effect(KILL cr);
8417
8418 format %{
8419 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8420 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8421 %}
8422
8423 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8424 aarch64_enc_cset_eq(res));
8425
8426 ins_pipe(pipe_slow);
8427 %}
8428
8429 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8430
8431 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8432 ins_cost(2 * VOLATILE_REF_COST);
8433
8434 effect(KILL cr);
8435
8436 format %{
8437 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8438 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8439 %}
8440
8441 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8442 aarch64_enc_cset_eq(res));
8443
8444 ins_pipe(pipe_slow);
8445 %}
8446
8447 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8448
8449 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8450 ins_cost(2 * VOLATILE_REF_COST);
8451
8452 effect(KILL cr);
8453
8454 format %{
8455 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8456 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8457 %}
8458
8459 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8460 aarch64_enc_cset_eq(res));
8461
8462 ins_pipe(pipe_slow);
8463 %}
8464
8465 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8466
8467 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8468 predicate(n->as_LoadStore()->barrier_data() == 0);
8469 ins_cost(2 * VOLATILE_REF_COST);
8470
8471 effect(KILL cr);
8472
8473 format %{
8474 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8475 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8476 %}
8477
8478 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8479 aarch64_enc_cset_eq(res));
8480
8481 ins_pipe(pipe_slow);
8482 %}
8483
8484 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8485
8486 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8487 predicate(n->as_LoadStore()->barrier_data() == 0);
8488 ins_cost(2 * VOLATILE_REF_COST);
8489
8490 effect(KILL cr);
8491
8492 format %{
8493 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8494 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8495 %}
8496
8497 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8498 aarch64_enc_cset_eq(res));
8499
8500 ins_pipe(pipe_slow);
8501 %}
8502
8503 // alternative CompareAndSwapX when we are eliding barriers
8504
8505 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8506
8507 predicate(needs_acquiring_load_exclusive(n));
8508 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8509 ins_cost(VOLATILE_REF_COST);
8510
8511 effect(KILL cr);
8512
8513 format %{
8514 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8515 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8516 %}
8517
8518 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8519 aarch64_enc_cset_eq(res));
8520
8521 ins_pipe(pipe_slow);
8522 %}
8523
8524 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8525
8526 predicate(needs_acquiring_load_exclusive(n));
8527 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8528 ins_cost(VOLATILE_REF_COST);
8529
8530 effect(KILL cr);
8531
8532 format %{
8533 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8534 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8535 %}
8536
8537 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8538 aarch64_enc_cset_eq(res));
8539
8540 ins_pipe(pipe_slow);
8541 %}
8542
8543 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8544
8545 predicate(needs_acquiring_load_exclusive(n));
8546 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8547 ins_cost(VOLATILE_REF_COST);
8548
8549 effect(KILL cr);
8550
8551 format %{
8552 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8553 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8554 %}
8555
8556 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8557 aarch64_enc_cset_eq(res));
8558
8559 ins_pipe(pipe_slow);
8560 %}
8561
8562 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8563
8564 predicate(needs_acquiring_load_exclusive(n));
8565 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8566 ins_cost(VOLATILE_REF_COST);
8567
8568 effect(KILL cr);
8569
8570 format %{
8571 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8572 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8573 %}
8574
8575 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8576 aarch64_enc_cset_eq(res));
8577
8578 ins_pipe(pipe_slow);
8579 %}
8580
8581 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8582
8583 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8584 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8585 ins_cost(VOLATILE_REF_COST);
8586
8587 effect(KILL cr);
8588
8589 format %{
8590 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8591 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8592 %}
8593
8594 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8595 aarch64_enc_cset_eq(res));
8596
8597 ins_pipe(pipe_slow);
8598 %}
8599
8600 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8601
8602 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8603 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8604 ins_cost(VOLATILE_REF_COST);
8605
8606 effect(KILL cr);
8607
8608 format %{
8609 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8610 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8611 %}
8612
8613 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8614 aarch64_enc_cset_eq(res));
8615
8616 ins_pipe(pipe_slow);
8617 %}
8618
8619
8620 // ---------------------------------------------------------------------
8621
8622 // BEGIN This section of the file is automatically generated. Do not edit --------------
8623
8624 // Sundry CAS operations. Note that release is always true,
8625 // regardless of the memory ordering of the CAS. This is because we
8626 // need the volatile case to be sequentially consistent but there is
8627 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8628 // can't check the type of memory ordering here, so we always emit a
8629 // STLXR.
8630
8631 // This section is generated from cas.m4
8632
8633
8634 // This pattern is generated automatically from cas.m4.
8635 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8636 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8637 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8638 ins_cost(2 * VOLATILE_REF_COST);
8639 effect(TEMP_DEF res, KILL cr);
8640 format %{
8641 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8642 %}
8643 ins_encode %{
8644 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8645 Assembler::byte, /*acquire*/ false, /*release*/ true,
8646 /*weak*/ false, $res$$Register);
8647 __ sxtbw($res$$Register, $res$$Register);
8648 %}
8649 ins_pipe(pipe_slow);
8650 %}
8651
8652 // This pattern is generated automatically from cas.m4.
8653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8654 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8655 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8656 ins_cost(2 * VOLATILE_REF_COST);
8657 effect(TEMP_DEF res, KILL cr);
8658 format %{
8659 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8660 %}
8661 ins_encode %{
8662 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8663 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8664 /*weak*/ false, $res$$Register);
8665 __ sxthw($res$$Register, $res$$Register);
8666 %}
8667 ins_pipe(pipe_slow);
8668 %}
8669
8670 // This pattern is generated automatically from cas.m4.
8671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8672 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8673 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8674 ins_cost(2 * VOLATILE_REF_COST);
8675 effect(TEMP_DEF res, KILL cr);
8676 format %{
8677 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8678 %}
8679 ins_encode %{
8680 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8681 Assembler::word, /*acquire*/ false, /*release*/ true,
8682 /*weak*/ false, $res$$Register);
8683 %}
8684 ins_pipe(pipe_slow);
8685 %}
8686
8687 // This pattern is generated automatically from cas.m4.
8688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8689 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8690 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8691 ins_cost(2 * VOLATILE_REF_COST);
8692 effect(TEMP_DEF res, KILL cr);
8693 format %{
8694 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8695 %}
8696 ins_encode %{
8697 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8698 Assembler::xword, /*acquire*/ false, /*release*/ true,
8699 /*weak*/ false, $res$$Register);
8700 %}
8701 ins_pipe(pipe_slow);
8702 %}
8703
8704 // This pattern is generated automatically from cas.m4.
8705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8706 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8707 predicate(n->as_LoadStore()->barrier_data() == 0);
8708 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8709 ins_cost(2 * VOLATILE_REF_COST);
8710 effect(TEMP_DEF res, KILL cr);
8711 format %{
8712 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8713 %}
8714 ins_encode %{
8715 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8716 Assembler::word, /*acquire*/ false, /*release*/ true,
8717 /*weak*/ false, $res$$Register);
8718 %}
8719 ins_pipe(pipe_slow);
8720 %}
8721
8722 // This pattern is generated automatically from cas.m4.
8723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8724 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8725 predicate(n->as_LoadStore()->barrier_data() == 0);
8726 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8727 ins_cost(2 * VOLATILE_REF_COST);
8728 effect(TEMP_DEF res, KILL cr);
8729 format %{
8730 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8731 %}
8732 ins_encode %{
8733 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8734 Assembler::xword, /*acquire*/ false, /*release*/ true,
8735 /*weak*/ false, $res$$Register);
8736 %}
8737 ins_pipe(pipe_slow);
8738 %}
8739
8740 // This pattern is generated automatically from cas.m4.
8741 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8742 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8743 predicate(needs_acquiring_load_exclusive(n));
8744 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8745 ins_cost(VOLATILE_REF_COST);
8746 effect(TEMP_DEF res, KILL cr);
8747 format %{
8748 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8749 %}
8750 ins_encode %{
8751 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8752 Assembler::byte, /*acquire*/ true, /*release*/ true,
8753 /*weak*/ false, $res$$Register);
8754 __ sxtbw($res$$Register, $res$$Register);
8755 %}
8756 ins_pipe(pipe_slow);
8757 %}
8758
8759 // This pattern is generated automatically from cas.m4.
8760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8761 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8762 predicate(needs_acquiring_load_exclusive(n));
8763 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8764 ins_cost(VOLATILE_REF_COST);
8765 effect(TEMP_DEF res, KILL cr);
8766 format %{
8767 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8768 %}
8769 ins_encode %{
8770 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8771 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8772 /*weak*/ false, $res$$Register);
8773 __ sxthw($res$$Register, $res$$Register);
8774 %}
8775 ins_pipe(pipe_slow);
8776 %}
8777
8778 // This pattern is generated automatically from cas.m4.
8779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8780 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8781 predicate(needs_acquiring_load_exclusive(n));
8782 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8783 ins_cost(VOLATILE_REF_COST);
8784 effect(TEMP_DEF res, KILL cr);
8785 format %{
8786 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8787 %}
8788 ins_encode %{
8789 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8790 Assembler::word, /*acquire*/ true, /*release*/ true,
8791 /*weak*/ false, $res$$Register);
8792 %}
8793 ins_pipe(pipe_slow);
8794 %}
8795
8796 // This pattern is generated automatically from cas.m4.
8797 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8798 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8799 predicate(needs_acquiring_load_exclusive(n));
8800 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8801 ins_cost(VOLATILE_REF_COST);
8802 effect(TEMP_DEF res, KILL cr);
8803 format %{
8804 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8805 %}
8806 ins_encode %{
8807 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8808 Assembler::xword, /*acquire*/ true, /*release*/ true,
8809 /*weak*/ false, $res$$Register);
8810 %}
8811 ins_pipe(pipe_slow);
8812 %}
8813
8814 // This pattern is generated automatically from cas.m4.
8815 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8816 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8817 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8818 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8819 ins_cost(VOLATILE_REF_COST);
8820 effect(TEMP_DEF res, KILL cr);
8821 format %{
8822 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8823 %}
8824 ins_encode %{
8825 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8826 Assembler::word, /*acquire*/ true, /*release*/ true,
8827 /*weak*/ false, $res$$Register);
8828 %}
8829 ins_pipe(pipe_slow);
8830 %}
8831
8832 // This pattern is generated automatically from cas.m4.
8833 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8834 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8835 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8836 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8837 ins_cost(VOLATILE_REF_COST);
8838 effect(TEMP_DEF res, KILL cr);
8839 format %{
8840 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8841 %}
8842 ins_encode %{
8843 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8844 Assembler::xword, /*acquire*/ true, /*release*/ true,
8845 /*weak*/ false, $res$$Register);
8846 %}
8847 ins_pipe(pipe_slow);
8848 %}
8849
8850 // This pattern is generated automatically from cas.m4.
8851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8852 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8853 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8854 ins_cost(2 * VOLATILE_REF_COST);
8855 effect(KILL cr);
8856 format %{
8857 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8858 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8859 %}
8860 ins_encode %{
8861 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8862 Assembler::byte, /*acquire*/ false, /*release*/ true,
8863 /*weak*/ true, noreg);
8864 __ csetw($res$$Register, Assembler::EQ);
8865 %}
8866 ins_pipe(pipe_slow);
8867 %}
8868
8869 // This pattern is generated automatically from cas.m4.
8870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8871 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8872 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8873 ins_cost(2 * VOLATILE_REF_COST);
8874 effect(KILL cr);
8875 format %{
8876 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8877 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8878 %}
8879 ins_encode %{
8880 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8881 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8882 /*weak*/ true, noreg);
8883 __ csetw($res$$Register, Assembler::EQ);
8884 %}
8885 ins_pipe(pipe_slow);
8886 %}
8887
8888 // This pattern is generated automatically from cas.m4.
8889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8890 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8891 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8892 ins_cost(2 * VOLATILE_REF_COST);
8893 effect(KILL cr);
8894 format %{
8895 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8896 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8897 %}
8898 ins_encode %{
8899 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8900 Assembler::word, /*acquire*/ false, /*release*/ true,
8901 /*weak*/ true, noreg);
8902 __ csetw($res$$Register, Assembler::EQ);
8903 %}
8904 ins_pipe(pipe_slow);
8905 %}
8906
8907 // This pattern is generated automatically from cas.m4.
8908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8909 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8910 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8911 ins_cost(2 * VOLATILE_REF_COST);
8912 effect(KILL cr);
8913 format %{
8914 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8915 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8916 %}
8917 ins_encode %{
8918 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8919 Assembler::xword, /*acquire*/ false, /*release*/ true,
8920 /*weak*/ true, noreg);
8921 __ csetw($res$$Register, Assembler::EQ);
8922 %}
8923 ins_pipe(pipe_slow);
8924 %}
8925
8926 // This pattern is generated automatically from cas.m4.
8927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8928 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8929 predicate(n->as_LoadStore()->barrier_data() == 0);
8930 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8931 ins_cost(2 * VOLATILE_REF_COST);
8932 effect(KILL cr);
8933 format %{
8934 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8935 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8936 %}
8937 ins_encode %{
8938 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8939 Assembler::word, /*acquire*/ false, /*release*/ true,
8940 /*weak*/ true, noreg);
8941 __ csetw($res$$Register, Assembler::EQ);
8942 %}
8943 ins_pipe(pipe_slow);
8944 %}
8945
8946 // This pattern is generated automatically from cas.m4.
8947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8948 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8949 predicate(n->as_LoadStore()->barrier_data() == 0);
8950 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8951 ins_cost(2 * VOLATILE_REF_COST);
8952 effect(KILL cr);
8953 format %{
8954 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8955 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8956 %}
8957 ins_encode %{
8958 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8959 Assembler::xword, /*acquire*/ false, /*release*/ true,
8960 /*weak*/ true, noreg);
8961 __ csetw($res$$Register, Assembler::EQ);
8962 %}
8963 ins_pipe(pipe_slow);
8964 %}
8965
8966 // This pattern is generated automatically from cas.m4.
8967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8968 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8969 predicate(needs_acquiring_load_exclusive(n));
8970 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8971 ins_cost(VOLATILE_REF_COST);
8972 effect(KILL cr);
8973 format %{
8974 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8975 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8976 %}
8977 ins_encode %{
8978 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8979 Assembler::byte, /*acquire*/ true, /*release*/ true,
8980 /*weak*/ true, noreg);
8981 __ csetw($res$$Register, Assembler::EQ);
8982 %}
8983 ins_pipe(pipe_slow);
8984 %}
8985
8986 // This pattern is generated automatically from cas.m4.
8987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8988 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8989 predicate(needs_acquiring_load_exclusive(n));
8990 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8991 ins_cost(VOLATILE_REF_COST);
8992 effect(KILL cr);
8993 format %{
8994 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8995 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8996 %}
8997 ins_encode %{
8998 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8999 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9000 /*weak*/ true, noreg);
9001 __ csetw($res$$Register, Assembler::EQ);
9002 %}
9003 ins_pipe(pipe_slow);
9004 %}
9005
9006 // This pattern is generated automatically from cas.m4.
9007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9008 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9009 predicate(needs_acquiring_load_exclusive(n));
9010 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9011 ins_cost(VOLATILE_REF_COST);
9012 effect(KILL cr);
9013 format %{
9014 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9015 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9016 %}
9017 ins_encode %{
9018 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9019 Assembler::word, /*acquire*/ true, /*release*/ true,
9020 /*weak*/ true, noreg);
9021 __ csetw($res$$Register, Assembler::EQ);
9022 %}
9023 ins_pipe(pipe_slow);
9024 %}
9025
9026 // This pattern is generated automatically from cas.m4.
9027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9028 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9029 predicate(needs_acquiring_load_exclusive(n));
9030 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9031 ins_cost(VOLATILE_REF_COST);
9032 effect(KILL cr);
9033 format %{
9034 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9035 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9036 %}
9037 ins_encode %{
9038 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9039 Assembler::xword, /*acquire*/ true, /*release*/ true,
9040 /*weak*/ true, noreg);
9041 __ csetw($res$$Register, Assembler::EQ);
9042 %}
9043 ins_pipe(pipe_slow);
9044 %}
9045
9046 // This pattern is generated automatically from cas.m4.
9047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9048 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9049 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9050 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9051 ins_cost(VOLATILE_REF_COST);
9052 effect(KILL cr);
9053 format %{
9054 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9055 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9056 %}
9057 ins_encode %{
9058 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9059 Assembler::word, /*acquire*/ true, /*release*/ true,
9060 /*weak*/ true, noreg);
9061 __ csetw($res$$Register, Assembler::EQ);
9062 %}
9063 ins_pipe(pipe_slow);
9064 %}
9065
9066 // This pattern is generated automatically from cas.m4.
9067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9068 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9069 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9070 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9071 ins_cost(VOLATILE_REF_COST);
9072 effect(KILL cr);
9073 format %{
9074 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9075 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9076 %}
9077 ins_encode %{
9078 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9079 Assembler::xword, /*acquire*/ true, /*release*/ true,
9080 /*weak*/ true, noreg);
9081 __ csetw($res$$Register, Assembler::EQ);
9082 %}
9083 ins_pipe(pipe_slow);
9084 %}
9085
9086 // END This section of the file is automatically generated. Do not edit --------------
9087 // ---------------------------------------------------------------------
9088
9089 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9090 match(Set prev (GetAndSetI mem newv));
9091 ins_cost(2 * VOLATILE_REF_COST);
9092 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9093 ins_encode %{
9094 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9095 %}
9096 ins_pipe(pipe_serial);
9097 %}
9098
9099 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9100 match(Set prev (GetAndSetL mem newv));
9101 ins_cost(2 * VOLATILE_REF_COST);
9102 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9103 ins_encode %{
9104 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9105 %}
9106 ins_pipe(pipe_serial);
9107 %}
9108
9109 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9110 predicate(n->as_LoadStore()->barrier_data() == 0);
9111 match(Set prev (GetAndSetN mem newv));
9112 ins_cost(2 * VOLATILE_REF_COST);
9113 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9114 ins_encode %{
9115 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9116 %}
9117 ins_pipe(pipe_serial);
9118 %}
9119
9120 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9121 predicate(n->as_LoadStore()->barrier_data() == 0);
9122 match(Set prev (GetAndSetP mem newv));
9123 ins_cost(2 * VOLATILE_REF_COST);
9124 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9125 ins_encode %{
9126 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9127 %}
9128 ins_pipe(pipe_serial);
9129 %}
9130
9131 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9132 predicate(needs_acquiring_load_exclusive(n));
9133 match(Set prev (GetAndSetI mem newv));
9134 ins_cost(VOLATILE_REF_COST);
9135 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9136 ins_encode %{
9137 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9138 %}
9139 ins_pipe(pipe_serial);
9140 %}
9141
9142 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9143 predicate(needs_acquiring_load_exclusive(n));
9144 match(Set prev (GetAndSetL mem newv));
9145 ins_cost(VOLATILE_REF_COST);
9146 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9147 ins_encode %{
9148 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9149 %}
9150 ins_pipe(pipe_serial);
9151 %}
9152
9153 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9154 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9155 match(Set prev (GetAndSetN mem newv));
9156 ins_cost(VOLATILE_REF_COST);
9157 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9158 ins_encode %{
9159 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9160 %}
9161 ins_pipe(pipe_serial);
9162 %}
9163
9164 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9165 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9166 match(Set prev (GetAndSetP mem newv));
9167 ins_cost(VOLATILE_REF_COST);
9168 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9169 ins_encode %{
9170 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9171 %}
9172 ins_pipe(pipe_serial);
9173 %}
9174
9175
9176 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9177 match(Set newval (GetAndAddL mem incr));
9178 ins_cost(2 * VOLATILE_REF_COST + 1);
9179 format %{ "get_and_addL $newval, [$mem], $incr" %}
9180 ins_encode %{
9181 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9182 %}
9183 ins_pipe(pipe_serial);
9184 %}
9185
9186 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9187 predicate(n->as_LoadStore()->result_not_used());
9188 match(Set dummy (GetAndAddL mem incr));
9189 ins_cost(2 * VOLATILE_REF_COST);
9190 format %{ "get_and_addL [$mem], $incr" %}
9191 ins_encode %{
9192 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9193 %}
9194 ins_pipe(pipe_serial);
9195 %}
9196
9197 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9198 match(Set newval (GetAndAddL mem incr));
9199 ins_cost(2 * VOLATILE_REF_COST + 1);
9200 format %{ "get_and_addL $newval, [$mem], $incr" %}
9201 ins_encode %{
9202 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9203 %}
9204 ins_pipe(pipe_serial);
9205 %}
9206
9207 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9208 predicate(n->as_LoadStore()->result_not_used());
9209 match(Set dummy (GetAndAddL mem incr));
9210 ins_cost(2 * VOLATILE_REF_COST);
9211 format %{ "get_and_addL [$mem], $incr" %}
9212 ins_encode %{
9213 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9214 %}
9215 ins_pipe(pipe_serial);
9216 %}
9217
9218 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9219 match(Set newval (GetAndAddI mem incr));
9220 ins_cost(2 * VOLATILE_REF_COST + 1);
9221 format %{ "get_and_addI $newval, [$mem], $incr" %}
9222 ins_encode %{
9223 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9224 %}
9225 ins_pipe(pipe_serial);
9226 %}
9227
9228 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9229 predicate(n->as_LoadStore()->result_not_used());
9230 match(Set dummy (GetAndAddI mem incr));
9231 ins_cost(2 * VOLATILE_REF_COST);
9232 format %{ "get_and_addI [$mem], $incr" %}
9233 ins_encode %{
9234 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9235 %}
9236 ins_pipe(pipe_serial);
9237 %}
9238
9239 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9240 match(Set newval (GetAndAddI mem incr));
9241 ins_cost(2 * VOLATILE_REF_COST + 1);
9242 format %{ "get_and_addI $newval, [$mem], $incr" %}
9243 ins_encode %{
9244 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9245 %}
9246 ins_pipe(pipe_serial);
9247 %}
9248
9249 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9250 predicate(n->as_LoadStore()->result_not_used());
9251 match(Set dummy (GetAndAddI mem incr));
9252 ins_cost(2 * VOLATILE_REF_COST);
9253 format %{ "get_and_addI [$mem], $incr" %}
9254 ins_encode %{
9255 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9256 %}
9257 ins_pipe(pipe_serial);
9258 %}
9259
9260 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9261 predicate(needs_acquiring_load_exclusive(n));
9262 match(Set newval (GetAndAddL mem incr));
9263 ins_cost(VOLATILE_REF_COST + 1);
9264 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9265 ins_encode %{
9266 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9267 %}
9268 ins_pipe(pipe_serial);
9269 %}
9270
9271 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9272 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9273 match(Set dummy (GetAndAddL mem incr));
9274 ins_cost(VOLATILE_REF_COST);
9275 format %{ "get_and_addL_acq [$mem], $incr" %}
9276 ins_encode %{
9277 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9278 %}
9279 ins_pipe(pipe_serial);
9280 %}
9281
9282 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9283 predicate(needs_acquiring_load_exclusive(n));
9284 match(Set newval (GetAndAddL mem incr));
9285 ins_cost(VOLATILE_REF_COST + 1);
9286 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9287 ins_encode %{
9288 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9289 %}
9290 ins_pipe(pipe_serial);
9291 %}
9292
9293 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9294 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9295 match(Set dummy (GetAndAddL mem incr));
9296 ins_cost(VOLATILE_REF_COST);
9297 format %{ "get_and_addL_acq [$mem], $incr" %}
9298 ins_encode %{
9299 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9300 %}
9301 ins_pipe(pipe_serial);
9302 %}
9303
9304 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9305 predicate(needs_acquiring_load_exclusive(n));
9306 match(Set newval (GetAndAddI mem incr));
9307 ins_cost(VOLATILE_REF_COST + 1);
9308 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9309 ins_encode %{
9310 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9311 %}
9312 ins_pipe(pipe_serial);
9313 %}
9314
9315 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9316 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9317 match(Set dummy (GetAndAddI mem incr));
9318 ins_cost(VOLATILE_REF_COST);
9319 format %{ "get_and_addI_acq [$mem], $incr" %}
9320 ins_encode %{
9321 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9322 %}
9323 ins_pipe(pipe_serial);
9324 %}
9325
9326 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9327 predicate(needs_acquiring_load_exclusive(n));
9328 match(Set newval (GetAndAddI mem incr));
9329 ins_cost(VOLATILE_REF_COST + 1);
9330 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9331 ins_encode %{
9332 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9333 %}
9334 ins_pipe(pipe_serial);
9335 %}
9336
9337 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9338 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9339 match(Set dummy (GetAndAddI mem incr));
9340 ins_cost(VOLATILE_REF_COST);
9341 format %{ "get_and_addI_acq [$mem], $incr" %}
9342 ins_encode %{
9343 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9344 %}
9345 ins_pipe(pipe_serial);
9346 %}
9347
9348 // Manifest a CmpU result in an integer register.
9349 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9350 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9351 %{
9352 match(Set dst (CmpU3 src1 src2));
9353 effect(KILL flags);
9354
9355 ins_cost(INSN_COST * 3);
9356 format %{
9357 "cmpw $src1, $src2\n\t"
9358 "csetw $dst, ne\n\t"
9359 "cnegw $dst, lo\t# CmpU3(reg)"
9360 %}
9361 ins_encode %{
9362 __ cmpw($src1$$Register, $src2$$Register);
9363 __ csetw($dst$$Register, Assembler::NE);
9364 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9365 %}
9366
9367 ins_pipe(pipe_class_default);
9368 %}
9369
9370 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9371 %{
9372 match(Set dst (CmpU3 src1 src2));
9373 effect(KILL flags);
9374
9375 ins_cost(INSN_COST * 3);
9376 format %{
9377 "subsw zr, $src1, $src2\n\t"
9378 "csetw $dst, ne\n\t"
9379 "cnegw $dst, lo\t# CmpU3(imm)"
9380 %}
9381 ins_encode %{
9382 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9383 __ csetw($dst$$Register, Assembler::NE);
9384 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9385 %}
9386
9387 ins_pipe(pipe_class_default);
9388 %}
9389
9390 // Manifest a CmpUL result in an integer register.
9391 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9392 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9393 %{
9394 match(Set dst (CmpUL3 src1 src2));
9395 effect(KILL flags);
9396
9397 ins_cost(INSN_COST * 3);
9398 format %{
9399 "cmp $src1, $src2\n\t"
9400 "csetw $dst, ne\n\t"
9401 "cnegw $dst, lo\t# CmpUL3(reg)"
9402 %}
9403 ins_encode %{
9404 __ cmp($src1$$Register, $src2$$Register);
9405 __ csetw($dst$$Register, Assembler::NE);
9406 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9407 %}
9408
9409 ins_pipe(pipe_class_default);
9410 %}
9411
9412 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9413 %{
9414 match(Set dst (CmpUL3 src1 src2));
9415 effect(KILL flags);
9416
9417 ins_cost(INSN_COST * 3);
9418 format %{
9419 "subs zr, $src1, $src2\n\t"
9420 "csetw $dst, ne\n\t"
9421 "cnegw $dst, lo\t# CmpUL3(imm)"
9422 %}
9423 ins_encode %{
9424 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9425 __ csetw($dst$$Register, Assembler::NE);
9426 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9427 %}
9428
9429 ins_pipe(pipe_class_default);
9430 %}
9431
9432 // Manifest a CmpL result in an integer register.
9433 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9434 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9435 %{
9436 match(Set dst (CmpL3 src1 src2));
9437 effect(KILL flags);
9438
9439 ins_cost(INSN_COST * 3);
9440 format %{
9441 "cmp $src1, $src2\n\t"
9442 "csetw $dst, ne\n\t"
9443 "cnegw $dst, lt\t# CmpL3(reg)"
9444 %}
9445 ins_encode %{
9446 __ cmp($src1$$Register, $src2$$Register);
9447 __ csetw($dst$$Register, Assembler::NE);
9448 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9449 %}
9450
9451 ins_pipe(pipe_class_default);
9452 %}
9453
9454 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9455 %{
9456 match(Set dst (CmpL3 src1 src2));
9457 effect(KILL flags);
9458
9459 ins_cost(INSN_COST * 3);
9460 format %{
9461 "subs zr, $src1, $src2\n\t"
9462 "csetw $dst, ne\n\t"
9463 "cnegw $dst, lt\t# CmpL3(imm)"
9464 %}
9465 ins_encode %{
9466 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9467 __ csetw($dst$$Register, Assembler::NE);
9468 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9469 %}
9470
9471 ins_pipe(pipe_class_default);
9472 %}
9473
9474 // ============================================================================
9475 // Conditional Move Instructions
9476
9477 // n.b. we have identical rules for both a signed compare op (cmpOp)
9478 // and an unsigned compare op (cmpOpU). it would be nice if we could
9479 // define an op class which merged both inputs and use it to type the
9480 // argument to a single rule. unfortunatelyt his fails because the
9481 // opclass does not live up to the COND_INTER interface of its
9482 // component operands. When the generic code tries to negate the
9483 // operand it ends up running the generci Machoper::negate method
9484 // which throws a ShouldNotHappen. So, we have to provide two flavours
9485 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9486
9487 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9488 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9489
9490 ins_cost(INSN_COST * 2);
9491 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9492
9493 ins_encode %{
9494 __ cselw(as_Register($dst$$reg),
9495 as_Register($src2$$reg),
9496 as_Register($src1$$reg),
9497 (Assembler::Condition)$cmp$$cmpcode);
9498 %}
9499
9500 ins_pipe(icond_reg_reg);
9501 %}
9502
9503 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9504 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9505
9506 ins_cost(INSN_COST * 2);
9507 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9508
9509 ins_encode %{
9510 __ cselw(as_Register($dst$$reg),
9511 as_Register($src2$$reg),
9512 as_Register($src1$$reg),
9513 (Assembler::Condition)$cmp$$cmpcode);
9514 %}
9515
9516 ins_pipe(icond_reg_reg);
9517 %}
9518
9519 // special cases where one arg is zero
9520
9521 // n.b. this is selected in preference to the rule above because it
9522 // avoids loading constant 0 into a source register
9523
9524 // TODO
9525 // we ought only to be able to cull one of these variants as the ideal
9526 // transforms ought always to order the zero consistently (to left/right?)
9527
9528 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9529 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9530
9531 ins_cost(INSN_COST * 2);
9532 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9533
9534 ins_encode %{
9535 __ cselw(as_Register($dst$$reg),
9536 as_Register($src$$reg),
9537 zr,
9538 (Assembler::Condition)$cmp$$cmpcode);
9539 %}
9540
9541 ins_pipe(icond_reg);
9542 %}
9543
9544 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9545 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9546
9547 ins_cost(INSN_COST * 2);
9548 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9549
9550 ins_encode %{
9551 __ cselw(as_Register($dst$$reg),
9552 as_Register($src$$reg),
9553 zr,
9554 (Assembler::Condition)$cmp$$cmpcode);
9555 %}
9556
9557 ins_pipe(icond_reg);
9558 %}
9559
9560 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9561 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9562
9563 ins_cost(INSN_COST * 2);
9564 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9565
9566 ins_encode %{
9567 __ cselw(as_Register($dst$$reg),
9568 zr,
9569 as_Register($src$$reg),
9570 (Assembler::Condition)$cmp$$cmpcode);
9571 %}
9572
9573 ins_pipe(icond_reg);
9574 %}
9575
9576 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9577 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9578
9579 ins_cost(INSN_COST * 2);
9580 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9581
9582 ins_encode %{
9583 __ cselw(as_Register($dst$$reg),
9584 zr,
9585 as_Register($src$$reg),
9586 (Assembler::Condition)$cmp$$cmpcode);
9587 %}
9588
9589 ins_pipe(icond_reg);
9590 %}
9591
9592 // special case for creating a boolean 0 or 1
9593
9594 // n.b. this is selected in preference to the rule above because it
9595 // avoids loading constants 0 and 1 into a source register
9596
9597 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9598 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9599
9600 ins_cost(INSN_COST * 2);
9601 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9602
9603 ins_encode %{
9604 // equivalently
9605 // cset(as_Register($dst$$reg),
9606 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9607 __ csincw(as_Register($dst$$reg),
9608 zr,
9609 zr,
9610 (Assembler::Condition)$cmp$$cmpcode);
9611 %}
9612
9613 ins_pipe(icond_none);
9614 %}
9615
9616 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9617 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9618
9619 ins_cost(INSN_COST * 2);
9620 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9621
9622 ins_encode %{
9623 // equivalently
9624 // cset(as_Register($dst$$reg),
9625 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9626 __ csincw(as_Register($dst$$reg),
9627 zr,
9628 zr,
9629 (Assembler::Condition)$cmp$$cmpcode);
9630 %}
9631
9632 ins_pipe(icond_none);
9633 %}
9634
9635 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9636 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9637
9638 ins_cost(INSN_COST * 2);
9639 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9640
9641 ins_encode %{
9642 __ csel(as_Register($dst$$reg),
9643 as_Register($src2$$reg),
9644 as_Register($src1$$reg),
9645 (Assembler::Condition)$cmp$$cmpcode);
9646 %}
9647
9648 ins_pipe(icond_reg_reg);
9649 %}
9650
9651 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9652 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9653
9654 ins_cost(INSN_COST * 2);
9655 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9656
9657 ins_encode %{
9658 __ csel(as_Register($dst$$reg),
9659 as_Register($src2$$reg),
9660 as_Register($src1$$reg),
9661 (Assembler::Condition)$cmp$$cmpcode);
9662 %}
9663
9664 ins_pipe(icond_reg_reg);
9665 %}
9666
9667 // special cases where one arg is zero
9668
9669 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9670 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9671
9672 ins_cost(INSN_COST * 2);
9673 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9674
9675 ins_encode %{
9676 __ csel(as_Register($dst$$reg),
9677 zr,
9678 as_Register($src$$reg),
9679 (Assembler::Condition)$cmp$$cmpcode);
9680 %}
9681
9682 ins_pipe(icond_reg);
9683 %}
9684
9685 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9686 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9687
9688 ins_cost(INSN_COST * 2);
9689 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9690
9691 ins_encode %{
9692 __ csel(as_Register($dst$$reg),
9693 zr,
9694 as_Register($src$$reg),
9695 (Assembler::Condition)$cmp$$cmpcode);
9696 %}
9697
9698 ins_pipe(icond_reg);
9699 %}
9700
9701 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9702 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9703
9704 ins_cost(INSN_COST * 2);
9705 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9706
9707 ins_encode %{
9708 __ csel(as_Register($dst$$reg),
9709 as_Register($src$$reg),
9710 zr,
9711 (Assembler::Condition)$cmp$$cmpcode);
9712 %}
9713
9714 ins_pipe(icond_reg);
9715 %}
9716
9717 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9718 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9719
9720 ins_cost(INSN_COST * 2);
9721 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9722
9723 ins_encode %{
9724 __ csel(as_Register($dst$$reg),
9725 as_Register($src$$reg),
9726 zr,
9727 (Assembler::Condition)$cmp$$cmpcode);
9728 %}
9729
9730 ins_pipe(icond_reg);
9731 %}
9732
9733 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9734 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9735
9736 ins_cost(INSN_COST * 2);
9737 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9738
9739 ins_encode %{
9740 __ csel(as_Register($dst$$reg),
9741 as_Register($src2$$reg),
9742 as_Register($src1$$reg),
9743 (Assembler::Condition)$cmp$$cmpcode);
9744 %}
9745
9746 ins_pipe(icond_reg_reg);
9747 %}
9748
9749 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9750 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9751
9752 ins_cost(INSN_COST * 2);
9753 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9754
9755 ins_encode %{
9756 __ csel(as_Register($dst$$reg),
9757 as_Register($src2$$reg),
9758 as_Register($src1$$reg),
9759 (Assembler::Condition)$cmp$$cmpcode);
9760 %}
9761
9762 ins_pipe(icond_reg_reg);
9763 %}
9764
9765 // special cases where one arg is zero
9766
9767 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9768 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9769
9770 ins_cost(INSN_COST * 2);
9771 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9772
9773 ins_encode %{
9774 __ csel(as_Register($dst$$reg),
9775 zr,
9776 as_Register($src$$reg),
9777 (Assembler::Condition)$cmp$$cmpcode);
9778 %}
9779
9780 ins_pipe(icond_reg);
9781 %}
9782
9783 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9784 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9785
9786 ins_cost(INSN_COST * 2);
9787 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9788
9789 ins_encode %{
9790 __ csel(as_Register($dst$$reg),
9791 zr,
9792 as_Register($src$$reg),
9793 (Assembler::Condition)$cmp$$cmpcode);
9794 %}
9795
9796 ins_pipe(icond_reg);
9797 %}
9798
9799 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9800 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9801
9802 ins_cost(INSN_COST * 2);
9803 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9804
9805 ins_encode %{
9806 __ csel(as_Register($dst$$reg),
9807 as_Register($src$$reg),
9808 zr,
9809 (Assembler::Condition)$cmp$$cmpcode);
9810 %}
9811
9812 ins_pipe(icond_reg);
9813 %}
9814
9815 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9816 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9817
9818 ins_cost(INSN_COST * 2);
9819 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9820
9821 ins_encode %{
9822 __ csel(as_Register($dst$$reg),
9823 as_Register($src$$reg),
9824 zr,
9825 (Assembler::Condition)$cmp$$cmpcode);
9826 %}
9827
9828 ins_pipe(icond_reg);
9829 %}
9830
9831 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9832 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9833
9834 ins_cost(INSN_COST * 2);
9835 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9836
9837 ins_encode %{
9838 __ cselw(as_Register($dst$$reg),
9839 as_Register($src2$$reg),
9840 as_Register($src1$$reg),
9841 (Assembler::Condition)$cmp$$cmpcode);
9842 %}
9843
9844 ins_pipe(icond_reg_reg);
9845 %}
9846
9847 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9848 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9849
9850 ins_cost(INSN_COST * 2);
9851 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9852
9853 ins_encode %{
9854 __ cselw(as_Register($dst$$reg),
9855 as_Register($src2$$reg),
9856 as_Register($src1$$reg),
9857 (Assembler::Condition)$cmp$$cmpcode);
9858 %}
9859
9860 ins_pipe(icond_reg_reg);
9861 %}
9862
9863 // special cases where one arg is zero
9864
9865 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9866 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9867
9868 ins_cost(INSN_COST * 2);
9869 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9870
9871 ins_encode %{
9872 __ cselw(as_Register($dst$$reg),
9873 zr,
9874 as_Register($src$$reg),
9875 (Assembler::Condition)$cmp$$cmpcode);
9876 %}
9877
9878 ins_pipe(icond_reg);
9879 %}
9880
9881 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9882 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9883
9884 ins_cost(INSN_COST * 2);
9885 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9886
9887 ins_encode %{
9888 __ cselw(as_Register($dst$$reg),
9889 zr,
9890 as_Register($src$$reg),
9891 (Assembler::Condition)$cmp$$cmpcode);
9892 %}
9893
9894 ins_pipe(icond_reg);
9895 %}
9896
9897 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9898 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9899
9900 ins_cost(INSN_COST * 2);
9901 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9902
9903 ins_encode %{
9904 __ cselw(as_Register($dst$$reg),
9905 as_Register($src$$reg),
9906 zr,
9907 (Assembler::Condition)$cmp$$cmpcode);
9908 %}
9909
9910 ins_pipe(icond_reg);
9911 %}
9912
9913 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9914 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9915
9916 ins_cost(INSN_COST * 2);
9917 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9918
9919 ins_encode %{
9920 __ cselw(as_Register($dst$$reg),
9921 as_Register($src$$reg),
9922 zr,
9923 (Assembler::Condition)$cmp$$cmpcode);
9924 %}
9925
9926 ins_pipe(icond_reg);
9927 %}
9928
9929 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9930 %{
9931 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9932
9933 ins_cost(INSN_COST * 3);
9934
9935 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9936 ins_encode %{
9937 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9938 __ fcsels(as_FloatRegister($dst$$reg),
9939 as_FloatRegister($src2$$reg),
9940 as_FloatRegister($src1$$reg),
9941 cond);
9942 %}
9943
9944 ins_pipe(fp_cond_reg_reg_s);
9945 %}
9946
9947 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9948 %{
9949 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9950
9951 ins_cost(INSN_COST * 3);
9952
9953 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9954 ins_encode %{
9955 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9956 __ fcsels(as_FloatRegister($dst$$reg),
9957 as_FloatRegister($src2$$reg),
9958 as_FloatRegister($src1$$reg),
9959 cond);
9960 %}
9961
9962 ins_pipe(fp_cond_reg_reg_s);
9963 %}
9964
9965 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9966 %{
9967 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9968
9969 ins_cost(INSN_COST * 3);
9970
9971 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9972 ins_encode %{
9973 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9974 __ fcseld(as_FloatRegister($dst$$reg),
9975 as_FloatRegister($src2$$reg),
9976 as_FloatRegister($src1$$reg),
9977 cond);
9978 %}
9979
9980 ins_pipe(fp_cond_reg_reg_d);
9981 %}
9982
9983 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9984 %{
9985 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9986
9987 ins_cost(INSN_COST * 3);
9988
9989 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9990 ins_encode %{
9991 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9992 __ fcseld(as_FloatRegister($dst$$reg),
9993 as_FloatRegister($src2$$reg),
9994 as_FloatRegister($src1$$reg),
9995 cond);
9996 %}
9997
9998 ins_pipe(fp_cond_reg_reg_d);
9999 %}
10000
10001 // ============================================================================
10002 // Arithmetic Instructions
10003 //
10004
10005 // Integer Addition
10006
10007 // TODO
10008 // these currently employ operations which do not set CR and hence are
10009 // not flagged as killing CR but we would like to isolate the cases
10010 // where we want to set flags from those where we don't. need to work
10011 // out how to do that.
10012
10013 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10014 match(Set dst (AddI src1 src2));
10015
10016 ins_cost(INSN_COST);
10017 format %{ "addw $dst, $src1, $src2" %}
10018
10019 ins_encode %{
10020 __ addw(as_Register($dst$$reg),
10021 as_Register($src1$$reg),
10022 as_Register($src2$$reg));
10023 %}
10024
10025 ins_pipe(ialu_reg_reg);
10026 %}
10027
10028 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10029 match(Set dst (AddI src1 src2));
10030
10031 ins_cost(INSN_COST);
10032 format %{ "addw $dst, $src1, $src2" %}
10033
10034 // use opcode to indicate that this is an add not a sub
10035 opcode(0x0);
10036
10037 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10038
10039 ins_pipe(ialu_reg_imm);
10040 %}
10041
10042 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10043 match(Set dst (AddI (ConvL2I src1) src2));
10044
10045 ins_cost(INSN_COST);
10046 format %{ "addw $dst, $src1, $src2" %}
10047
10048 // use opcode to indicate that this is an add not a sub
10049 opcode(0x0);
10050
10051 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10052
10053 ins_pipe(ialu_reg_imm);
10054 %}
10055
10056 // Pointer Addition
10057 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10058 match(Set dst (AddP src1 src2));
10059
10060 ins_cost(INSN_COST);
10061 format %{ "add $dst, $src1, $src2\t# ptr" %}
10062
10063 ins_encode %{
10064 __ add(as_Register($dst$$reg),
10065 as_Register($src1$$reg),
10066 as_Register($src2$$reg));
10067 %}
10068
10069 ins_pipe(ialu_reg_reg);
10070 %}
10071
10072 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10073 match(Set dst (AddP src1 (ConvI2L src2)));
10074
10075 ins_cost(1.9 * INSN_COST);
10076 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10077
10078 ins_encode %{
10079 __ add(as_Register($dst$$reg),
10080 as_Register($src1$$reg),
10081 as_Register($src2$$reg), ext::sxtw);
10082 %}
10083
10084 ins_pipe(ialu_reg_reg);
10085 %}
10086
10087 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10088 match(Set dst (AddP src1 (LShiftL src2 scale)));
10089
10090 ins_cost(1.9 * INSN_COST);
10091 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10092
10093 ins_encode %{
10094 __ lea(as_Register($dst$$reg),
10095 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10096 Address::lsl($scale$$constant)));
10097 %}
10098
10099 ins_pipe(ialu_reg_reg_shift);
10100 %}
10101
10102 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10103 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10104
10105 ins_cost(1.9 * INSN_COST);
10106 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10107
10108 ins_encode %{
10109 __ lea(as_Register($dst$$reg),
10110 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10111 Address::sxtw($scale$$constant)));
10112 %}
10113
10114 ins_pipe(ialu_reg_reg_shift);
10115 %}
10116
10117 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10118 match(Set dst (LShiftL (ConvI2L src) scale));
10119
10120 ins_cost(INSN_COST);
10121 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10122
10123 ins_encode %{
10124 __ sbfiz(as_Register($dst$$reg),
10125 as_Register($src$$reg),
10126 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10127 %}
10128
10129 ins_pipe(ialu_reg_shift);
10130 %}
10131
10132 // Pointer Immediate Addition
10133 // n.b. this needs to be more expensive than using an indirect memory
10134 // operand
10135 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10136 match(Set dst (AddP src1 src2));
10137
10138 ins_cost(INSN_COST);
10139 format %{ "add $dst, $src1, $src2\t# ptr" %}
10140
10141 // use opcode to indicate that this is an add not a sub
10142 opcode(0x0);
10143
10144 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10145
10146 ins_pipe(ialu_reg_imm);
10147 %}
10148
10149 // Long Addition
10150 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10151
10152 match(Set dst (AddL src1 src2));
10153
10154 ins_cost(INSN_COST);
10155 format %{ "add $dst, $src1, $src2" %}
10156
10157 ins_encode %{
10158 __ add(as_Register($dst$$reg),
10159 as_Register($src1$$reg),
10160 as_Register($src2$$reg));
10161 %}
10162
10163 ins_pipe(ialu_reg_reg);
10164 %}
10165
10166 // No constant pool entries requiredLong Immediate Addition.
10167 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10168 match(Set dst (AddL src1 src2));
10169
10170 ins_cost(INSN_COST);
10171 format %{ "add $dst, $src1, $src2" %}
10172
10173 // use opcode to indicate that this is an add not a sub
10174 opcode(0x0);
10175
10176 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10177
10178 ins_pipe(ialu_reg_imm);
10179 %}
10180
10181 // Integer Subtraction
10182 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10183 match(Set dst (SubI src1 src2));
10184
10185 ins_cost(INSN_COST);
10186 format %{ "subw $dst, $src1, $src2" %}
10187
10188 ins_encode %{
10189 __ subw(as_Register($dst$$reg),
10190 as_Register($src1$$reg),
10191 as_Register($src2$$reg));
10192 %}
10193
10194 ins_pipe(ialu_reg_reg);
10195 %}
10196
10197 // Immediate Subtraction
10198 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10199 match(Set dst (SubI src1 src2));
10200
10201 ins_cost(INSN_COST);
10202 format %{ "subw $dst, $src1, $src2" %}
10203
10204 // use opcode to indicate that this is a sub not an add
10205 opcode(0x1);
10206
10207 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10208
10209 ins_pipe(ialu_reg_imm);
10210 %}
10211
10212 // Long Subtraction
10213 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10214
10215 match(Set dst (SubL src1 src2));
10216
10217 ins_cost(INSN_COST);
10218 format %{ "sub $dst, $src1, $src2" %}
10219
10220 ins_encode %{
10221 __ sub(as_Register($dst$$reg),
10222 as_Register($src1$$reg),
10223 as_Register($src2$$reg));
10224 %}
10225
10226 ins_pipe(ialu_reg_reg);
10227 %}
10228
10229 // No constant pool entries requiredLong Immediate Subtraction.
10230 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10231 match(Set dst (SubL src1 src2));
10232
10233 ins_cost(INSN_COST);
10234 format %{ "sub$dst, $src1, $src2" %}
10235
10236 // use opcode to indicate that this is a sub not an add
10237 opcode(0x1);
10238
10239 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10240
10241 ins_pipe(ialu_reg_imm);
10242 %}
10243
10244 // Integer Negation (special case for sub)
10245
10246 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10247 match(Set dst (SubI zero src));
10248
10249 ins_cost(INSN_COST);
10250 format %{ "negw $dst, $src\t# int" %}
10251
10252 ins_encode %{
10253 __ negw(as_Register($dst$$reg),
10254 as_Register($src$$reg));
10255 %}
10256
10257 ins_pipe(ialu_reg);
10258 %}
10259
10260 // Long Negation
10261
10262 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10263 match(Set dst (SubL zero src));
10264
10265 ins_cost(INSN_COST);
10266 format %{ "neg $dst, $src\t# long" %}
10267
10268 ins_encode %{
10269 __ neg(as_Register($dst$$reg),
10270 as_Register($src$$reg));
10271 %}
10272
10273 ins_pipe(ialu_reg);
10274 %}
10275
10276 // Integer Multiply
10277
10278 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10279 match(Set dst (MulI src1 src2));
10280
10281 ins_cost(INSN_COST * 3);
10282 format %{ "mulw $dst, $src1, $src2" %}
10283
10284 ins_encode %{
10285 __ mulw(as_Register($dst$$reg),
10286 as_Register($src1$$reg),
10287 as_Register($src2$$reg));
10288 %}
10289
10290 ins_pipe(imul_reg_reg);
10291 %}
10292
10293 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10294 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10295
10296 ins_cost(INSN_COST * 3);
10297 format %{ "smull $dst, $src1, $src2" %}
10298
10299 ins_encode %{
10300 __ smull(as_Register($dst$$reg),
10301 as_Register($src1$$reg),
10302 as_Register($src2$$reg));
10303 %}
10304
10305 ins_pipe(imul_reg_reg);
10306 %}
10307
10308 // Long Multiply
10309
10310 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10311 match(Set dst (MulL src1 src2));
10312
10313 ins_cost(INSN_COST * 5);
10314 format %{ "mul $dst, $src1, $src2" %}
10315
10316 ins_encode %{
10317 __ mul(as_Register($dst$$reg),
10318 as_Register($src1$$reg),
10319 as_Register($src2$$reg));
10320 %}
10321
10322 ins_pipe(lmul_reg_reg);
10323 %}
10324
10325 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10326 %{
10327 match(Set dst (MulHiL src1 src2));
10328
10329 ins_cost(INSN_COST * 7);
10330 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10331
10332 ins_encode %{
10333 __ smulh(as_Register($dst$$reg),
10334 as_Register($src1$$reg),
10335 as_Register($src2$$reg));
10336 %}
10337
10338 ins_pipe(lmul_reg_reg);
10339 %}
10340
10341 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10342 %{
10343 match(Set dst (UMulHiL src1 src2));
10344
10345 ins_cost(INSN_COST * 7);
10346 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10347
10348 ins_encode %{
10349 __ umulh(as_Register($dst$$reg),
10350 as_Register($src1$$reg),
10351 as_Register($src2$$reg));
10352 %}
10353
10354 ins_pipe(lmul_reg_reg);
10355 %}
10356
10357 // Combined Integer Multiply & Add/Sub
10358
10359 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10360 match(Set dst (AddI src3 (MulI src1 src2)));
10361
10362 ins_cost(INSN_COST * 3);
10363 format %{ "madd $dst, $src1, $src2, $src3" %}
10364
10365 ins_encode %{
10366 __ maddw(as_Register($dst$$reg),
10367 as_Register($src1$$reg),
10368 as_Register($src2$$reg),
10369 as_Register($src3$$reg));
10370 %}
10371
10372 ins_pipe(imac_reg_reg);
10373 %}
10374
10375 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10376 match(Set dst (SubI src3 (MulI src1 src2)));
10377
10378 ins_cost(INSN_COST * 3);
10379 format %{ "msub $dst, $src1, $src2, $src3" %}
10380
10381 ins_encode %{
10382 __ msubw(as_Register($dst$$reg),
10383 as_Register($src1$$reg),
10384 as_Register($src2$$reg),
10385 as_Register($src3$$reg));
10386 %}
10387
10388 ins_pipe(imac_reg_reg);
10389 %}
10390
10391 // Combined Integer Multiply & Neg
10392
10393 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10394 match(Set dst (MulI (SubI zero src1) src2));
10395
10396 ins_cost(INSN_COST * 3);
10397 format %{ "mneg $dst, $src1, $src2" %}
10398
10399 ins_encode %{
10400 __ mnegw(as_Register($dst$$reg),
10401 as_Register($src1$$reg),
10402 as_Register($src2$$reg));
10403 %}
10404
10405 ins_pipe(imac_reg_reg);
10406 %}
10407
10408 // Combined Long Multiply & Add/Sub
10409
10410 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10411 match(Set dst (AddL src3 (MulL src1 src2)));
10412
10413 ins_cost(INSN_COST * 5);
10414 format %{ "madd $dst, $src1, $src2, $src3" %}
10415
10416 ins_encode %{
10417 __ madd(as_Register($dst$$reg),
10418 as_Register($src1$$reg),
10419 as_Register($src2$$reg),
10420 as_Register($src3$$reg));
10421 %}
10422
10423 ins_pipe(lmac_reg_reg);
10424 %}
10425
10426 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10427 match(Set dst (SubL src3 (MulL src1 src2)));
10428
10429 ins_cost(INSN_COST * 5);
10430 format %{ "msub $dst, $src1, $src2, $src3" %}
10431
10432 ins_encode %{
10433 __ msub(as_Register($dst$$reg),
10434 as_Register($src1$$reg),
10435 as_Register($src2$$reg),
10436 as_Register($src3$$reg));
10437 %}
10438
10439 ins_pipe(lmac_reg_reg);
10440 %}
10441
10442 // Combined Long Multiply & Neg
10443
10444 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10445 match(Set dst (MulL (SubL zero src1) src2));
10446
10447 ins_cost(INSN_COST * 5);
10448 format %{ "mneg $dst, $src1, $src2" %}
10449
10450 ins_encode %{
10451 __ mneg(as_Register($dst$$reg),
10452 as_Register($src1$$reg),
10453 as_Register($src2$$reg));
10454 %}
10455
10456 ins_pipe(lmac_reg_reg);
10457 %}
10458
10459 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10460
10461 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10462 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10463
10464 ins_cost(INSN_COST * 3);
10465 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10466
10467 ins_encode %{
10468 __ smaddl(as_Register($dst$$reg),
10469 as_Register($src1$$reg),
10470 as_Register($src2$$reg),
10471 as_Register($src3$$reg));
10472 %}
10473
10474 ins_pipe(imac_reg_reg);
10475 %}
10476
10477 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10478 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10479
10480 ins_cost(INSN_COST * 3);
10481 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10482
10483 ins_encode %{
10484 __ smsubl(as_Register($dst$$reg),
10485 as_Register($src1$$reg),
10486 as_Register($src2$$reg),
10487 as_Register($src3$$reg));
10488 %}
10489
10490 ins_pipe(imac_reg_reg);
10491 %}
10492
10493 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10494 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10495
10496 ins_cost(INSN_COST * 3);
10497 format %{ "smnegl $dst, $src1, $src2" %}
10498
10499 ins_encode %{
10500 __ smnegl(as_Register($dst$$reg),
10501 as_Register($src1$$reg),
10502 as_Register($src2$$reg));
10503 %}
10504
10505 ins_pipe(imac_reg_reg);
10506 %}
10507
10508 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10509
10510 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10511 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10512
10513 ins_cost(INSN_COST * 5);
10514 format %{ "mulw rscratch1, $src1, $src2\n\t"
10515 "maddw $dst, $src3, $src4, rscratch1" %}
10516
10517 ins_encode %{
10518 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10519 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10520
10521 ins_pipe(imac_reg_reg);
10522 %}
10523
10524 // Integer Divide
10525
10526 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10527 match(Set dst (DivI src1 src2));
10528
10529 ins_cost(INSN_COST * 19);
10530 format %{ "sdivw $dst, $src1, $src2" %}
10531
10532 ins_encode(aarch64_enc_divw(dst, src1, src2));
10533 ins_pipe(idiv_reg_reg);
10534 %}
10535
10536 // Long Divide
10537
10538 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10539 match(Set dst (DivL src1 src2));
10540
10541 ins_cost(INSN_COST * 35);
10542 format %{ "sdiv $dst, $src1, $src2" %}
10543
10544 ins_encode(aarch64_enc_div(dst, src1, src2));
10545 ins_pipe(ldiv_reg_reg);
10546 %}
10547
10548 // Integer Remainder
10549
10550 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10551 match(Set dst (ModI src1 src2));
10552
10553 ins_cost(INSN_COST * 22);
10554 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10555 "msubw $dst, rscratch1, $src2, $src1" %}
10556
10557 ins_encode(aarch64_enc_modw(dst, src1, src2));
10558 ins_pipe(idiv_reg_reg);
10559 %}
10560
10561 // Long Remainder
10562
10563 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10564 match(Set dst (ModL src1 src2));
10565
10566 ins_cost(INSN_COST * 38);
10567 format %{ "sdiv rscratch1, $src1, $src2\n"
10568 "msub $dst, rscratch1, $src2, $src1" %}
10569
10570 ins_encode(aarch64_enc_mod(dst, src1, src2));
10571 ins_pipe(ldiv_reg_reg);
10572 %}
10573
10574 // Unsigned Integer Divide
10575
10576 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10577 match(Set dst (UDivI src1 src2));
10578
10579 ins_cost(INSN_COST * 19);
10580 format %{ "udivw $dst, $src1, $src2" %}
10581
10582 ins_encode %{
10583 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10584 %}
10585
10586 ins_pipe(idiv_reg_reg);
10587 %}
10588
10589 // Unsigned Long Divide
10590
10591 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10592 match(Set dst (UDivL src1 src2));
10593
10594 ins_cost(INSN_COST * 35);
10595 format %{ "udiv $dst, $src1, $src2" %}
10596
10597 ins_encode %{
10598 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10599 %}
10600
10601 ins_pipe(ldiv_reg_reg);
10602 %}
10603
10604 // Unsigned Integer Remainder
10605
10606 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10607 match(Set dst (UModI src1 src2));
10608
10609 ins_cost(INSN_COST * 22);
10610 format %{ "udivw rscratch1, $src1, $src2\n\t"
10611 "msubw $dst, rscratch1, $src2, $src1" %}
10612
10613 ins_encode %{
10614 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10615 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10616 %}
10617
10618 ins_pipe(idiv_reg_reg);
10619 %}
10620
10621 // Unsigned Long Remainder
10622
10623 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10624 match(Set dst (UModL src1 src2));
10625
10626 ins_cost(INSN_COST * 38);
10627 format %{ "udiv rscratch1, $src1, $src2\n"
10628 "msub $dst, rscratch1, $src2, $src1" %}
10629
10630 ins_encode %{
10631 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10632 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10633 %}
10634
10635 ins_pipe(ldiv_reg_reg);
10636 %}
10637
10638 // Integer Shifts
10639
10640 // Shift Left Register
10641 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10642 match(Set dst (LShiftI src1 src2));
10643
10644 ins_cost(INSN_COST * 2);
10645 format %{ "lslvw $dst, $src1, $src2" %}
10646
10647 ins_encode %{
10648 __ lslvw(as_Register($dst$$reg),
10649 as_Register($src1$$reg),
10650 as_Register($src2$$reg));
10651 %}
10652
10653 ins_pipe(ialu_reg_reg_vshift);
10654 %}
10655
10656 // Shift Left Immediate
10657 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10658 match(Set dst (LShiftI src1 src2));
10659
10660 ins_cost(INSN_COST);
10661 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10662
10663 ins_encode %{
10664 __ lslw(as_Register($dst$$reg),
10665 as_Register($src1$$reg),
10666 $src2$$constant & 0x1f);
10667 %}
10668
10669 ins_pipe(ialu_reg_shift);
10670 %}
10671
10672 // Shift Right Logical Register
10673 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10674 match(Set dst (URShiftI src1 src2));
10675
10676 ins_cost(INSN_COST * 2);
10677 format %{ "lsrvw $dst, $src1, $src2" %}
10678
10679 ins_encode %{
10680 __ lsrvw(as_Register($dst$$reg),
10681 as_Register($src1$$reg),
10682 as_Register($src2$$reg));
10683 %}
10684
10685 ins_pipe(ialu_reg_reg_vshift);
10686 %}
10687
10688 // Shift Right Logical Immediate
10689 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10690 match(Set dst (URShiftI src1 src2));
10691
10692 ins_cost(INSN_COST);
10693 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10694
10695 ins_encode %{
10696 __ lsrw(as_Register($dst$$reg),
10697 as_Register($src1$$reg),
10698 $src2$$constant & 0x1f);
10699 %}
10700
10701 ins_pipe(ialu_reg_shift);
10702 %}
10703
10704 // Shift Right Arithmetic Register
10705 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10706 match(Set dst (RShiftI src1 src2));
10707
10708 ins_cost(INSN_COST * 2);
10709 format %{ "asrvw $dst, $src1, $src2" %}
10710
10711 ins_encode %{
10712 __ asrvw(as_Register($dst$$reg),
10713 as_Register($src1$$reg),
10714 as_Register($src2$$reg));
10715 %}
10716
10717 ins_pipe(ialu_reg_reg_vshift);
10718 %}
10719
10720 // Shift Right Arithmetic Immediate
10721 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10722 match(Set dst (RShiftI src1 src2));
10723
10724 ins_cost(INSN_COST);
10725 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10726
10727 ins_encode %{
10728 __ asrw(as_Register($dst$$reg),
10729 as_Register($src1$$reg),
10730 $src2$$constant & 0x1f);
10731 %}
10732
10733 ins_pipe(ialu_reg_shift);
10734 %}
10735
10736 // Combined Int Mask and Right Shift (using UBFM)
10737 // TODO
10738
10739 // Long Shifts
10740
10741 // Shift Left Register
10742 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10743 match(Set dst (LShiftL src1 src2));
10744
10745 ins_cost(INSN_COST * 2);
10746 format %{ "lslv $dst, $src1, $src2" %}
10747
10748 ins_encode %{
10749 __ lslv(as_Register($dst$$reg),
10750 as_Register($src1$$reg),
10751 as_Register($src2$$reg));
10752 %}
10753
10754 ins_pipe(ialu_reg_reg_vshift);
10755 %}
10756
10757 // Shift Left Immediate
10758 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10759 match(Set dst (LShiftL src1 src2));
10760
10761 ins_cost(INSN_COST);
10762 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10763
10764 ins_encode %{
10765 __ lsl(as_Register($dst$$reg),
10766 as_Register($src1$$reg),
10767 $src2$$constant & 0x3f);
10768 %}
10769
10770 ins_pipe(ialu_reg_shift);
10771 %}
10772
10773 // Shift Right Logical Register
10774 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10775 match(Set dst (URShiftL src1 src2));
10776
10777 ins_cost(INSN_COST * 2);
10778 format %{ "lsrv $dst, $src1, $src2" %}
10779
10780 ins_encode %{
10781 __ lsrv(as_Register($dst$$reg),
10782 as_Register($src1$$reg),
10783 as_Register($src2$$reg));
10784 %}
10785
10786 ins_pipe(ialu_reg_reg_vshift);
10787 %}
10788
10789 // Shift Right Logical Immediate
10790 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10791 match(Set dst (URShiftL src1 src2));
10792
10793 ins_cost(INSN_COST);
10794 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10795
10796 ins_encode %{
10797 __ lsr(as_Register($dst$$reg),
10798 as_Register($src1$$reg),
10799 $src2$$constant & 0x3f);
10800 %}
10801
10802 ins_pipe(ialu_reg_shift);
10803 %}
10804
10805 // A special-case pattern for card table stores.
10806 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10807 match(Set dst (URShiftL (CastP2X src1) src2));
10808
10809 ins_cost(INSN_COST);
10810 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10811
10812 ins_encode %{
10813 __ lsr(as_Register($dst$$reg),
10814 as_Register($src1$$reg),
10815 $src2$$constant & 0x3f);
10816 %}
10817
10818 ins_pipe(ialu_reg_shift);
10819 %}
10820
10821 // Shift Right Arithmetic Register
10822 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10823 match(Set dst (RShiftL src1 src2));
10824
10825 ins_cost(INSN_COST * 2);
10826 format %{ "asrv $dst, $src1, $src2" %}
10827
10828 ins_encode %{
10829 __ asrv(as_Register($dst$$reg),
10830 as_Register($src1$$reg),
10831 as_Register($src2$$reg));
10832 %}
10833
10834 ins_pipe(ialu_reg_reg_vshift);
10835 %}
10836
10837 // Shift Right Arithmetic Immediate
10838 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10839 match(Set dst (RShiftL src1 src2));
10840
10841 ins_cost(INSN_COST);
10842 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10843
10844 ins_encode %{
10845 __ asr(as_Register($dst$$reg),
10846 as_Register($src1$$reg),
10847 $src2$$constant & 0x3f);
10848 %}
10849
10850 ins_pipe(ialu_reg_shift);
10851 %}
10852
10853 // BEGIN This section of the file is automatically generated. Do not edit --------------
10854 // This section is generated from aarch64_ad.m4
10855
10856 // This pattern is automatically generated from aarch64_ad.m4
10857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10858 instruct regL_not_reg(iRegLNoSp dst,
10859 iRegL src1, immL_M1 m1,
10860 rFlagsReg cr) %{
10861 match(Set dst (XorL src1 m1));
10862 ins_cost(INSN_COST);
10863 format %{ "eon $dst, $src1, zr" %}
10864
10865 ins_encode %{
10866 __ eon(as_Register($dst$$reg),
10867 as_Register($src1$$reg),
10868 zr,
10869 Assembler::LSL, 0);
10870 %}
10871
10872 ins_pipe(ialu_reg);
10873 %}
10874
10875 // This pattern is automatically generated from aarch64_ad.m4
10876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10877 instruct regI_not_reg(iRegINoSp dst,
10878 iRegIorL2I src1, immI_M1 m1,
10879 rFlagsReg cr) %{
10880 match(Set dst (XorI src1 m1));
10881 ins_cost(INSN_COST);
10882 format %{ "eonw $dst, $src1, zr" %}
10883
10884 ins_encode %{
10885 __ eonw(as_Register($dst$$reg),
10886 as_Register($src1$$reg),
10887 zr,
10888 Assembler::LSL, 0);
10889 %}
10890
10891 ins_pipe(ialu_reg);
10892 %}
10893
10894 // This pattern is automatically generated from aarch64_ad.m4
10895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10896 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10897 immI0 zero, iRegIorL2I src1, immI src2) %{
10898 match(Set dst (SubI zero (URShiftI src1 src2)));
10899
10900 ins_cost(1.9 * INSN_COST);
10901 format %{ "negw $dst, $src1, LSR $src2" %}
10902
10903 ins_encode %{
10904 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10905 Assembler::LSR, $src2$$constant & 0x1f);
10906 %}
10907
10908 ins_pipe(ialu_reg_shift);
10909 %}
10910
10911 // This pattern is automatically generated from aarch64_ad.m4
10912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10913 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10914 immI0 zero, iRegIorL2I src1, immI src2) %{
10915 match(Set dst (SubI zero (RShiftI src1 src2)));
10916
10917 ins_cost(1.9 * INSN_COST);
10918 format %{ "negw $dst, $src1, ASR $src2" %}
10919
10920 ins_encode %{
10921 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10922 Assembler::ASR, $src2$$constant & 0x1f);
10923 %}
10924
10925 ins_pipe(ialu_reg_shift);
10926 %}
10927
10928 // This pattern is automatically generated from aarch64_ad.m4
10929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10930 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10931 immI0 zero, iRegIorL2I src1, immI src2) %{
10932 match(Set dst (SubI zero (LShiftI src1 src2)));
10933
10934 ins_cost(1.9 * INSN_COST);
10935 format %{ "negw $dst, $src1, LSL $src2" %}
10936
10937 ins_encode %{
10938 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10939 Assembler::LSL, $src2$$constant & 0x1f);
10940 %}
10941
10942 ins_pipe(ialu_reg_shift);
10943 %}
10944
10945 // This pattern is automatically generated from aarch64_ad.m4
10946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10947 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10948 immL0 zero, iRegL src1, immI src2) %{
10949 match(Set dst (SubL zero (URShiftL src1 src2)));
10950
10951 ins_cost(1.9 * INSN_COST);
10952 format %{ "neg $dst, $src1, LSR $src2" %}
10953
10954 ins_encode %{
10955 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10956 Assembler::LSR, $src2$$constant & 0x3f);
10957 %}
10958
10959 ins_pipe(ialu_reg_shift);
10960 %}
10961
10962 // This pattern is automatically generated from aarch64_ad.m4
10963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10964 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10965 immL0 zero, iRegL src1, immI src2) %{
10966 match(Set dst (SubL zero (RShiftL src1 src2)));
10967
10968 ins_cost(1.9 * INSN_COST);
10969 format %{ "neg $dst, $src1, ASR $src2" %}
10970
10971 ins_encode %{
10972 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10973 Assembler::ASR, $src2$$constant & 0x3f);
10974 %}
10975
10976 ins_pipe(ialu_reg_shift);
10977 %}
10978
10979 // This pattern is automatically generated from aarch64_ad.m4
10980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10981 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10982 immL0 zero, iRegL src1, immI src2) %{
10983 match(Set dst (SubL zero (LShiftL src1 src2)));
10984
10985 ins_cost(1.9 * INSN_COST);
10986 format %{ "neg $dst, $src1, LSL $src2" %}
10987
10988 ins_encode %{
10989 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10990 Assembler::LSL, $src2$$constant & 0x3f);
10991 %}
10992
10993 ins_pipe(ialu_reg_shift);
10994 %}
10995
10996 // This pattern is automatically generated from aarch64_ad.m4
10997 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10998 instruct AndI_reg_not_reg(iRegINoSp dst,
10999 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11000 match(Set dst (AndI src1 (XorI src2 m1)));
11001 ins_cost(INSN_COST);
11002 format %{ "bicw $dst, $src1, $src2" %}
11003
11004 ins_encode %{
11005 __ bicw(as_Register($dst$$reg),
11006 as_Register($src1$$reg),
11007 as_Register($src2$$reg),
11008 Assembler::LSL, 0);
11009 %}
11010
11011 ins_pipe(ialu_reg_reg);
11012 %}
11013
11014 // This pattern is automatically generated from aarch64_ad.m4
11015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11016 instruct AndL_reg_not_reg(iRegLNoSp dst,
11017 iRegL src1, iRegL src2, immL_M1 m1) %{
11018 match(Set dst (AndL src1 (XorL src2 m1)));
11019 ins_cost(INSN_COST);
11020 format %{ "bic $dst, $src1, $src2" %}
11021
11022 ins_encode %{
11023 __ bic(as_Register($dst$$reg),
11024 as_Register($src1$$reg),
11025 as_Register($src2$$reg),
11026 Assembler::LSL, 0);
11027 %}
11028
11029 ins_pipe(ialu_reg_reg);
11030 %}
11031
11032 // This pattern is automatically generated from aarch64_ad.m4
11033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11034 instruct OrI_reg_not_reg(iRegINoSp dst,
11035 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11036 match(Set dst (OrI src1 (XorI src2 m1)));
11037 ins_cost(INSN_COST);
11038 format %{ "ornw $dst, $src1, $src2" %}
11039
11040 ins_encode %{
11041 __ ornw(as_Register($dst$$reg),
11042 as_Register($src1$$reg),
11043 as_Register($src2$$reg),
11044 Assembler::LSL, 0);
11045 %}
11046
11047 ins_pipe(ialu_reg_reg);
11048 %}
11049
11050 // This pattern is automatically generated from aarch64_ad.m4
11051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11052 instruct OrL_reg_not_reg(iRegLNoSp dst,
11053 iRegL src1, iRegL src2, immL_M1 m1) %{
11054 match(Set dst (OrL src1 (XorL src2 m1)));
11055 ins_cost(INSN_COST);
11056 format %{ "orn $dst, $src1, $src2" %}
11057
11058 ins_encode %{
11059 __ orn(as_Register($dst$$reg),
11060 as_Register($src1$$reg),
11061 as_Register($src2$$reg),
11062 Assembler::LSL, 0);
11063 %}
11064
11065 ins_pipe(ialu_reg_reg);
11066 %}
11067
11068 // This pattern is automatically generated from aarch64_ad.m4
11069 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11070 instruct XorI_reg_not_reg(iRegINoSp dst,
11071 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11072 match(Set dst (XorI m1 (XorI src2 src1)));
11073 ins_cost(INSN_COST);
11074 format %{ "eonw $dst, $src1, $src2" %}
11075
11076 ins_encode %{
11077 __ eonw(as_Register($dst$$reg),
11078 as_Register($src1$$reg),
11079 as_Register($src2$$reg),
11080 Assembler::LSL, 0);
11081 %}
11082
11083 ins_pipe(ialu_reg_reg);
11084 %}
11085
11086 // This pattern is automatically generated from aarch64_ad.m4
11087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11088 instruct XorL_reg_not_reg(iRegLNoSp dst,
11089 iRegL src1, iRegL src2, immL_M1 m1) %{
11090 match(Set dst (XorL m1 (XorL src2 src1)));
11091 ins_cost(INSN_COST);
11092 format %{ "eon $dst, $src1, $src2" %}
11093
11094 ins_encode %{
11095 __ eon(as_Register($dst$$reg),
11096 as_Register($src1$$reg),
11097 as_Register($src2$$reg),
11098 Assembler::LSL, 0);
11099 %}
11100
11101 ins_pipe(ialu_reg_reg);
11102 %}
11103
11104 // This pattern is automatically generated from aarch64_ad.m4
11105 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11106 // val & (-1 ^ (val >>> shift)) ==> bicw
11107 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11108 iRegIorL2I src1, iRegIorL2I src2,
11109 immI src3, immI_M1 src4) %{
11110 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11111 ins_cost(1.9 * INSN_COST);
11112 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11113
11114 ins_encode %{
11115 __ bicw(as_Register($dst$$reg),
11116 as_Register($src1$$reg),
11117 as_Register($src2$$reg),
11118 Assembler::LSR,
11119 $src3$$constant & 0x1f);
11120 %}
11121
11122 ins_pipe(ialu_reg_reg_shift);
11123 %}
11124
11125 // This pattern is automatically generated from aarch64_ad.m4
11126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11127 // val & (-1 ^ (val >>> shift)) ==> bic
11128 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11129 iRegL src1, iRegL src2,
11130 immI src3, immL_M1 src4) %{
11131 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11132 ins_cost(1.9 * INSN_COST);
11133 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11134
11135 ins_encode %{
11136 __ bic(as_Register($dst$$reg),
11137 as_Register($src1$$reg),
11138 as_Register($src2$$reg),
11139 Assembler::LSR,
11140 $src3$$constant & 0x3f);
11141 %}
11142
11143 ins_pipe(ialu_reg_reg_shift);
11144 %}
11145
11146 // This pattern is automatically generated from aarch64_ad.m4
11147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11148 // val & (-1 ^ (val >> shift)) ==> bicw
11149 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11150 iRegIorL2I src1, iRegIorL2I src2,
11151 immI src3, immI_M1 src4) %{
11152 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11153 ins_cost(1.9 * INSN_COST);
11154 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11155
11156 ins_encode %{
11157 __ bicw(as_Register($dst$$reg),
11158 as_Register($src1$$reg),
11159 as_Register($src2$$reg),
11160 Assembler::ASR,
11161 $src3$$constant & 0x1f);
11162 %}
11163
11164 ins_pipe(ialu_reg_reg_shift);
11165 %}
11166
11167 // This pattern is automatically generated from aarch64_ad.m4
11168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11169 // val & (-1 ^ (val >> shift)) ==> bic
11170 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11171 iRegL src1, iRegL src2,
11172 immI src3, immL_M1 src4) %{
11173 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11174 ins_cost(1.9 * INSN_COST);
11175 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11176
11177 ins_encode %{
11178 __ bic(as_Register($dst$$reg),
11179 as_Register($src1$$reg),
11180 as_Register($src2$$reg),
11181 Assembler::ASR,
11182 $src3$$constant & 0x3f);
11183 %}
11184
11185 ins_pipe(ialu_reg_reg_shift);
11186 %}
11187
11188 // This pattern is automatically generated from aarch64_ad.m4
11189 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11190 // val & (-1 ^ (val ror shift)) ==> bicw
11191 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11192 iRegIorL2I src1, iRegIorL2I src2,
11193 immI src3, immI_M1 src4) %{
11194 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11195 ins_cost(1.9 * INSN_COST);
11196 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11197
11198 ins_encode %{
11199 __ bicw(as_Register($dst$$reg),
11200 as_Register($src1$$reg),
11201 as_Register($src2$$reg),
11202 Assembler::ROR,
11203 $src3$$constant & 0x1f);
11204 %}
11205
11206 ins_pipe(ialu_reg_reg_shift);
11207 %}
11208
11209 // This pattern is automatically generated from aarch64_ad.m4
11210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11211 // val & (-1 ^ (val ror shift)) ==> bic
11212 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11213 iRegL src1, iRegL src2,
11214 immI src3, immL_M1 src4) %{
11215 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11216 ins_cost(1.9 * INSN_COST);
11217 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11218
11219 ins_encode %{
11220 __ bic(as_Register($dst$$reg),
11221 as_Register($src1$$reg),
11222 as_Register($src2$$reg),
11223 Assembler::ROR,
11224 $src3$$constant & 0x3f);
11225 %}
11226
11227 ins_pipe(ialu_reg_reg_shift);
11228 %}
11229
11230 // This pattern is automatically generated from aarch64_ad.m4
11231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11232 // val & (-1 ^ (val << shift)) ==> bicw
11233 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11234 iRegIorL2I src1, iRegIorL2I src2,
11235 immI src3, immI_M1 src4) %{
11236 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11237 ins_cost(1.9 * INSN_COST);
11238 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11239
11240 ins_encode %{
11241 __ bicw(as_Register($dst$$reg),
11242 as_Register($src1$$reg),
11243 as_Register($src2$$reg),
11244 Assembler::LSL,
11245 $src3$$constant & 0x1f);
11246 %}
11247
11248 ins_pipe(ialu_reg_reg_shift);
11249 %}
11250
11251 // This pattern is automatically generated from aarch64_ad.m4
11252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11253 // val & (-1 ^ (val << shift)) ==> bic
11254 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11255 iRegL src1, iRegL src2,
11256 immI src3, immL_M1 src4) %{
11257 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11258 ins_cost(1.9 * INSN_COST);
11259 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11260
11261 ins_encode %{
11262 __ bic(as_Register($dst$$reg),
11263 as_Register($src1$$reg),
11264 as_Register($src2$$reg),
11265 Assembler::LSL,
11266 $src3$$constant & 0x3f);
11267 %}
11268
11269 ins_pipe(ialu_reg_reg_shift);
11270 %}
11271
11272 // This pattern is automatically generated from aarch64_ad.m4
11273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11274 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11275 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11276 iRegIorL2I src1, iRegIorL2I src2,
11277 immI src3, immI_M1 src4) %{
11278 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11279 ins_cost(1.9 * INSN_COST);
11280 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11281
11282 ins_encode %{
11283 __ eonw(as_Register($dst$$reg),
11284 as_Register($src1$$reg),
11285 as_Register($src2$$reg),
11286 Assembler::LSR,
11287 $src3$$constant & 0x1f);
11288 %}
11289
11290 ins_pipe(ialu_reg_reg_shift);
11291 %}
11292
11293 // This pattern is automatically generated from aarch64_ad.m4
11294 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11295 // val ^ (-1 ^ (val >>> shift)) ==> eon
11296 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11297 iRegL src1, iRegL src2,
11298 immI src3, immL_M1 src4) %{
11299 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11300 ins_cost(1.9 * INSN_COST);
11301 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11302
11303 ins_encode %{
11304 __ eon(as_Register($dst$$reg),
11305 as_Register($src1$$reg),
11306 as_Register($src2$$reg),
11307 Assembler::LSR,
11308 $src3$$constant & 0x3f);
11309 %}
11310
11311 ins_pipe(ialu_reg_reg_shift);
11312 %}
11313
11314 // This pattern is automatically generated from aarch64_ad.m4
11315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11316 // val ^ (-1 ^ (val >> shift)) ==> eonw
11317 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11318 iRegIorL2I src1, iRegIorL2I src2,
11319 immI src3, immI_M1 src4) %{
11320 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11321 ins_cost(1.9 * INSN_COST);
11322 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11323
11324 ins_encode %{
11325 __ eonw(as_Register($dst$$reg),
11326 as_Register($src1$$reg),
11327 as_Register($src2$$reg),
11328 Assembler::ASR,
11329 $src3$$constant & 0x1f);
11330 %}
11331
11332 ins_pipe(ialu_reg_reg_shift);
11333 %}
11334
11335 // This pattern is automatically generated from aarch64_ad.m4
11336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11337 // val ^ (-1 ^ (val >> shift)) ==> eon
11338 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11339 iRegL src1, iRegL src2,
11340 immI src3, immL_M1 src4) %{
11341 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11342 ins_cost(1.9 * INSN_COST);
11343 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11344
11345 ins_encode %{
11346 __ eon(as_Register($dst$$reg),
11347 as_Register($src1$$reg),
11348 as_Register($src2$$reg),
11349 Assembler::ASR,
11350 $src3$$constant & 0x3f);
11351 %}
11352
11353 ins_pipe(ialu_reg_reg_shift);
11354 %}
11355
11356 // This pattern is automatically generated from aarch64_ad.m4
11357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11358 // val ^ (-1 ^ (val ror shift)) ==> eonw
11359 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11360 iRegIorL2I src1, iRegIorL2I src2,
11361 immI src3, immI_M1 src4) %{
11362 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11363 ins_cost(1.9 * INSN_COST);
11364 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11365
11366 ins_encode %{
11367 __ eonw(as_Register($dst$$reg),
11368 as_Register($src1$$reg),
11369 as_Register($src2$$reg),
11370 Assembler::ROR,
11371 $src3$$constant & 0x1f);
11372 %}
11373
11374 ins_pipe(ialu_reg_reg_shift);
11375 %}
11376
11377 // This pattern is automatically generated from aarch64_ad.m4
11378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11379 // val ^ (-1 ^ (val ror shift)) ==> eon
11380 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11381 iRegL src1, iRegL src2,
11382 immI src3, immL_M1 src4) %{
11383 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11384 ins_cost(1.9 * INSN_COST);
11385 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11386
11387 ins_encode %{
11388 __ eon(as_Register($dst$$reg),
11389 as_Register($src1$$reg),
11390 as_Register($src2$$reg),
11391 Assembler::ROR,
11392 $src3$$constant & 0x3f);
11393 %}
11394
11395 ins_pipe(ialu_reg_reg_shift);
11396 %}
11397
11398 // This pattern is automatically generated from aarch64_ad.m4
11399 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11400 // val ^ (-1 ^ (val << shift)) ==> eonw
11401 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11402 iRegIorL2I src1, iRegIorL2I src2,
11403 immI src3, immI_M1 src4) %{
11404 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11405 ins_cost(1.9 * INSN_COST);
11406 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11407
11408 ins_encode %{
11409 __ eonw(as_Register($dst$$reg),
11410 as_Register($src1$$reg),
11411 as_Register($src2$$reg),
11412 Assembler::LSL,
11413 $src3$$constant & 0x1f);
11414 %}
11415
11416 ins_pipe(ialu_reg_reg_shift);
11417 %}
11418
11419 // This pattern is automatically generated from aarch64_ad.m4
11420 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11421 // val ^ (-1 ^ (val << shift)) ==> eon
11422 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11423 iRegL src1, iRegL src2,
11424 immI src3, immL_M1 src4) %{
11425 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11426 ins_cost(1.9 * INSN_COST);
11427 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11428
11429 ins_encode %{
11430 __ eon(as_Register($dst$$reg),
11431 as_Register($src1$$reg),
11432 as_Register($src2$$reg),
11433 Assembler::LSL,
11434 $src3$$constant & 0x3f);
11435 %}
11436
11437 ins_pipe(ialu_reg_reg_shift);
11438 %}
11439
11440 // This pattern is automatically generated from aarch64_ad.m4
11441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11442 // val | (-1 ^ (val >>> shift)) ==> ornw
11443 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11444 iRegIorL2I src1, iRegIorL2I src2,
11445 immI src3, immI_M1 src4) %{
11446 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11447 ins_cost(1.9 * INSN_COST);
11448 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11449
11450 ins_encode %{
11451 __ ornw(as_Register($dst$$reg),
11452 as_Register($src1$$reg),
11453 as_Register($src2$$reg),
11454 Assembler::LSR,
11455 $src3$$constant & 0x1f);
11456 %}
11457
11458 ins_pipe(ialu_reg_reg_shift);
11459 %}
11460
11461 // This pattern is automatically generated from aarch64_ad.m4
11462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11463 // val | (-1 ^ (val >>> shift)) ==> orn
11464 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11465 iRegL src1, iRegL src2,
11466 immI src3, immL_M1 src4) %{
11467 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11468 ins_cost(1.9 * INSN_COST);
11469 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11470
11471 ins_encode %{
11472 __ orn(as_Register($dst$$reg),
11473 as_Register($src1$$reg),
11474 as_Register($src2$$reg),
11475 Assembler::LSR,
11476 $src3$$constant & 0x3f);
11477 %}
11478
11479 ins_pipe(ialu_reg_reg_shift);
11480 %}
11481
11482 // This pattern is automatically generated from aarch64_ad.m4
11483 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11484 // val | (-1 ^ (val >> shift)) ==> ornw
11485 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11486 iRegIorL2I src1, iRegIorL2I src2,
11487 immI src3, immI_M1 src4) %{
11488 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11489 ins_cost(1.9 * INSN_COST);
11490 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11491
11492 ins_encode %{
11493 __ ornw(as_Register($dst$$reg),
11494 as_Register($src1$$reg),
11495 as_Register($src2$$reg),
11496 Assembler::ASR,
11497 $src3$$constant & 0x1f);
11498 %}
11499
11500 ins_pipe(ialu_reg_reg_shift);
11501 %}
11502
11503 // This pattern is automatically generated from aarch64_ad.m4
11504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11505 // val | (-1 ^ (val >> shift)) ==> orn
11506 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11507 iRegL src1, iRegL src2,
11508 immI src3, immL_M1 src4) %{
11509 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11510 ins_cost(1.9 * INSN_COST);
11511 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11512
11513 ins_encode %{
11514 __ orn(as_Register($dst$$reg),
11515 as_Register($src1$$reg),
11516 as_Register($src2$$reg),
11517 Assembler::ASR,
11518 $src3$$constant & 0x3f);
11519 %}
11520
11521 ins_pipe(ialu_reg_reg_shift);
11522 %}
11523
11524 // This pattern is automatically generated from aarch64_ad.m4
11525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11526 // val | (-1 ^ (val ror shift)) ==> ornw
11527 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11528 iRegIorL2I src1, iRegIorL2I src2,
11529 immI src3, immI_M1 src4) %{
11530 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11531 ins_cost(1.9 * INSN_COST);
11532 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11533
11534 ins_encode %{
11535 __ ornw(as_Register($dst$$reg),
11536 as_Register($src1$$reg),
11537 as_Register($src2$$reg),
11538 Assembler::ROR,
11539 $src3$$constant & 0x1f);
11540 %}
11541
11542 ins_pipe(ialu_reg_reg_shift);
11543 %}
11544
11545 // This pattern is automatically generated from aarch64_ad.m4
11546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11547 // val | (-1 ^ (val ror shift)) ==> orn
11548 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11549 iRegL src1, iRegL src2,
11550 immI src3, immL_M1 src4) %{
11551 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11552 ins_cost(1.9 * INSN_COST);
11553 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11554
11555 ins_encode %{
11556 __ orn(as_Register($dst$$reg),
11557 as_Register($src1$$reg),
11558 as_Register($src2$$reg),
11559 Assembler::ROR,
11560 $src3$$constant & 0x3f);
11561 %}
11562
11563 ins_pipe(ialu_reg_reg_shift);
11564 %}
11565
11566 // This pattern is automatically generated from aarch64_ad.m4
11567 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11568 // val | (-1 ^ (val << shift)) ==> ornw
11569 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11570 iRegIorL2I src1, iRegIorL2I src2,
11571 immI src3, immI_M1 src4) %{
11572 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11573 ins_cost(1.9 * INSN_COST);
11574 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11575
11576 ins_encode %{
11577 __ ornw(as_Register($dst$$reg),
11578 as_Register($src1$$reg),
11579 as_Register($src2$$reg),
11580 Assembler::LSL,
11581 $src3$$constant & 0x1f);
11582 %}
11583
11584 ins_pipe(ialu_reg_reg_shift);
11585 %}
11586
11587 // This pattern is automatically generated from aarch64_ad.m4
11588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11589 // val | (-1 ^ (val << shift)) ==> orn
11590 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11591 iRegL src1, iRegL src2,
11592 immI src3, immL_M1 src4) %{
11593 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11594 ins_cost(1.9 * INSN_COST);
11595 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11596
11597 ins_encode %{
11598 __ orn(as_Register($dst$$reg),
11599 as_Register($src1$$reg),
11600 as_Register($src2$$reg),
11601 Assembler::LSL,
11602 $src3$$constant & 0x3f);
11603 %}
11604
11605 ins_pipe(ialu_reg_reg_shift);
11606 %}
11607
11608 // This pattern is automatically generated from aarch64_ad.m4
11609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11610 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11611 iRegIorL2I src1, iRegIorL2I src2,
11612 immI src3) %{
11613 match(Set dst (AndI src1 (URShiftI src2 src3)));
11614
11615 ins_cost(1.9 * INSN_COST);
11616 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11617
11618 ins_encode %{
11619 __ andw(as_Register($dst$$reg),
11620 as_Register($src1$$reg),
11621 as_Register($src2$$reg),
11622 Assembler::LSR,
11623 $src3$$constant & 0x1f);
11624 %}
11625
11626 ins_pipe(ialu_reg_reg_shift);
11627 %}
11628
11629 // This pattern is automatically generated from aarch64_ad.m4
11630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11631 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11632 iRegL src1, iRegL src2,
11633 immI src3) %{
11634 match(Set dst (AndL src1 (URShiftL src2 src3)));
11635
11636 ins_cost(1.9 * INSN_COST);
11637 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11638
11639 ins_encode %{
11640 __ andr(as_Register($dst$$reg),
11641 as_Register($src1$$reg),
11642 as_Register($src2$$reg),
11643 Assembler::LSR,
11644 $src3$$constant & 0x3f);
11645 %}
11646
11647 ins_pipe(ialu_reg_reg_shift);
11648 %}
11649
11650 // This pattern is automatically generated from aarch64_ad.m4
11651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11652 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11653 iRegIorL2I src1, iRegIorL2I src2,
11654 immI src3) %{
11655 match(Set dst (AndI src1 (RShiftI src2 src3)));
11656
11657 ins_cost(1.9 * INSN_COST);
11658 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11659
11660 ins_encode %{
11661 __ andw(as_Register($dst$$reg),
11662 as_Register($src1$$reg),
11663 as_Register($src2$$reg),
11664 Assembler::ASR,
11665 $src3$$constant & 0x1f);
11666 %}
11667
11668 ins_pipe(ialu_reg_reg_shift);
11669 %}
11670
11671 // This pattern is automatically generated from aarch64_ad.m4
11672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11673 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11674 iRegL src1, iRegL src2,
11675 immI src3) %{
11676 match(Set dst (AndL src1 (RShiftL src2 src3)));
11677
11678 ins_cost(1.9 * INSN_COST);
11679 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11680
11681 ins_encode %{
11682 __ andr(as_Register($dst$$reg),
11683 as_Register($src1$$reg),
11684 as_Register($src2$$reg),
11685 Assembler::ASR,
11686 $src3$$constant & 0x3f);
11687 %}
11688
11689 ins_pipe(ialu_reg_reg_shift);
11690 %}
11691
11692 // This pattern is automatically generated from aarch64_ad.m4
11693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11694 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11695 iRegIorL2I src1, iRegIorL2I src2,
11696 immI src3) %{
11697 match(Set dst (AndI src1 (LShiftI src2 src3)));
11698
11699 ins_cost(1.9 * INSN_COST);
11700 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11701
11702 ins_encode %{
11703 __ andw(as_Register($dst$$reg),
11704 as_Register($src1$$reg),
11705 as_Register($src2$$reg),
11706 Assembler::LSL,
11707 $src3$$constant & 0x1f);
11708 %}
11709
11710 ins_pipe(ialu_reg_reg_shift);
11711 %}
11712
11713 // This pattern is automatically generated from aarch64_ad.m4
11714 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11715 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11716 iRegL src1, iRegL src2,
11717 immI src3) %{
11718 match(Set dst (AndL src1 (LShiftL src2 src3)));
11719
11720 ins_cost(1.9 * INSN_COST);
11721 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11722
11723 ins_encode %{
11724 __ andr(as_Register($dst$$reg),
11725 as_Register($src1$$reg),
11726 as_Register($src2$$reg),
11727 Assembler::LSL,
11728 $src3$$constant & 0x3f);
11729 %}
11730
11731 ins_pipe(ialu_reg_reg_shift);
11732 %}
11733
11734 // This pattern is automatically generated from aarch64_ad.m4
11735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11736 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11737 iRegIorL2I src1, iRegIorL2I src2,
11738 immI src3) %{
11739 match(Set dst (AndI src1 (RotateRight src2 src3)));
11740
11741 ins_cost(1.9 * INSN_COST);
11742 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11743
11744 ins_encode %{
11745 __ andw(as_Register($dst$$reg),
11746 as_Register($src1$$reg),
11747 as_Register($src2$$reg),
11748 Assembler::ROR,
11749 $src3$$constant & 0x1f);
11750 %}
11751
11752 ins_pipe(ialu_reg_reg_shift);
11753 %}
11754
11755 // This pattern is automatically generated from aarch64_ad.m4
11756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11757 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11758 iRegL src1, iRegL src2,
11759 immI src3) %{
11760 match(Set dst (AndL src1 (RotateRight src2 src3)));
11761
11762 ins_cost(1.9 * INSN_COST);
11763 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11764
11765 ins_encode %{
11766 __ andr(as_Register($dst$$reg),
11767 as_Register($src1$$reg),
11768 as_Register($src2$$reg),
11769 Assembler::ROR,
11770 $src3$$constant & 0x3f);
11771 %}
11772
11773 ins_pipe(ialu_reg_reg_shift);
11774 %}
11775
11776 // This pattern is automatically generated from aarch64_ad.m4
11777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11778 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11779 iRegIorL2I src1, iRegIorL2I src2,
11780 immI src3) %{
11781 match(Set dst (XorI src1 (URShiftI src2 src3)));
11782
11783 ins_cost(1.9 * INSN_COST);
11784 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11785
11786 ins_encode %{
11787 __ eorw(as_Register($dst$$reg),
11788 as_Register($src1$$reg),
11789 as_Register($src2$$reg),
11790 Assembler::LSR,
11791 $src3$$constant & 0x1f);
11792 %}
11793
11794 ins_pipe(ialu_reg_reg_shift);
11795 %}
11796
11797 // This pattern is automatically generated from aarch64_ad.m4
11798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11799 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11800 iRegL src1, iRegL src2,
11801 immI src3) %{
11802 match(Set dst (XorL src1 (URShiftL src2 src3)));
11803
11804 ins_cost(1.9 * INSN_COST);
11805 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11806
11807 ins_encode %{
11808 __ eor(as_Register($dst$$reg),
11809 as_Register($src1$$reg),
11810 as_Register($src2$$reg),
11811 Assembler::LSR,
11812 $src3$$constant & 0x3f);
11813 %}
11814
11815 ins_pipe(ialu_reg_reg_shift);
11816 %}
11817
11818 // This pattern is automatically generated from aarch64_ad.m4
11819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11820 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11821 iRegIorL2I src1, iRegIorL2I src2,
11822 immI src3) %{
11823 match(Set dst (XorI src1 (RShiftI src2 src3)));
11824
11825 ins_cost(1.9 * INSN_COST);
11826 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11827
11828 ins_encode %{
11829 __ eorw(as_Register($dst$$reg),
11830 as_Register($src1$$reg),
11831 as_Register($src2$$reg),
11832 Assembler::ASR,
11833 $src3$$constant & 0x1f);
11834 %}
11835
11836 ins_pipe(ialu_reg_reg_shift);
11837 %}
11838
11839 // This pattern is automatically generated from aarch64_ad.m4
11840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11841 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11842 iRegL src1, iRegL src2,
11843 immI src3) %{
11844 match(Set dst (XorL src1 (RShiftL src2 src3)));
11845
11846 ins_cost(1.9 * INSN_COST);
11847 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11848
11849 ins_encode %{
11850 __ eor(as_Register($dst$$reg),
11851 as_Register($src1$$reg),
11852 as_Register($src2$$reg),
11853 Assembler::ASR,
11854 $src3$$constant & 0x3f);
11855 %}
11856
11857 ins_pipe(ialu_reg_reg_shift);
11858 %}
11859
11860 // This pattern is automatically generated from aarch64_ad.m4
11861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11862 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11863 iRegIorL2I src1, iRegIorL2I src2,
11864 immI src3) %{
11865 match(Set dst (XorI src1 (LShiftI src2 src3)));
11866
11867 ins_cost(1.9 * INSN_COST);
11868 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11869
11870 ins_encode %{
11871 __ eorw(as_Register($dst$$reg),
11872 as_Register($src1$$reg),
11873 as_Register($src2$$reg),
11874 Assembler::LSL,
11875 $src3$$constant & 0x1f);
11876 %}
11877
11878 ins_pipe(ialu_reg_reg_shift);
11879 %}
11880
11881 // This pattern is automatically generated from aarch64_ad.m4
11882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11883 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11884 iRegL src1, iRegL src2,
11885 immI src3) %{
11886 match(Set dst (XorL src1 (LShiftL src2 src3)));
11887
11888 ins_cost(1.9 * INSN_COST);
11889 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11890
11891 ins_encode %{
11892 __ eor(as_Register($dst$$reg),
11893 as_Register($src1$$reg),
11894 as_Register($src2$$reg),
11895 Assembler::LSL,
11896 $src3$$constant & 0x3f);
11897 %}
11898
11899 ins_pipe(ialu_reg_reg_shift);
11900 %}
11901
11902 // This pattern is automatically generated from aarch64_ad.m4
11903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11904 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11905 iRegIorL2I src1, iRegIorL2I src2,
11906 immI src3) %{
11907 match(Set dst (XorI src1 (RotateRight src2 src3)));
11908
11909 ins_cost(1.9 * INSN_COST);
11910 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11911
11912 ins_encode %{
11913 __ eorw(as_Register($dst$$reg),
11914 as_Register($src1$$reg),
11915 as_Register($src2$$reg),
11916 Assembler::ROR,
11917 $src3$$constant & 0x1f);
11918 %}
11919
11920 ins_pipe(ialu_reg_reg_shift);
11921 %}
11922
11923 // This pattern is automatically generated from aarch64_ad.m4
11924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11925 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11926 iRegL src1, iRegL src2,
11927 immI src3) %{
11928 match(Set dst (XorL src1 (RotateRight src2 src3)));
11929
11930 ins_cost(1.9 * INSN_COST);
11931 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11932
11933 ins_encode %{
11934 __ eor(as_Register($dst$$reg),
11935 as_Register($src1$$reg),
11936 as_Register($src2$$reg),
11937 Assembler::ROR,
11938 $src3$$constant & 0x3f);
11939 %}
11940
11941 ins_pipe(ialu_reg_reg_shift);
11942 %}
11943
11944 // This pattern is automatically generated from aarch64_ad.m4
11945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11946 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11947 iRegIorL2I src1, iRegIorL2I src2,
11948 immI src3) %{
11949 match(Set dst (OrI src1 (URShiftI src2 src3)));
11950
11951 ins_cost(1.9 * INSN_COST);
11952 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11953
11954 ins_encode %{
11955 __ orrw(as_Register($dst$$reg),
11956 as_Register($src1$$reg),
11957 as_Register($src2$$reg),
11958 Assembler::LSR,
11959 $src3$$constant & 0x1f);
11960 %}
11961
11962 ins_pipe(ialu_reg_reg_shift);
11963 %}
11964
11965 // This pattern is automatically generated from aarch64_ad.m4
11966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11967 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11968 iRegL src1, iRegL src2,
11969 immI src3) %{
11970 match(Set dst (OrL src1 (URShiftL src2 src3)));
11971
11972 ins_cost(1.9 * INSN_COST);
11973 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11974
11975 ins_encode %{
11976 __ orr(as_Register($dst$$reg),
11977 as_Register($src1$$reg),
11978 as_Register($src2$$reg),
11979 Assembler::LSR,
11980 $src3$$constant & 0x3f);
11981 %}
11982
11983 ins_pipe(ialu_reg_reg_shift);
11984 %}
11985
11986 // This pattern is automatically generated from aarch64_ad.m4
11987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11988 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11989 iRegIorL2I src1, iRegIorL2I src2,
11990 immI src3) %{
11991 match(Set dst (OrI src1 (RShiftI src2 src3)));
11992
11993 ins_cost(1.9 * INSN_COST);
11994 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11995
11996 ins_encode %{
11997 __ orrw(as_Register($dst$$reg),
11998 as_Register($src1$$reg),
11999 as_Register($src2$$reg),
12000 Assembler::ASR,
12001 $src3$$constant & 0x1f);
12002 %}
12003
12004 ins_pipe(ialu_reg_reg_shift);
12005 %}
12006
12007 // This pattern is automatically generated from aarch64_ad.m4
12008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12009 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12010 iRegL src1, iRegL src2,
12011 immI src3) %{
12012 match(Set dst (OrL src1 (RShiftL src2 src3)));
12013
12014 ins_cost(1.9 * INSN_COST);
12015 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12016
12017 ins_encode %{
12018 __ orr(as_Register($dst$$reg),
12019 as_Register($src1$$reg),
12020 as_Register($src2$$reg),
12021 Assembler::ASR,
12022 $src3$$constant & 0x3f);
12023 %}
12024
12025 ins_pipe(ialu_reg_reg_shift);
12026 %}
12027
12028 // This pattern is automatically generated from aarch64_ad.m4
12029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12030 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12031 iRegIorL2I src1, iRegIorL2I src2,
12032 immI src3) %{
12033 match(Set dst (OrI src1 (LShiftI src2 src3)));
12034
12035 ins_cost(1.9 * INSN_COST);
12036 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12037
12038 ins_encode %{
12039 __ orrw(as_Register($dst$$reg),
12040 as_Register($src1$$reg),
12041 as_Register($src2$$reg),
12042 Assembler::LSL,
12043 $src3$$constant & 0x1f);
12044 %}
12045
12046 ins_pipe(ialu_reg_reg_shift);
12047 %}
12048
12049 // This pattern is automatically generated from aarch64_ad.m4
12050 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12051 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12052 iRegL src1, iRegL src2,
12053 immI src3) %{
12054 match(Set dst (OrL src1 (LShiftL src2 src3)));
12055
12056 ins_cost(1.9 * INSN_COST);
12057 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12058
12059 ins_encode %{
12060 __ orr(as_Register($dst$$reg),
12061 as_Register($src1$$reg),
12062 as_Register($src2$$reg),
12063 Assembler::LSL,
12064 $src3$$constant & 0x3f);
12065 %}
12066
12067 ins_pipe(ialu_reg_reg_shift);
12068 %}
12069
12070 // This pattern is automatically generated from aarch64_ad.m4
12071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12072 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12073 iRegIorL2I src1, iRegIorL2I src2,
12074 immI src3) %{
12075 match(Set dst (OrI src1 (RotateRight src2 src3)));
12076
12077 ins_cost(1.9 * INSN_COST);
12078 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12079
12080 ins_encode %{
12081 __ orrw(as_Register($dst$$reg),
12082 as_Register($src1$$reg),
12083 as_Register($src2$$reg),
12084 Assembler::ROR,
12085 $src3$$constant & 0x1f);
12086 %}
12087
12088 ins_pipe(ialu_reg_reg_shift);
12089 %}
12090
12091 // This pattern is automatically generated from aarch64_ad.m4
12092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12093 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12094 iRegL src1, iRegL src2,
12095 immI src3) %{
12096 match(Set dst (OrL src1 (RotateRight src2 src3)));
12097
12098 ins_cost(1.9 * INSN_COST);
12099 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12100
12101 ins_encode %{
12102 __ orr(as_Register($dst$$reg),
12103 as_Register($src1$$reg),
12104 as_Register($src2$$reg),
12105 Assembler::ROR,
12106 $src3$$constant & 0x3f);
12107 %}
12108
12109 ins_pipe(ialu_reg_reg_shift);
12110 %}
12111
12112 // This pattern is automatically generated from aarch64_ad.m4
12113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12114 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12115 iRegIorL2I src1, iRegIorL2I src2,
12116 immI src3) %{
12117 match(Set dst (AddI src1 (URShiftI src2 src3)));
12118
12119 ins_cost(1.9 * INSN_COST);
12120 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12121
12122 ins_encode %{
12123 __ addw(as_Register($dst$$reg),
12124 as_Register($src1$$reg),
12125 as_Register($src2$$reg),
12126 Assembler::LSR,
12127 $src3$$constant & 0x1f);
12128 %}
12129
12130 ins_pipe(ialu_reg_reg_shift);
12131 %}
12132
12133 // This pattern is automatically generated from aarch64_ad.m4
12134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12135 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12136 iRegL src1, iRegL src2,
12137 immI src3) %{
12138 match(Set dst (AddL src1 (URShiftL src2 src3)));
12139
12140 ins_cost(1.9 * INSN_COST);
12141 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12142
12143 ins_encode %{
12144 __ add(as_Register($dst$$reg),
12145 as_Register($src1$$reg),
12146 as_Register($src2$$reg),
12147 Assembler::LSR,
12148 $src3$$constant & 0x3f);
12149 %}
12150
12151 ins_pipe(ialu_reg_reg_shift);
12152 %}
12153
12154 // This pattern is automatically generated from aarch64_ad.m4
12155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12156 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12157 iRegIorL2I src1, iRegIorL2I src2,
12158 immI src3) %{
12159 match(Set dst (AddI src1 (RShiftI src2 src3)));
12160
12161 ins_cost(1.9 * INSN_COST);
12162 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12163
12164 ins_encode %{
12165 __ addw(as_Register($dst$$reg),
12166 as_Register($src1$$reg),
12167 as_Register($src2$$reg),
12168 Assembler::ASR,
12169 $src3$$constant & 0x1f);
12170 %}
12171
12172 ins_pipe(ialu_reg_reg_shift);
12173 %}
12174
12175 // This pattern is automatically generated from aarch64_ad.m4
12176 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12177 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12178 iRegL src1, iRegL src2,
12179 immI src3) %{
12180 match(Set dst (AddL src1 (RShiftL src2 src3)));
12181
12182 ins_cost(1.9 * INSN_COST);
12183 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12184
12185 ins_encode %{
12186 __ add(as_Register($dst$$reg),
12187 as_Register($src1$$reg),
12188 as_Register($src2$$reg),
12189 Assembler::ASR,
12190 $src3$$constant & 0x3f);
12191 %}
12192
12193 ins_pipe(ialu_reg_reg_shift);
12194 %}
12195
12196 // This pattern is automatically generated from aarch64_ad.m4
12197 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12198 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12199 iRegIorL2I src1, iRegIorL2I src2,
12200 immI src3) %{
12201 match(Set dst (AddI src1 (LShiftI src2 src3)));
12202
12203 ins_cost(1.9 * INSN_COST);
12204 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12205
12206 ins_encode %{
12207 __ addw(as_Register($dst$$reg),
12208 as_Register($src1$$reg),
12209 as_Register($src2$$reg),
12210 Assembler::LSL,
12211 $src3$$constant & 0x1f);
12212 %}
12213
12214 ins_pipe(ialu_reg_reg_shift);
12215 %}
12216
12217 // This pattern is automatically generated from aarch64_ad.m4
12218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12219 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12220 iRegL src1, iRegL src2,
12221 immI src3) %{
12222 match(Set dst (AddL src1 (LShiftL src2 src3)));
12223
12224 ins_cost(1.9 * INSN_COST);
12225 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12226
12227 ins_encode %{
12228 __ add(as_Register($dst$$reg),
12229 as_Register($src1$$reg),
12230 as_Register($src2$$reg),
12231 Assembler::LSL,
12232 $src3$$constant & 0x3f);
12233 %}
12234
12235 ins_pipe(ialu_reg_reg_shift);
12236 %}
12237
12238 // This pattern is automatically generated from aarch64_ad.m4
12239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12240 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12241 iRegIorL2I src1, iRegIorL2I src2,
12242 immI src3) %{
12243 match(Set dst (SubI src1 (URShiftI src2 src3)));
12244
12245 ins_cost(1.9 * INSN_COST);
12246 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12247
12248 ins_encode %{
12249 __ subw(as_Register($dst$$reg),
12250 as_Register($src1$$reg),
12251 as_Register($src2$$reg),
12252 Assembler::LSR,
12253 $src3$$constant & 0x1f);
12254 %}
12255
12256 ins_pipe(ialu_reg_reg_shift);
12257 %}
12258
12259 // This pattern is automatically generated from aarch64_ad.m4
12260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12261 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12262 iRegL src1, iRegL src2,
12263 immI src3) %{
12264 match(Set dst (SubL src1 (URShiftL src2 src3)));
12265
12266 ins_cost(1.9 * INSN_COST);
12267 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12268
12269 ins_encode %{
12270 __ sub(as_Register($dst$$reg),
12271 as_Register($src1$$reg),
12272 as_Register($src2$$reg),
12273 Assembler::LSR,
12274 $src3$$constant & 0x3f);
12275 %}
12276
12277 ins_pipe(ialu_reg_reg_shift);
12278 %}
12279
12280 // This pattern is automatically generated from aarch64_ad.m4
12281 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12282 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12283 iRegIorL2I src1, iRegIorL2I src2,
12284 immI src3) %{
12285 match(Set dst (SubI src1 (RShiftI src2 src3)));
12286
12287 ins_cost(1.9 * INSN_COST);
12288 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12289
12290 ins_encode %{
12291 __ subw(as_Register($dst$$reg),
12292 as_Register($src1$$reg),
12293 as_Register($src2$$reg),
12294 Assembler::ASR,
12295 $src3$$constant & 0x1f);
12296 %}
12297
12298 ins_pipe(ialu_reg_reg_shift);
12299 %}
12300
12301 // This pattern is automatically generated from aarch64_ad.m4
12302 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12303 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12304 iRegL src1, iRegL src2,
12305 immI src3) %{
12306 match(Set dst (SubL src1 (RShiftL src2 src3)));
12307
12308 ins_cost(1.9 * INSN_COST);
12309 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12310
12311 ins_encode %{
12312 __ sub(as_Register($dst$$reg),
12313 as_Register($src1$$reg),
12314 as_Register($src2$$reg),
12315 Assembler::ASR,
12316 $src3$$constant & 0x3f);
12317 %}
12318
12319 ins_pipe(ialu_reg_reg_shift);
12320 %}
12321
12322 // This pattern is automatically generated from aarch64_ad.m4
12323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12324 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12325 iRegIorL2I src1, iRegIorL2I src2,
12326 immI src3) %{
12327 match(Set dst (SubI src1 (LShiftI src2 src3)));
12328
12329 ins_cost(1.9 * INSN_COST);
12330 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12331
12332 ins_encode %{
12333 __ subw(as_Register($dst$$reg),
12334 as_Register($src1$$reg),
12335 as_Register($src2$$reg),
12336 Assembler::LSL,
12337 $src3$$constant & 0x1f);
12338 %}
12339
12340 ins_pipe(ialu_reg_reg_shift);
12341 %}
12342
12343 // This pattern is automatically generated from aarch64_ad.m4
12344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12345 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12346 iRegL src1, iRegL src2,
12347 immI src3) %{
12348 match(Set dst (SubL src1 (LShiftL src2 src3)));
12349
12350 ins_cost(1.9 * INSN_COST);
12351 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12352
12353 ins_encode %{
12354 __ sub(as_Register($dst$$reg),
12355 as_Register($src1$$reg),
12356 as_Register($src2$$reg),
12357 Assembler::LSL,
12358 $src3$$constant & 0x3f);
12359 %}
12360
12361 ins_pipe(ialu_reg_reg_shift);
12362 %}
12363
12364 // This pattern is automatically generated from aarch64_ad.m4
12365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12366
12367 // Shift Left followed by Shift Right.
12368 // This idiom is used by the compiler for the i2b bytecode etc.
12369 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12370 %{
12371 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12372 ins_cost(INSN_COST * 2);
12373 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12374 ins_encode %{
12375 int lshift = $lshift_count$$constant & 63;
12376 int rshift = $rshift_count$$constant & 63;
12377 int s = 63 - lshift;
12378 int r = (rshift - lshift) & 63;
12379 __ sbfm(as_Register($dst$$reg),
12380 as_Register($src$$reg),
12381 r, s);
12382 %}
12383
12384 ins_pipe(ialu_reg_shift);
12385 %}
12386
12387 // This pattern is automatically generated from aarch64_ad.m4
12388 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12389
12390 // Shift Left followed by Shift Right.
12391 // This idiom is used by the compiler for the i2b bytecode etc.
12392 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12393 %{
12394 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12395 ins_cost(INSN_COST * 2);
12396 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12397 ins_encode %{
12398 int lshift = $lshift_count$$constant & 31;
12399 int rshift = $rshift_count$$constant & 31;
12400 int s = 31 - lshift;
12401 int r = (rshift - lshift) & 31;
12402 __ sbfmw(as_Register($dst$$reg),
12403 as_Register($src$$reg),
12404 r, s);
12405 %}
12406
12407 ins_pipe(ialu_reg_shift);
12408 %}
12409
12410 // This pattern is automatically generated from aarch64_ad.m4
12411 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12412
12413 // Shift Left followed by Shift Right.
12414 // This idiom is used by the compiler for the i2b bytecode etc.
12415 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12416 %{
12417 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12418 ins_cost(INSN_COST * 2);
12419 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12420 ins_encode %{
12421 int lshift = $lshift_count$$constant & 63;
12422 int rshift = $rshift_count$$constant & 63;
12423 int s = 63 - lshift;
12424 int r = (rshift - lshift) & 63;
12425 __ ubfm(as_Register($dst$$reg),
12426 as_Register($src$$reg),
12427 r, s);
12428 %}
12429
12430 ins_pipe(ialu_reg_shift);
12431 %}
12432
12433 // This pattern is automatically generated from aarch64_ad.m4
12434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12435
12436 // Shift Left followed by Shift Right.
12437 // This idiom is used by the compiler for the i2b bytecode etc.
12438 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12439 %{
12440 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12441 ins_cost(INSN_COST * 2);
12442 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12443 ins_encode %{
12444 int lshift = $lshift_count$$constant & 31;
12445 int rshift = $rshift_count$$constant & 31;
12446 int s = 31 - lshift;
12447 int r = (rshift - lshift) & 31;
12448 __ ubfmw(as_Register($dst$$reg),
12449 as_Register($src$$reg),
12450 r, s);
12451 %}
12452
12453 ins_pipe(ialu_reg_shift);
12454 %}
12455
12456 // Bitfield extract with shift & mask
12457
12458 // This pattern is automatically generated from aarch64_ad.m4
12459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12460 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12461 %{
12462 match(Set dst (AndI (URShiftI src rshift) mask));
12463 // Make sure we are not going to exceed what ubfxw can do.
12464 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12465
12466 ins_cost(INSN_COST);
12467 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12468 ins_encode %{
12469 int rshift = $rshift$$constant & 31;
12470 intptr_t mask = $mask$$constant;
12471 int width = exact_log2(mask+1);
12472 __ ubfxw(as_Register($dst$$reg),
12473 as_Register($src$$reg), rshift, width);
12474 %}
12475 ins_pipe(ialu_reg_shift);
12476 %}
12477
12478 // This pattern is automatically generated from aarch64_ad.m4
12479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12480 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12481 %{
12482 match(Set dst (AndL (URShiftL src rshift) mask));
12483 // Make sure we are not going to exceed what ubfx can do.
12484 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12485
12486 ins_cost(INSN_COST);
12487 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12488 ins_encode %{
12489 int rshift = $rshift$$constant & 63;
12490 intptr_t mask = $mask$$constant;
12491 int width = exact_log2_long(mask+1);
12492 __ ubfx(as_Register($dst$$reg),
12493 as_Register($src$$reg), rshift, width);
12494 %}
12495 ins_pipe(ialu_reg_shift);
12496 %}
12497
12498
12499 // This pattern is automatically generated from aarch64_ad.m4
12500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12501
12502 // We can use ubfx when extending an And with a mask when we know mask
12503 // is positive. We know that because immI_bitmask guarantees it.
12504 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12505 %{
12506 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12507 // Make sure we are not going to exceed what ubfxw can do.
12508 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12509
12510 ins_cost(INSN_COST * 2);
12511 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12512 ins_encode %{
12513 int rshift = $rshift$$constant & 31;
12514 intptr_t mask = $mask$$constant;
12515 int width = exact_log2(mask+1);
12516 __ ubfx(as_Register($dst$$reg),
12517 as_Register($src$$reg), rshift, width);
12518 %}
12519 ins_pipe(ialu_reg_shift);
12520 %}
12521
12522
12523 // This pattern is automatically generated from aarch64_ad.m4
12524 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12525
12526 // We can use ubfiz when masking by a positive number and then left shifting the result.
12527 // We know that the mask is positive because immI_bitmask guarantees it.
12528 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12529 %{
12530 match(Set dst (LShiftI (AndI src mask) lshift));
12531 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12532
12533 ins_cost(INSN_COST);
12534 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12535 ins_encode %{
12536 int lshift = $lshift$$constant & 31;
12537 intptr_t mask = $mask$$constant;
12538 int width = exact_log2(mask+1);
12539 __ ubfizw(as_Register($dst$$reg),
12540 as_Register($src$$reg), lshift, width);
12541 %}
12542 ins_pipe(ialu_reg_shift);
12543 %}
12544
12545 // This pattern is automatically generated from aarch64_ad.m4
12546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12547
12548 // We can use ubfiz when masking by a positive number and then left shifting the result.
12549 // We know that the mask is positive because immL_bitmask guarantees it.
12550 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12551 %{
12552 match(Set dst (LShiftL (AndL src mask) lshift));
12553 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12554
12555 ins_cost(INSN_COST);
12556 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12557 ins_encode %{
12558 int lshift = $lshift$$constant & 63;
12559 intptr_t mask = $mask$$constant;
12560 int width = exact_log2_long(mask+1);
12561 __ ubfiz(as_Register($dst$$reg),
12562 as_Register($src$$reg), lshift, width);
12563 %}
12564 ins_pipe(ialu_reg_shift);
12565 %}
12566
12567 // This pattern is automatically generated from aarch64_ad.m4
12568 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12569
12570 // We can use ubfiz when masking by a positive number and then left shifting the result.
12571 // We know that the mask is positive because immI_bitmask guarantees it.
12572 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12573 %{
12574 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12575 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12576
12577 ins_cost(INSN_COST);
12578 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12579 ins_encode %{
12580 int lshift = $lshift$$constant & 31;
12581 intptr_t mask = $mask$$constant;
12582 int width = exact_log2(mask+1);
12583 __ ubfizw(as_Register($dst$$reg),
12584 as_Register($src$$reg), lshift, width);
12585 %}
12586 ins_pipe(ialu_reg_shift);
12587 %}
12588
12589 // This pattern is automatically generated from aarch64_ad.m4
12590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12591
12592 // We can use ubfiz when masking by a positive number and then left shifting the result.
12593 // We know that the mask is positive because immL_bitmask guarantees it.
12594 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12595 %{
12596 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12597 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12598
12599 ins_cost(INSN_COST);
12600 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12601 ins_encode %{
12602 int lshift = $lshift$$constant & 63;
12603 intptr_t mask = $mask$$constant;
12604 int width = exact_log2_long(mask+1);
12605 __ ubfiz(as_Register($dst$$reg),
12606 as_Register($src$$reg), lshift, width);
12607 %}
12608 ins_pipe(ialu_reg_shift);
12609 %}
12610
12611
12612 // This pattern is automatically generated from aarch64_ad.m4
12613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12614
12615 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12616 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12617 %{
12618 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12619 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12620
12621 ins_cost(INSN_COST);
12622 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12623 ins_encode %{
12624 int lshift = $lshift$$constant & 63;
12625 intptr_t mask = $mask$$constant;
12626 int width = exact_log2(mask+1);
12627 __ ubfiz(as_Register($dst$$reg),
12628 as_Register($src$$reg), lshift, width);
12629 %}
12630 ins_pipe(ialu_reg_shift);
12631 %}
12632
12633 // This pattern is automatically generated from aarch64_ad.m4
12634 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12635
12636 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12637 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12638 %{
12639 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12640 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12641
12642 ins_cost(INSN_COST);
12643 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12644 ins_encode %{
12645 int lshift = $lshift$$constant & 31;
12646 intptr_t mask = $mask$$constant;
12647 int width = exact_log2(mask+1);
12648 __ ubfiz(as_Register($dst$$reg),
12649 as_Register($src$$reg), lshift, width);
12650 %}
12651 ins_pipe(ialu_reg_shift);
12652 %}
12653
12654 // This pattern is automatically generated from aarch64_ad.m4
12655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12656
12657 // Can skip int2long conversions after AND with small bitmask
12658 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12659 %{
12660 match(Set dst (ConvI2L (AndI src msk)));
12661 ins_cost(INSN_COST);
12662 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12663 ins_encode %{
12664 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12665 %}
12666 ins_pipe(ialu_reg_shift);
12667 %}
12668
12669
12670 // Rotations
12671
12672 // This pattern is automatically generated from aarch64_ad.m4
12673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12674 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12675 %{
12676 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12677 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12678
12679 ins_cost(INSN_COST);
12680 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12681
12682 ins_encode %{
12683 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12684 $rshift$$constant & 63);
12685 %}
12686 ins_pipe(ialu_reg_reg_extr);
12687 %}
12688
12689
12690 // This pattern is automatically generated from aarch64_ad.m4
12691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12692 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12693 %{
12694 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12695 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12696
12697 ins_cost(INSN_COST);
12698 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12699
12700 ins_encode %{
12701 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12702 $rshift$$constant & 31);
12703 %}
12704 ins_pipe(ialu_reg_reg_extr);
12705 %}
12706
12707
12708 // This pattern is automatically generated from aarch64_ad.m4
12709 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12710 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12711 %{
12712 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12713 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12714
12715 ins_cost(INSN_COST);
12716 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12717
12718 ins_encode %{
12719 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12720 $rshift$$constant & 63);
12721 %}
12722 ins_pipe(ialu_reg_reg_extr);
12723 %}
12724
12725
12726 // This pattern is automatically generated from aarch64_ad.m4
12727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12728 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12729 %{
12730 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12731 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12732
12733 ins_cost(INSN_COST);
12734 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12735
12736 ins_encode %{
12737 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12738 $rshift$$constant & 31);
12739 %}
12740 ins_pipe(ialu_reg_reg_extr);
12741 %}
12742
12743 // This pattern is automatically generated from aarch64_ad.m4
12744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12745 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12746 %{
12747 match(Set dst (RotateRight src shift));
12748
12749 ins_cost(INSN_COST);
12750 format %{ "ror $dst, $src, $shift" %}
12751
12752 ins_encode %{
12753 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12754 $shift$$constant & 0x1f);
12755 %}
12756 ins_pipe(ialu_reg_reg_vshift);
12757 %}
12758
12759 // This pattern is automatically generated from aarch64_ad.m4
12760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12761 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12762 %{
12763 match(Set dst (RotateRight src shift));
12764
12765 ins_cost(INSN_COST);
12766 format %{ "ror $dst, $src, $shift" %}
12767
12768 ins_encode %{
12769 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12770 $shift$$constant & 0x3f);
12771 %}
12772 ins_pipe(ialu_reg_reg_vshift);
12773 %}
12774
12775 // This pattern is automatically generated from aarch64_ad.m4
12776 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12777 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12778 %{
12779 match(Set dst (RotateRight src shift));
12780
12781 ins_cost(INSN_COST);
12782 format %{ "ror $dst, $src, $shift" %}
12783
12784 ins_encode %{
12785 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12786 %}
12787 ins_pipe(ialu_reg_reg_vshift);
12788 %}
12789
12790 // This pattern is automatically generated from aarch64_ad.m4
12791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12792 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12793 %{
12794 match(Set dst (RotateRight src shift));
12795
12796 ins_cost(INSN_COST);
12797 format %{ "ror $dst, $src, $shift" %}
12798
12799 ins_encode %{
12800 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12801 %}
12802 ins_pipe(ialu_reg_reg_vshift);
12803 %}
12804
12805 // This pattern is automatically generated from aarch64_ad.m4
12806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12807 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12808 %{
12809 match(Set dst (RotateLeft src shift));
12810
12811 ins_cost(INSN_COST);
12812 format %{ "rol $dst, $src, $shift" %}
12813
12814 ins_encode %{
12815 __ subw(rscratch1, zr, as_Register($shift$$reg));
12816 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12817 %}
12818 ins_pipe(ialu_reg_reg_vshift);
12819 %}
12820
12821 // This pattern is automatically generated from aarch64_ad.m4
12822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12823 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12824 %{
12825 match(Set dst (RotateLeft src shift));
12826
12827 ins_cost(INSN_COST);
12828 format %{ "rol $dst, $src, $shift" %}
12829
12830 ins_encode %{
12831 __ subw(rscratch1, zr, as_Register($shift$$reg));
12832 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12833 %}
12834 ins_pipe(ialu_reg_reg_vshift);
12835 %}
12836
12837
12838 // Add/subtract (extended)
12839
12840 // This pattern is automatically generated from aarch64_ad.m4
12841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12842 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12843 %{
12844 match(Set dst (AddL src1 (ConvI2L src2)));
12845 ins_cost(INSN_COST);
12846 format %{ "add $dst, $src1, $src2, sxtw" %}
12847
12848 ins_encode %{
12849 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12850 as_Register($src2$$reg), ext::sxtw);
12851 %}
12852 ins_pipe(ialu_reg_reg);
12853 %}
12854
12855 // This pattern is automatically generated from aarch64_ad.m4
12856 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12857 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12858 %{
12859 match(Set dst (SubL src1 (ConvI2L src2)));
12860 ins_cost(INSN_COST);
12861 format %{ "sub $dst, $src1, $src2, sxtw" %}
12862
12863 ins_encode %{
12864 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12865 as_Register($src2$$reg), ext::sxtw);
12866 %}
12867 ins_pipe(ialu_reg_reg);
12868 %}
12869
12870 // This pattern is automatically generated from aarch64_ad.m4
12871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12872 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12873 %{
12874 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12875 ins_cost(INSN_COST);
12876 format %{ "add $dst, $src1, $src2, sxth" %}
12877
12878 ins_encode %{
12879 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12880 as_Register($src2$$reg), ext::sxth);
12881 %}
12882 ins_pipe(ialu_reg_reg);
12883 %}
12884
12885 // This pattern is automatically generated from aarch64_ad.m4
12886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12887 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12888 %{
12889 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12890 ins_cost(INSN_COST);
12891 format %{ "add $dst, $src1, $src2, sxtb" %}
12892
12893 ins_encode %{
12894 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12895 as_Register($src2$$reg), ext::sxtb);
12896 %}
12897 ins_pipe(ialu_reg_reg);
12898 %}
12899
12900 // This pattern is automatically generated from aarch64_ad.m4
12901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12902 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12903 %{
12904 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12905 ins_cost(INSN_COST);
12906 format %{ "add $dst, $src1, $src2, uxtb" %}
12907
12908 ins_encode %{
12909 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12910 as_Register($src2$$reg), ext::uxtb);
12911 %}
12912 ins_pipe(ialu_reg_reg);
12913 %}
12914
12915 // This pattern is automatically generated from aarch64_ad.m4
12916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12917 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12918 %{
12919 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12920 ins_cost(INSN_COST);
12921 format %{ "add $dst, $src1, $src2, sxth" %}
12922
12923 ins_encode %{
12924 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12925 as_Register($src2$$reg), ext::sxth);
12926 %}
12927 ins_pipe(ialu_reg_reg);
12928 %}
12929
12930 // This pattern is automatically generated from aarch64_ad.m4
12931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12932 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12933 %{
12934 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12935 ins_cost(INSN_COST);
12936 format %{ "add $dst, $src1, $src2, sxtw" %}
12937
12938 ins_encode %{
12939 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12940 as_Register($src2$$reg), ext::sxtw);
12941 %}
12942 ins_pipe(ialu_reg_reg);
12943 %}
12944
12945 // This pattern is automatically generated from aarch64_ad.m4
12946 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12947 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12948 %{
12949 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12950 ins_cost(INSN_COST);
12951 format %{ "add $dst, $src1, $src2, sxtb" %}
12952
12953 ins_encode %{
12954 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12955 as_Register($src2$$reg), ext::sxtb);
12956 %}
12957 ins_pipe(ialu_reg_reg);
12958 %}
12959
12960 // This pattern is automatically generated from aarch64_ad.m4
12961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12962 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12963 %{
12964 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12965 ins_cost(INSN_COST);
12966 format %{ "add $dst, $src1, $src2, uxtb" %}
12967
12968 ins_encode %{
12969 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12970 as_Register($src2$$reg), ext::uxtb);
12971 %}
12972 ins_pipe(ialu_reg_reg);
12973 %}
12974
12975 // This pattern is automatically generated from aarch64_ad.m4
12976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12977 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12978 %{
12979 match(Set dst (AddI src1 (AndI src2 mask)));
12980 ins_cost(INSN_COST);
12981 format %{ "addw $dst, $src1, $src2, uxtb" %}
12982
12983 ins_encode %{
12984 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12985 as_Register($src2$$reg), ext::uxtb);
12986 %}
12987 ins_pipe(ialu_reg_reg);
12988 %}
12989
12990 // This pattern is automatically generated from aarch64_ad.m4
12991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12992 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12993 %{
12994 match(Set dst (AddI src1 (AndI src2 mask)));
12995 ins_cost(INSN_COST);
12996 format %{ "addw $dst, $src1, $src2, uxth" %}
12997
12998 ins_encode %{
12999 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13000 as_Register($src2$$reg), ext::uxth);
13001 %}
13002 ins_pipe(ialu_reg_reg);
13003 %}
13004
13005 // This pattern is automatically generated from aarch64_ad.m4
13006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13007 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13008 %{
13009 match(Set dst (AddL src1 (AndL src2 mask)));
13010 ins_cost(INSN_COST);
13011 format %{ "add $dst, $src1, $src2, uxtb" %}
13012
13013 ins_encode %{
13014 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13015 as_Register($src2$$reg), ext::uxtb);
13016 %}
13017 ins_pipe(ialu_reg_reg);
13018 %}
13019
13020 // This pattern is automatically generated from aarch64_ad.m4
13021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13022 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13023 %{
13024 match(Set dst (AddL src1 (AndL src2 mask)));
13025 ins_cost(INSN_COST);
13026 format %{ "add $dst, $src1, $src2, uxth" %}
13027
13028 ins_encode %{
13029 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13030 as_Register($src2$$reg), ext::uxth);
13031 %}
13032 ins_pipe(ialu_reg_reg);
13033 %}
13034
13035 // This pattern is automatically generated from aarch64_ad.m4
13036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13037 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13038 %{
13039 match(Set dst (AddL src1 (AndL src2 mask)));
13040 ins_cost(INSN_COST);
13041 format %{ "add $dst, $src1, $src2, uxtw" %}
13042
13043 ins_encode %{
13044 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13045 as_Register($src2$$reg), ext::uxtw);
13046 %}
13047 ins_pipe(ialu_reg_reg);
13048 %}
13049
13050 // This pattern is automatically generated from aarch64_ad.m4
13051 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13052 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13053 %{
13054 match(Set dst (SubI src1 (AndI src2 mask)));
13055 ins_cost(INSN_COST);
13056 format %{ "subw $dst, $src1, $src2, uxtb" %}
13057
13058 ins_encode %{
13059 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13060 as_Register($src2$$reg), ext::uxtb);
13061 %}
13062 ins_pipe(ialu_reg_reg);
13063 %}
13064
13065 // This pattern is automatically generated from aarch64_ad.m4
13066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13067 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13068 %{
13069 match(Set dst (SubI src1 (AndI src2 mask)));
13070 ins_cost(INSN_COST);
13071 format %{ "subw $dst, $src1, $src2, uxth" %}
13072
13073 ins_encode %{
13074 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13075 as_Register($src2$$reg), ext::uxth);
13076 %}
13077 ins_pipe(ialu_reg_reg);
13078 %}
13079
13080 // This pattern is automatically generated from aarch64_ad.m4
13081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13082 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13083 %{
13084 match(Set dst (SubL src1 (AndL src2 mask)));
13085 ins_cost(INSN_COST);
13086 format %{ "sub $dst, $src1, $src2, uxtb" %}
13087
13088 ins_encode %{
13089 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13090 as_Register($src2$$reg), ext::uxtb);
13091 %}
13092 ins_pipe(ialu_reg_reg);
13093 %}
13094
13095 // This pattern is automatically generated from aarch64_ad.m4
13096 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13097 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13098 %{
13099 match(Set dst (SubL src1 (AndL src2 mask)));
13100 ins_cost(INSN_COST);
13101 format %{ "sub $dst, $src1, $src2, uxth" %}
13102
13103 ins_encode %{
13104 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13105 as_Register($src2$$reg), ext::uxth);
13106 %}
13107 ins_pipe(ialu_reg_reg);
13108 %}
13109
13110 // This pattern is automatically generated from aarch64_ad.m4
13111 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13112 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13113 %{
13114 match(Set dst (SubL src1 (AndL src2 mask)));
13115 ins_cost(INSN_COST);
13116 format %{ "sub $dst, $src1, $src2, uxtw" %}
13117
13118 ins_encode %{
13119 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13120 as_Register($src2$$reg), ext::uxtw);
13121 %}
13122 ins_pipe(ialu_reg_reg);
13123 %}
13124
13125
13126 // This pattern is automatically generated from aarch64_ad.m4
13127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13128 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13129 %{
13130 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13131 ins_cost(1.9 * INSN_COST);
13132 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13133
13134 ins_encode %{
13135 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13136 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13137 %}
13138 ins_pipe(ialu_reg_reg_shift);
13139 %}
13140
13141 // This pattern is automatically generated from aarch64_ad.m4
13142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13143 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13144 %{
13145 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13146 ins_cost(1.9 * INSN_COST);
13147 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13148
13149 ins_encode %{
13150 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13151 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13152 %}
13153 ins_pipe(ialu_reg_reg_shift);
13154 %}
13155
13156 // This pattern is automatically generated from aarch64_ad.m4
13157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13158 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13159 %{
13160 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13161 ins_cost(1.9 * INSN_COST);
13162 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13163
13164 ins_encode %{
13165 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13166 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13167 %}
13168 ins_pipe(ialu_reg_reg_shift);
13169 %}
13170
13171 // This pattern is automatically generated from aarch64_ad.m4
13172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13173 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13174 %{
13175 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13176 ins_cost(1.9 * INSN_COST);
13177 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13178
13179 ins_encode %{
13180 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13181 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13182 %}
13183 ins_pipe(ialu_reg_reg_shift);
13184 %}
13185
13186 // This pattern is automatically generated from aarch64_ad.m4
13187 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13188 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13189 %{
13190 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13191 ins_cost(1.9 * INSN_COST);
13192 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13193
13194 ins_encode %{
13195 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13196 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13197 %}
13198 ins_pipe(ialu_reg_reg_shift);
13199 %}
13200
13201 // This pattern is automatically generated from aarch64_ad.m4
13202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13203 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13204 %{
13205 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13206 ins_cost(1.9 * INSN_COST);
13207 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13208
13209 ins_encode %{
13210 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13211 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13212 %}
13213 ins_pipe(ialu_reg_reg_shift);
13214 %}
13215
13216 // This pattern is automatically generated from aarch64_ad.m4
13217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13218 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13219 %{
13220 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13221 ins_cost(1.9 * INSN_COST);
13222 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13223
13224 ins_encode %{
13225 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13226 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13227 %}
13228 ins_pipe(ialu_reg_reg_shift);
13229 %}
13230
13231 // This pattern is automatically generated from aarch64_ad.m4
13232 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13233 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13234 %{
13235 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13236 ins_cost(1.9 * INSN_COST);
13237 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13238
13239 ins_encode %{
13240 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13241 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13242 %}
13243 ins_pipe(ialu_reg_reg_shift);
13244 %}
13245
13246 // This pattern is automatically generated from aarch64_ad.m4
13247 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13248 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13249 %{
13250 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13251 ins_cost(1.9 * INSN_COST);
13252 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13253
13254 ins_encode %{
13255 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13256 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13257 %}
13258 ins_pipe(ialu_reg_reg_shift);
13259 %}
13260
13261 // This pattern is automatically generated from aarch64_ad.m4
13262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13263 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13264 %{
13265 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13266 ins_cost(1.9 * INSN_COST);
13267 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13268
13269 ins_encode %{
13270 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13271 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13272 %}
13273 ins_pipe(ialu_reg_reg_shift);
13274 %}
13275
13276 // This pattern is automatically generated from aarch64_ad.m4
13277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13278 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13279 %{
13280 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13281 ins_cost(1.9 * INSN_COST);
13282 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13283
13284 ins_encode %{
13285 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13286 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13287 %}
13288 ins_pipe(ialu_reg_reg_shift);
13289 %}
13290
13291 // This pattern is automatically generated from aarch64_ad.m4
13292 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13293 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13294 %{
13295 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13296 ins_cost(1.9 * INSN_COST);
13297 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13298
13299 ins_encode %{
13300 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13301 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13302 %}
13303 ins_pipe(ialu_reg_reg_shift);
13304 %}
13305
13306 // This pattern is automatically generated from aarch64_ad.m4
13307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13308 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13309 %{
13310 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13311 ins_cost(1.9 * INSN_COST);
13312 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13313
13314 ins_encode %{
13315 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13316 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13317 %}
13318 ins_pipe(ialu_reg_reg_shift);
13319 %}
13320
13321 // This pattern is automatically generated from aarch64_ad.m4
13322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13323 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13324 %{
13325 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13326 ins_cost(1.9 * INSN_COST);
13327 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13328
13329 ins_encode %{
13330 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13331 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13332 %}
13333 ins_pipe(ialu_reg_reg_shift);
13334 %}
13335
13336 // This pattern is automatically generated from aarch64_ad.m4
13337 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13338 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13339 %{
13340 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13341 ins_cost(1.9 * INSN_COST);
13342 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13343
13344 ins_encode %{
13345 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13346 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13347 %}
13348 ins_pipe(ialu_reg_reg_shift);
13349 %}
13350
13351 // This pattern is automatically generated from aarch64_ad.m4
13352 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13353 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13354 %{
13355 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13356 ins_cost(1.9 * INSN_COST);
13357 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13358
13359 ins_encode %{
13360 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13361 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13362 %}
13363 ins_pipe(ialu_reg_reg_shift);
13364 %}
13365
13366 // This pattern is automatically generated from aarch64_ad.m4
13367 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13368 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13369 %{
13370 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13371 ins_cost(1.9 * INSN_COST);
13372 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13373
13374 ins_encode %{
13375 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13376 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13377 %}
13378 ins_pipe(ialu_reg_reg_shift);
13379 %}
13380
13381 // This pattern is automatically generated from aarch64_ad.m4
13382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13383 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13384 %{
13385 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13386 ins_cost(1.9 * INSN_COST);
13387 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13388
13389 ins_encode %{
13390 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13391 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13392 %}
13393 ins_pipe(ialu_reg_reg_shift);
13394 %}
13395
13396 // This pattern is automatically generated from aarch64_ad.m4
13397 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13398 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13399 %{
13400 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13401 ins_cost(1.9 * INSN_COST);
13402 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13403
13404 ins_encode %{
13405 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13406 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13407 %}
13408 ins_pipe(ialu_reg_reg_shift);
13409 %}
13410
13411 // This pattern is automatically generated from aarch64_ad.m4
13412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13413 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13414 %{
13415 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13416 ins_cost(1.9 * INSN_COST);
13417 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13418
13419 ins_encode %{
13420 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13421 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13422 %}
13423 ins_pipe(ialu_reg_reg_shift);
13424 %}
13425
13426 // This pattern is automatically generated from aarch64_ad.m4
13427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13428 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13429 %{
13430 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13431 ins_cost(1.9 * INSN_COST);
13432 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13433
13434 ins_encode %{
13435 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13436 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13437 %}
13438 ins_pipe(ialu_reg_reg_shift);
13439 %}
13440
13441 // This pattern is automatically generated from aarch64_ad.m4
13442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13443 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13444 %{
13445 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13446 ins_cost(1.9 * INSN_COST);
13447 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13448
13449 ins_encode %{
13450 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13451 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13452 %}
13453 ins_pipe(ialu_reg_reg_shift);
13454 %}
13455
13456 // This pattern is automatically generated from aarch64_ad.m4
13457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13458 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13459 %{
13460 effect(DEF dst, USE src1, USE src2, USE cr);
13461 ins_cost(INSN_COST * 2);
13462 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13463
13464 ins_encode %{
13465 __ cselw($dst$$Register,
13466 $src1$$Register,
13467 $src2$$Register,
13468 Assembler::LT);
13469 %}
13470 ins_pipe(icond_reg_reg);
13471 %}
13472
13473 // This pattern is automatically generated from aarch64_ad.m4
13474 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13475 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13476 %{
13477 effect(DEF dst, USE src1, USE src2, USE cr);
13478 ins_cost(INSN_COST * 2);
13479 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13480
13481 ins_encode %{
13482 __ cselw($dst$$Register,
13483 $src1$$Register,
13484 $src2$$Register,
13485 Assembler::GT);
13486 %}
13487 ins_pipe(icond_reg_reg);
13488 %}
13489
13490 // This pattern is automatically generated from aarch64_ad.m4
13491 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13492 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13493 %{
13494 effect(DEF dst, USE src1, USE cr);
13495 ins_cost(INSN_COST * 2);
13496 format %{ "cselw $dst, $src1, zr lt\t" %}
13497
13498 ins_encode %{
13499 __ cselw($dst$$Register,
13500 $src1$$Register,
13501 zr,
13502 Assembler::LT);
13503 %}
13504 ins_pipe(icond_reg);
13505 %}
13506
13507 // This pattern is automatically generated from aarch64_ad.m4
13508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13509 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13510 %{
13511 effect(DEF dst, USE src1, USE cr);
13512 ins_cost(INSN_COST * 2);
13513 format %{ "cselw $dst, $src1, zr gt\t" %}
13514
13515 ins_encode %{
13516 __ cselw($dst$$Register,
13517 $src1$$Register,
13518 zr,
13519 Assembler::GT);
13520 %}
13521 ins_pipe(icond_reg);
13522 %}
13523
13524 // This pattern is automatically generated from aarch64_ad.m4
13525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13526 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13527 %{
13528 effect(DEF dst, USE src1, USE cr);
13529 ins_cost(INSN_COST * 2);
13530 format %{ "csincw $dst, $src1, zr le\t" %}
13531
13532 ins_encode %{
13533 __ csincw($dst$$Register,
13534 $src1$$Register,
13535 zr,
13536 Assembler::LE);
13537 %}
13538 ins_pipe(icond_reg);
13539 %}
13540
13541 // This pattern is automatically generated from aarch64_ad.m4
13542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13543 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13544 %{
13545 effect(DEF dst, USE src1, USE cr);
13546 ins_cost(INSN_COST * 2);
13547 format %{ "csincw $dst, $src1, zr gt\t" %}
13548
13549 ins_encode %{
13550 __ csincw($dst$$Register,
13551 $src1$$Register,
13552 zr,
13553 Assembler::GT);
13554 %}
13555 ins_pipe(icond_reg);
13556 %}
13557
13558 // This pattern is automatically generated from aarch64_ad.m4
13559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13560 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13561 %{
13562 effect(DEF dst, USE src1, USE cr);
13563 ins_cost(INSN_COST * 2);
13564 format %{ "csinvw $dst, $src1, zr lt\t" %}
13565
13566 ins_encode %{
13567 __ csinvw($dst$$Register,
13568 $src1$$Register,
13569 zr,
13570 Assembler::LT);
13571 %}
13572 ins_pipe(icond_reg);
13573 %}
13574
13575 // This pattern is automatically generated from aarch64_ad.m4
13576 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13577 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13578 %{
13579 effect(DEF dst, USE src1, USE cr);
13580 ins_cost(INSN_COST * 2);
13581 format %{ "csinvw $dst, $src1, zr ge\t" %}
13582
13583 ins_encode %{
13584 __ csinvw($dst$$Register,
13585 $src1$$Register,
13586 zr,
13587 Assembler::GE);
13588 %}
13589 ins_pipe(icond_reg);
13590 %}
13591
13592 // This pattern is automatically generated from aarch64_ad.m4
13593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13594 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13595 %{
13596 match(Set dst (MinI src imm));
13597 ins_cost(INSN_COST * 3);
13598 expand %{
13599 rFlagsReg cr;
13600 compI_reg_imm0(cr, src);
13601 cmovI_reg_imm0_lt(dst, src, cr);
13602 %}
13603 %}
13604
13605 // This pattern is automatically generated from aarch64_ad.m4
13606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13607 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13608 %{
13609 match(Set dst (MinI imm src));
13610 ins_cost(INSN_COST * 3);
13611 expand %{
13612 rFlagsReg cr;
13613 compI_reg_imm0(cr, src);
13614 cmovI_reg_imm0_lt(dst, src, cr);
13615 %}
13616 %}
13617
13618 // This pattern is automatically generated from aarch64_ad.m4
13619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13620 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13621 %{
13622 match(Set dst (MinI src imm));
13623 ins_cost(INSN_COST * 3);
13624 expand %{
13625 rFlagsReg cr;
13626 compI_reg_imm0(cr, src);
13627 cmovI_reg_imm1_le(dst, src, cr);
13628 %}
13629 %}
13630
13631 // This pattern is automatically generated from aarch64_ad.m4
13632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13633 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13634 %{
13635 match(Set dst (MinI imm src));
13636 ins_cost(INSN_COST * 3);
13637 expand %{
13638 rFlagsReg cr;
13639 compI_reg_imm0(cr, src);
13640 cmovI_reg_imm1_le(dst, src, cr);
13641 %}
13642 %}
13643
13644 // This pattern is automatically generated from aarch64_ad.m4
13645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13646 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13647 %{
13648 match(Set dst (MinI src imm));
13649 ins_cost(INSN_COST * 3);
13650 expand %{
13651 rFlagsReg cr;
13652 compI_reg_imm0(cr, src);
13653 cmovI_reg_immM1_lt(dst, src, cr);
13654 %}
13655 %}
13656
13657 // This pattern is automatically generated from aarch64_ad.m4
13658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13659 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13660 %{
13661 match(Set dst (MinI imm src));
13662 ins_cost(INSN_COST * 3);
13663 expand %{
13664 rFlagsReg cr;
13665 compI_reg_imm0(cr, src);
13666 cmovI_reg_immM1_lt(dst, src, cr);
13667 %}
13668 %}
13669
13670 // This pattern is automatically generated from aarch64_ad.m4
13671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13672 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13673 %{
13674 match(Set dst (MaxI src imm));
13675 ins_cost(INSN_COST * 3);
13676 expand %{
13677 rFlagsReg cr;
13678 compI_reg_imm0(cr, src);
13679 cmovI_reg_imm0_gt(dst, src, cr);
13680 %}
13681 %}
13682
13683 // This pattern is automatically generated from aarch64_ad.m4
13684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13685 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13686 %{
13687 match(Set dst (MaxI imm src));
13688 ins_cost(INSN_COST * 3);
13689 expand %{
13690 rFlagsReg cr;
13691 compI_reg_imm0(cr, src);
13692 cmovI_reg_imm0_gt(dst, src, cr);
13693 %}
13694 %}
13695
13696 // This pattern is automatically generated from aarch64_ad.m4
13697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13698 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13699 %{
13700 match(Set dst (MaxI src imm));
13701 ins_cost(INSN_COST * 3);
13702 expand %{
13703 rFlagsReg cr;
13704 compI_reg_imm0(cr, src);
13705 cmovI_reg_imm1_gt(dst, src, cr);
13706 %}
13707 %}
13708
13709 // This pattern is automatically generated from aarch64_ad.m4
13710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13711 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13712 %{
13713 match(Set dst (MaxI imm src));
13714 ins_cost(INSN_COST * 3);
13715 expand %{
13716 rFlagsReg cr;
13717 compI_reg_imm0(cr, src);
13718 cmovI_reg_imm1_gt(dst, src, cr);
13719 %}
13720 %}
13721
13722 // This pattern is automatically generated from aarch64_ad.m4
13723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13724 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13725 %{
13726 match(Set dst (MaxI src imm));
13727 ins_cost(INSN_COST * 3);
13728 expand %{
13729 rFlagsReg cr;
13730 compI_reg_imm0(cr, src);
13731 cmovI_reg_immM1_ge(dst, src, cr);
13732 %}
13733 %}
13734
13735 // This pattern is automatically generated from aarch64_ad.m4
13736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13737 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13738 %{
13739 match(Set dst (MaxI imm src));
13740 ins_cost(INSN_COST * 3);
13741 expand %{
13742 rFlagsReg cr;
13743 compI_reg_imm0(cr, src);
13744 cmovI_reg_immM1_ge(dst, src, cr);
13745 %}
13746 %}
13747
13748 // This pattern is automatically generated from aarch64_ad.m4
13749 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13750 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13751 %{
13752 match(Set dst (ReverseI src));
13753 ins_cost(INSN_COST);
13754 format %{ "rbitw $dst, $src" %}
13755 ins_encode %{
13756 __ rbitw($dst$$Register, $src$$Register);
13757 %}
13758 ins_pipe(ialu_reg);
13759 %}
13760
13761 // This pattern is automatically generated from aarch64_ad.m4
13762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13763 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13764 %{
13765 match(Set dst (ReverseL src));
13766 ins_cost(INSN_COST);
13767 format %{ "rbit $dst, $src" %}
13768 ins_encode %{
13769 __ rbit($dst$$Register, $src$$Register);
13770 %}
13771 ins_pipe(ialu_reg);
13772 %}
13773
13774
13775 // END This section of the file is automatically generated. Do not edit --------------
13776
13777
13778 // ============================================================================
13779 // Floating Point Arithmetic Instructions
13780
13781 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13782 match(Set dst (AddHF src1 src2));
13783 format %{ "faddh $dst, $src1, $src2" %}
13784 ins_encode %{
13785 __ faddh($dst$$FloatRegister,
13786 $src1$$FloatRegister,
13787 $src2$$FloatRegister);
13788 %}
13789 ins_pipe(fp_dop_reg_reg_s);
13790 %}
13791
13792 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13793 match(Set dst (AddF src1 src2));
13794
13795 ins_cost(INSN_COST * 5);
13796 format %{ "fadds $dst, $src1, $src2" %}
13797
13798 ins_encode %{
13799 __ fadds(as_FloatRegister($dst$$reg),
13800 as_FloatRegister($src1$$reg),
13801 as_FloatRegister($src2$$reg));
13802 %}
13803
13804 ins_pipe(fp_dop_reg_reg_s);
13805 %}
13806
13807 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13808 match(Set dst (AddD src1 src2));
13809
13810 ins_cost(INSN_COST * 5);
13811 format %{ "faddd $dst, $src1, $src2" %}
13812
13813 ins_encode %{
13814 __ faddd(as_FloatRegister($dst$$reg),
13815 as_FloatRegister($src1$$reg),
13816 as_FloatRegister($src2$$reg));
13817 %}
13818
13819 ins_pipe(fp_dop_reg_reg_d);
13820 %}
13821
13822 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13823 match(Set dst (SubHF src1 src2));
13824 format %{ "fsubh $dst, $src1, $src2" %}
13825 ins_encode %{
13826 __ fsubh($dst$$FloatRegister,
13827 $src1$$FloatRegister,
13828 $src2$$FloatRegister);
13829 %}
13830 ins_pipe(fp_dop_reg_reg_s);
13831 %}
13832
13833 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13834 match(Set dst (SubF src1 src2));
13835
13836 ins_cost(INSN_COST * 5);
13837 format %{ "fsubs $dst, $src1, $src2" %}
13838
13839 ins_encode %{
13840 __ fsubs(as_FloatRegister($dst$$reg),
13841 as_FloatRegister($src1$$reg),
13842 as_FloatRegister($src2$$reg));
13843 %}
13844
13845 ins_pipe(fp_dop_reg_reg_s);
13846 %}
13847
13848 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13849 match(Set dst (SubD src1 src2));
13850
13851 ins_cost(INSN_COST * 5);
13852 format %{ "fsubd $dst, $src1, $src2" %}
13853
13854 ins_encode %{
13855 __ fsubd(as_FloatRegister($dst$$reg),
13856 as_FloatRegister($src1$$reg),
13857 as_FloatRegister($src2$$reg));
13858 %}
13859
13860 ins_pipe(fp_dop_reg_reg_d);
13861 %}
13862
13863 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13864 match(Set dst (MulHF src1 src2));
13865 format %{ "fmulh $dst, $src1, $src2" %}
13866 ins_encode %{
13867 __ fmulh($dst$$FloatRegister,
13868 $src1$$FloatRegister,
13869 $src2$$FloatRegister);
13870 %}
13871 ins_pipe(fp_dop_reg_reg_s);
13872 %}
13873
13874 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13875 match(Set dst (MulF src1 src2));
13876
13877 ins_cost(INSN_COST * 6);
13878 format %{ "fmuls $dst, $src1, $src2" %}
13879
13880 ins_encode %{
13881 __ fmuls(as_FloatRegister($dst$$reg),
13882 as_FloatRegister($src1$$reg),
13883 as_FloatRegister($src2$$reg));
13884 %}
13885
13886 ins_pipe(fp_dop_reg_reg_s);
13887 %}
13888
13889 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13890 match(Set dst (MulD src1 src2));
13891
13892 ins_cost(INSN_COST * 6);
13893 format %{ "fmuld $dst, $src1, $src2" %}
13894
13895 ins_encode %{
13896 __ fmuld(as_FloatRegister($dst$$reg),
13897 as_FloatRegister($src1$$reg),
13898 as_FloatRegister($src2$$reg));
13899 %}
13900
13901 ins_pipe(fp_dop_reg_reg_d);
13902 %}
13903
13904 // src1 * src2 + src3 (half-precision float)
13905 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13906 match(Set dst (FmaHF src3 (Binary src1 src2)));
13907 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13908 ins_encode %{
13909 assert(UseFMA, "Needs FMA instructions support.");
13910 __ fmaddh($dst$$FloatRegister,
13911 $src1$$FloatRegister,
13912 $src2$$FloatRegister,
13913 $src3$$FloatRegister);
13914 %}
13915 ins_pipe(pipe_class_default);
13916 %}
13917
13918 // src1 * src2 + src3
13919 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13920 match(Set dst (FmaF src3 (Binary src1 src2)));
13921
13922 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13923
13924 ins_encode %{
13925 assert(UseFMA, "Needs FMA instructions support.");
13926 __ fmadds(as_FloatRegister($dst$$reg),
13927 as_FloatRegister($src1$$reg),
13928 as_FloatRegister($src2$$reg),
13929 as_FloatRegister($src3$$reg));
13930 %}
13931
13932 ins_pipe(pipe_class_default);
13933 %}
13934
13935 // src1 * src2 + src3
13936 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13937 match(Set dst (FmaD src3 (Binary src1 src2)));
13938
13939 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13940
13941 ins_encode %{
13942 assert(UseFMA, "Needs FMA instructions support.");
13943 __ fmaddd(as_FloatRegister($dst$$reg),
13944 as_FloatRegister($src1$$reg),
13945 as_FloatRegister($src2$$reg),
13946 as_FloatRegister($src3$$reg));
13947 %}
13948
13949 ins_pipe(pipe_class_default);
13950 %}
13951
13952 // src1 * (-src2) + src3
13953 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13954 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13955 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13956
13957 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13958
13959 ins_encode %{
13960 assert(UseFMA, "Needs FMA instructions support.");
13961 __ fmsubs(as_FloatRegister($dst$$reg),
13962 as_FloatRegister($src1$$reg),
13963 as_FloatRegister($src2$$reg),
13964 as_FloatRegister($src3$$reg));
13965 %}
13966
13967 ins_pipe(pipe_class_default);
13968 %}
13969
13970 // src1 * (-src2) + src3
13971 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13972 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13973 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13974
13975 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13976
13977 ins_encode %{
13978 assert(UseFMA, "Needs FMA instructions support.");
13979 __ fmsubd(as_FloatRegister($dst$$reg),
13980 as_FloatRegister($src1$$reg),
13981 as_FloatRegister($src2$$reg),
13982 as_FloatRegister($src3$$reg));
13983 %}
13984
13985 ins_pipe(pipe_class_default);
13986 %}
13987
13988 // src1 * (-src2) - src3
13989 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13990 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13991 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13992
13993 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13994
13995 ins_encode %{
13996 assert(UseFMA, "Needs FMA instructions support.");
13997 __ fnmadds(as_FloatRegister($dst$$reg),
13998 as_FloatRegister($src1$$reg),
13999 as_FloatRegister($src2$$reg),
14000 as_FloatRegister($src3$$reg));
14001 %}
14002
14003 ins_pipe(pipe_class_default);
14004 %}
14005
14006 // src1 * (-src2) - src3
14007 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14008 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14009 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14010
14011 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14012
14013 ins_encode %{
14014 assert(UseFMA, "Needs FMA instructions support.");
14015 __ fnmaddd(as_FloatRegister($dst$$reg),
14016 as_FloatRegister($src1$$reg),
14017 as_FloatRegister($src2$$reg),
14018 as_FloatRegister($src3$$reg));
14019 %}
14020
14021 ins_pipe(pipe_class_default);
14022 %}
14023
14024 // src1 * src2 - src3
14025 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14026 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14027
14028 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14029
14030 ins_encode %{
14031 assert(UseFMA, "Needs FMA instructions support.");
14032 __ fnmsubs(as_FloatRegister($dst$$reg),
14033 as_FloatRegister($src1$$reg),
14034 as_FloatRegister($src2$$reg),
14035 as_FloatRegister($src3$$reg));
14036 %}
14037
14038 ins_pipe(pipe_class_default);
14039 %}
14040
14041 // src1 * src2 - src3
14042 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14043 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14044
14045 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14046
14047 ins_encode %{
14048 assert(UseFMA, "Needs FMA instructions support.");
14049 // n.b. insn name should be fnmsubd
14050 __ fnmsub(as_FloatRegister($dst$$reg),
14051 as_FloatRegister($src1$$reg),
14052 as_FloatRegister($src2$$reg),
14053 as_FloatRegister($src3$$reg));
14054 %}
14055
14056 ins_pipe(pipe_class_default);
14057 %}
14058
14059 // Math.max(HH)H (half-precision float)
14060 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14061 match(Set dst (MaxHF src1 src2));
14062 format %{ "fmaxh $dst, $src1, $src2" %}
14063 ins_encode %{
14064 __ fmaxh($dst$$FloatRegister,
14065 $src1$$FloatRegister,
14066 $src2$$FloatRegister);
14067 %}
14068 ins_pipe(fp_dop_reg_reg_s);
14069 %}
14070
14071 // Math.min(HH)H (half-precision float)
14072 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14073 match(Set dst (MinHF src1 src2));
14074 format %{ "fminh $dst, $src1, $src2" %}
14075 ins_encode %{
14076 __ fminh($dst$$FloatRegister,
14077 $src1$$FloatRegister,
14078 $src2$$FloatRegister);
14079 %}
14080 ins_pipe(fp_dop_reg_reg_s);
14081 %}
14082
14083 // Math.max(FF)F
14084 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14085 match(Set dst (MaxF src1 src2));
14086
14087 format %{ "fmaxs $dst, $src1, $src2" %}
14088 ins_encode %{
14089 __ fmaxs(as_FloatRegister($dst$$reg),
14090 as_FloatRegister($src1$$reg),
14091 as_FloatRegister($src2$$reg));
14092 %}
14093
14094 ins_pipe(fp_dop_reg_reg_s);
14095 %}
14096
14097 // Math.min(FF)F
14098 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14099 match(Set dst (MinF src1 src2));
14100
14101 format %{ "fmins $dst, $src1, $src2" %}
14102 ins_encode %{
14103 __ fmins(as_FloatRegister($dst$$reg),
14104 as_FloatRegister($src1$$reg),
14105 as_FloatRegister($src2$$reg));
14106 %}
14107
14108 ins_pipe(fp_dop_reg_reg_s);
14109 %}
14110
14111 // Math.max(DD)D
14112 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14113 match(Set dst (MaxD src1 src2));
14114
14115 format %{ "fmaxd $dst, $src1, $src2" %}
14116 ins_encode %{
14117 __ fmaxd(as_FloatRegister($dst$$reg),
14118 as_FloatRegister($src1$$reg),
14119 as_FloatRegister($src2$$reg));
14120 %}
14121
14122 ins_pipe(fp_dop_reg_reg_d);
14123 %}
14124
14125 // Math.min(DD)D
14126 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14127 match(Set dst (MinD src1 src2));
14128
14129 format %{ "fmind $dst, $src1, $src2" %}
14130 ins_encode %{
14131 __ fmind(as_FloatRegister($dst$$reg),
14132 as_FloatRegister($src1$$reg),
14133 as_FloatRegister($src2$$reg));
14134 %}
14135
14136 ins_pipe(fp_dop_reg_reg_d);
14137 %}
14138
14139 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14140 match(Set dst (DivHF src1 src2));
14141 format %{ "fdivh $dst, $src1, $src2" %}
14142 ins_encode %{
14143 __ fdivh($dst$$FloatRegister,
14144 $src1$$FloatRegister,
14145 $src2$$FloatRegister);
14146 %}
14147 ins_pipe(fp_div_s);
14148 %}
14149
14150 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14151 match(Set dst (DivF src1 src2));
14152
14153 ins_cost(INSN_COST * 18);
14154 format %{ "fdivs $dst, $src1, $src2" %}
14155
14156 ins_encode %{
14157 __ fdivs(as_FloatRegister($dst$$reg),
14158 as_FloatRegister($src1$$reg),
14159 as_FloatRegister($src2$$reg));
14160 %}
14161
14162 ins_pipe(fp_div_s);
14163 %}
14164
14165 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14166 match(Set dst (DivD src1 src2));
14167
14168 ins_cost(INSN_COST * 32);
14169 format %{ "fdivd $dst, $src1, $src2" %}
14170
14171 ins_encode %{
14172 __ fdivd(as_FloatRegister($dst$$reg),
14173 as_FloatRegister($src1$$reg),
14174 as_FloatRegister($src2$$reg));
14175 %}
14176
14177 ins_pipe(fp_div_d);
14178 %}
14179
14180 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14181 match(Set dst (NegF src));
14182
14183 ins_cost(INSN_COST * 3);
14184 format %{ "fneg $dst, $src" %}
14185
14186 ins_encode %{
14187 __ fnegs(as_FloatRegister($dst$$reg),
14188 as_FloatRegister($src$$reg));
14189 %}
14190
14191 ins_pipe(fp_uop_s);
14192 %}
14193
14194 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14195 match(Set dst (NegD src));
14196
14197 ins_cost(INSN_COST * 3);
14198 format %{ "fnegd $dst, $src" %}
14199
14200 ins_encode %{
14201 __ fnegd(as_FloatRegister($dst$$reg),
14202 as_FloatRegister($src$$reg));
14203 %}
14204
14205 ins_pipe(fp_uop_d);
14206 %}
14207
14208 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14209 %{
14210 match(Set dst (AbsI src));
14211
14212 effect(KILL cr);
14213 ins_cost(INSN_COST * 2);
14214 format %{ "cmpw $src, zr\n\t"
14215 "cnegw $dst, $src, Assembler::LT\t# int abs"
14216 %}
14217
14218 ins_encode %{
14219 __ cmpw(as_Register($src$$reg), zr);
14220 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14221 %}
14222 ins_pipe(pipe_class_default);
14223 %}
14224
14225 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14226 %{
14227 match(Set dst (AbsL src));
14228
14229 effect(KILL cr);
14230 ins_cost(INSN_COST * 2);
14231 format %{ "cmp $src, zr\n\t"
14232 "cneg $dst, $src, Assembler::LT\t# long abs"
14233 %}
14234
14235 ins_encode %{
14236 __ cmp(as_Register($src$$reg), zr);
14237 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14238 %}
14239 ins_pipe(pipe_class_default);
14240 %}
14241
14242 instruct absF_reg(vRegF dst, vRegF src) %{
14243 match(Set dst (AbsF src));
14244
14245 ins_cost(INSN_COST * 3);
14246 format %{ "fabss $dst, $src" %}
14247 ins_encode %{
14248 __ fabss(as_FloatRegister($dst$$reg),
14249 as_FloatRegister($src$$reg));
14250 %}
14251
14252 ins_pipe(fp_uop_s);
14253 %}
14254
14255 instruct absD_reg(vRegD dst, vRegD src) %{
14256 match(Set dst (AbsD src));
14257
14258 ins_cost(INSN_COST * 3);
14259 format %{ "fabsd $dst, $src" %}
14260 ins_encode %{
14261 __ fabsd(as_FloatRegister($dst$$reg),
14262 as_FloatRegister($src$$reg));
14263 %}
14264
14265 ins_pipe(fp_uop_d);
14266 %}
14267
14268 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14269 match(Set dst (AbsF (SubF src1 src2)));
14270
14271 ins_cost(INSN_COST * 3);
14272 format %{ "fabds $dst, $src1, $src2" %}
14273 ins_encode %{
14274 __ fabds(as_FloatRegister($dst$$reg),
14275 as_FloatRegister($src1$$reg),
14276 as_FloatRegister($src2$$reg));
14277 %}
14278
14279 ins_pipe(fp_uop_s);
14280 %}
14281
14282 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14283 match(Set dst (AbsD (SubD src1 src2)));
14284
14285 ins_cost(INSN_COST * 3);
14286 format %{ "fabdd $dst, $src1, $src2" %}
14287 ins_encode %{
14288 __ fabdd(as_FloatRegister($dst$$reg),
14289 as_FloatRegister($src1$$reg),
14290 as_FloatRegister($src2$$reg));
14291 %}
14292
14293 ins_pipe(fp_uop_d);
14294 %}
14295
14296 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14297 match(Set dst (SqrtD src));
14298
14299 ins_cost(INSN_COST * 50);
14300 format %{ "fsqrtd $dst, $src" %}
14301 ins_encode %{
14302 __ fsqrtd(as_FloatRegister($dst$$reg),
14303 as_FloatRegister($src$$reg));
14304 %}
14305
14306 ins_pipe(fp_div_s);
14307 %}
14308
14309 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14310 match(Set dst (SqrtF src));
14311
14312 ins_cost(INSN_COST * 50);
14313 format %{ "fsqrts $dst, $src" %}
14314 ins_encode %{
14315 __ fsqrts(as_FloatRegister($dst$$reg),
14316 as_FloatRegister($src$$reg));
14317 %}
14318
14319 ins_pipe(fp_div_d);
14320 %}
14321
14322 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14323 match(Set dst (SqrtHF src));
14324 format %{ "fsqrth $dst, $src" %}
14325 ins_encode %{
14326 __ fsqrth($dst$$FloatRegister,
14327 $src$$FloatRegister);
14328 %}
14329 ins_pipe(fp_div_s);
14330 %}
14331
14332 // Math.rint, floor, ceil
14333 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14334 match(Set dst (RoundDoubleMode src rmode));
14335 format %{ "frint $dst, $src, $rmode" %}
14336 ins_encode %{
14337 switch ($rmode$$constant) {
14338 case RoundDoubleModeNode::rmode_rint:
14339 __ frintnd(as_FloatRegister($dst$$reg),
14340 as_FloatRegister($src$$reg));
14341 break;
14342 case RoundDoubleModeNode::rmode_floor:
14343 __ frintmd(as_FloatRegister($dst$$reg),
14344 as_FloatRegister($src$$reg));
14345 break;
14346 case RoundDoubleModeNode::rmode_ceil:
14347 __ frintpd(as_FloatRegister($dst$$reg),
14348 as_FloatRegister($src$$reg));
14349 break;
14350 }
14351 %}
14352 ins_pipe(fp_uop_d);
14353 %}
14354
14355 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14356 match(Set dst (CopySignD src1 (Binary src2 zero)));
14357 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14358 format %{ "CopySignD $dst $src1 $src2" %}
14359 ins_encode %{
14360 FloatRegister dst = as_FloatRegister($dst$$reg),
14361 src1 = as_FloatRegister($src1$$reg),
14362 src2 = as_FloatRegister($src2$$reg),
14363 zero = as_FloatRegister($zero$$reg);
14364 __ fnegd(dst, zero);
14365 __ bsl(dst, __ T8B, src2, src1);
14366 %}
14367 ins_pipe(fp_uop_d);
14368 %}
14369
14370 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14371 match(Set dst (CopySignF src1 src2));
14372 effect(TEMP_DEF dst, USE src1, USE src2);
14373 format %{ "CopySignF $dst $src1 $src2" %}
14374 ins_encode %{
14375 FloatRegister dst = as_FloatRegister($dst$$reg),
14376 src1 = as_FloatRegister($src1$$reg),
14377 src2 = as_FloatRegister($src2$$reg);
14378 __ movi(dst, __ T2S, 0x80, 24);
14379 __ bsl(dst, __ T8B, src2, src1);
14380 %}
14381 ins_pipe(fp_uop_d);
14382 %}
14383
14384 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14385 match(Set dst (SignumD src (Binary zero one)));
14386 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14387 format %{ "signumD $dst, $src" %}
14388 ins_encode %{
14389 FloatRegister src = as_FloatRegister($src$$reg),
14390 dst = as_FloatRegister($dst$$reg),
14391 zero = as_FloatRegister($zero$$reg),
14392 one = as_FloatRegister($one$$reg);
14393 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14394 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14395 // Bit selection instruction gets bit from "one" for each enabled bit in
14396 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14397 // NaN the whole "src" will be copied because "dst" is zero. For all other
14398 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14399 // from "src", and all other bits are copied from 1.0.
14400 __ bsl(dst, __ T8B, one, src);
14401 %}
14402 ins_pipe(fp_uop_d);
14403 %}
14404
14405 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14406 match(Set dst (SignumF src (Binary zero one)));
14407 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14408 format %{ "signumF $dst, $src" %}
14409 ins_encode %{
14410 FloatRegister src = as_FloatRegister($src$$reg),
14411 dst = as_FloatRegister($dst$$reg),
14412 zero = as_FloatRegister($zero$$reg),
14413 one = as_FloatRegister($one$$reg);
14414 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14415 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14416 // Bit selection instruction gets bit from "one" for each enabled bit in
14417 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14418 // NaN the whole "src" will be copied because "dst" is zero. For all other
14419 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14420 // from "src", and all other bits are copied from 1.0.
14421 __ bsl(dst, __ T8B, one, src);
14422 %}
14423 ins_pipe(fp_uop_d);
14424 %}
14425
14426 instruct onspinwait() %{
14427 match(OnSpinWait);
14428 ins_cost(INSN_COST);
14429
14430 format %{ "onspinwait" %}
14431
14432 ins_encode %{
14433 __ spin_wait();
14434 %}
14435 ins_pipe(pipe_class_empty);
14436 %}
14437
14438 // ============================================================================
14439 // Logical Instructions
14440
14441 // Integer Logical Instructions
14442
14443 // And Instructions
14444
14445
14446 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14447 match(Set dst (AndI src1 src2));
14448
14449 format %{ "andw $dst, $src1, $src2\t# int" %}
14450
14451 ins_cost(INSN_COST);
14452 ins_encode %{
14453 __ andw(as_Register($dst$$reg),
14454 as_Register($src1$$reg),
14455 as_Register($src2$$reg));
14456 %}
14457
14458 ins_pipe(ialu_reg_reg);
14459 %}
14460
14461 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14462 match(Set dst (AndI src1 src2));
14463
14464 format %{ "andsw $dst, $src1, $src2\t# int" %}
14465
14466 ins_cost(INSN_COST);
14467 ins_encode %{
14468 __ andw(as_Register($dst$$reg),
14469 as_Register($src1$$reg),
14470 (uint64_t)($src2$$constant));
14471 %}
14472
14473 ins_pipe(ialu_reg_imm);
14474 %}
14475
14476 // Or Instructions
14477
14478 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14479 match(Set dst (OrI src1 src2));
14480
14481 format %{ "orrw $dst, $src1, $src2\t# int" %}
14482
14483 ins_cost(INSN_COST);
14484 ins_encode %{
14485 __ orrw(as_Register($dst$$reg),
14486 as_Register($src1$$reg),
14487 as_Register($src2$$reg));
14488 %}
14489
14490 ins_pipe(ialu_reg_reg);
14491 %}
14492
14493 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14494 match(Set dst (OrI src1 src2));
14495
14496 format %{ "orrw $dst, $src1, $src2\t# int" %}
14497
14498 ins_cost(INSN_COST);
14499 ins_encode %{
14500 __ orrw(as_Register($dst$$reg),
14501 as_Register($src1$$reg),
14502 (uint64_t)($src2$$constant));
14503 %}
14504
14505 ins_pipe(ialu_reg_imm);
14506 %}
14507
14508 // Xor Instructions
14509
14510 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14511 match(Set dst (XorI src1 src2));
14512
14513 format %{ "eorw $dst, $src1, $src2\t# int" %}
14514
14515 ins_cost(INSN_COST);
14516 ins_encode %{
14517 __ eorw(as_Register($dst$$reg),
14518 as_Register($src1$$reg),
14519 as_Register($src2$$reg));
14520 %}
14521
14522 ins_pipe(ialu_reg_reg);
14523 %}
14524
14525 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14526 match(Set dst (XorI src1 src2));
14527
14528 format %{ "eorw $dst, $src1, $src2\t# int" %}
14529
14530 ins_cost(INSN_COST);
14531 ins_encode %{
14532 __ eorw(as_Register($dst$$reg),
14533 as_Register($src1$$reg),
14534 (uint64_t)($src2$$constant));
14535 %}
14536
14537 ins_pipe(ialu_reg_imm);
14538 %}
14539
14540 // Long Logical Instructions
14541 // TODO
14542
14543 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14544 match(Set dst (AndL src1 src2));
14545
14546 format %{ "and $dst, $src1, $src2\t# int" %}
14547
14548 ins_cost(INSN_COST);
14549 ins_encode %{
14550 __ andr(as_Register($dst$$reg),
14551 as_Register($src1$$reg),
14552 as_Register($src2$$reg));
14553 %}
14554
14555 ins_pipe(ialu_reg_reg);
14556 %}
14557
14558 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14559 match(Set dst (AndL src1 src2));
14560
14561 format %{ "and $dst, $src1, $src2\t# int" %}
14562
14563 ins_cost(INSN_COST);
14564 ins_encode %{
14565 __ andr(as_Register($dst$$reg),
14566 as_Register($src1$$reg),
14567 (uint64_t)($src2$$constant));
14568 %}
14569
14570 ins_pipe(ialu_reg_imm);
14571 %}
14572
14573 // Or Instructions
14574
14575 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14576 match(Set dst (OrL src1 src2));
14577
14578 format %{ "orr $dst, $src1, $src2\t# int" %}
14579
14580 ins_cost(INSN_COST);
14581 ins_encode %{
14582 __ orr(as_Register($dst$$reg),
14583 as_Register($src1$$reg),
14584 as_Register($src2$$reg));
14585 %}
14586
14587 ins_pipe(ialu_reg_reg);
14588 %}
14589
14590 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14591 match(Set dst (OrL src1 src2));
14592
14593 format %{ "orr $dst, $src1, $src2\t# int" %}
14594
14595 ins_cost(INSN_COST);
14596 ins_encode %{
14597 __ orr(as_Register($dst$$reg),
14598 as_Register($src1$$reg),
14599 (uint64_t)($src2$$constant));
14600 %}
14601
14602 ins_pipe(ialu_reg_imm);
14603 %}
14604
14605 // Xor Instructions
14606
14607 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14608 match(Set dst (XorL src1 src2));
14609
14610 format %{ "eor $dst, $src1, $src2\t# int" %}
14611
14612 ins_cost(INSN_COST);
14613 ins_encode %{
14614 __ eor(as_Register($dst$$reg),
14615 as_Register($src1$$reg),
14616 as_Register($src2$$reg));
14617 %}
14618
14619 ins_pipe(ialu_reg_reg);
14620 %}
14621
14622 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14623 match(Set dst (XorL src1 src2));
14624
14625 ins_cost(INSN_COST);
14626 format %{ "eor $dst, $src1, $src2\t# int" %}
14627
14628 ins_encode %{
14629 __ eor(as_Register($dst$$reg),
14630 as_Register($src1$$reg),
14631 (uint64_t)($src2$$constant));
14632 %}
14633
14634 ins_pipe(ialu_reg_imm);
14635 %}
14636
14637 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14638 %{
14639 match(Set dst (ConvI2L src));
14640
14641 ins_cost(INSN_COST);
14642 format %{ "sxtw $dst, $src\t# i2l" %}
14643 ins_encode %{
14644 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14645 %}
14646 ins_pipe(ialu_reg_shift);
14647 %}
14648
14649 // this pattern occurs in bigmath arithmetic
14650 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14651 %{
14652 match(Set dst (AndL (ConvI2L src) mask));
14653
14654 ins_cost(INSN_COST);
14655 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14656 ins_encode %{
14657 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14658 %}
14659
14660 ins_pipe(ialu_reg_shift);
14661 %}
14662
14663 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14664 match(Set dst (ConvL2I src));
14665
14666 ins_cost(INSN_COST);
14667 format %{ "movw $dst, $src \t// l2i" %}
14668
14669 ins_encode %{
14670 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14671 %}
14672
14673 ins_pipe(ialu_reg);
14674 %}
14675
14676 instruct convD2F_reg(vRegF dst, vRegD src) %{
14677 match(Set dst (ConvD2F src));
14678
14679 ins_cost(INSN_COST * 5);
14680 format %{ "fcvtd $dst, $src \t// d2f" %}
14681
14682 ins_encode %{
14683 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14684 %}
14685
14686 ins_pipe(fp_d2f);
14687 %}
14688
14689 instruct convF2D_reg(vRegD dst, vRegF src) %{
14690 match(Set dst (ConvF2D src));
14691
14692 ins_cost(INSN_COST * 5);
14693 format %{ "fcvts $dst, $src \t// f2d" %}
14694
14695 ins_encode %{
14696 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14697 %}
14698
14699 ins_pipe(fp_f2d);
14700 %}
14701
14702 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14703 match(Set dst (ConvF2I src));
14704
14705 ins_cost(INSN_COST * 5);
14706 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14707
14708 ins_encode %{
14709 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14710 %}
14711
14712 ins_pipe(fp_f2i);
14713 %}
14714
14715 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14716 match(Set dst (ConvF2L src));
14717
14718 ins_cost(INSN_COST * 5);
14719 format %{ "fcvtzs $dst, $src \t// f2l" %}
14720
14721 ins_encode %{
14722 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14723 %}
14724
14725 ins_pipe(fp_f2l);
14726 %}
14727
14728 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14729 match(Set dst (ConvF2HF src));
14730 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14731 "smov $dst, $tmp\t# move result from $tmp to $dst"
14732 %}
14733 effect(TEMP tmp);
14734 ins_encode %{
14735 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14736 %}
14737 ins_pipe(pipe_slow);
14738 %}
14739
14740 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14741 match(Set dst (ConvHF2F src));
14742 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14743 "fcvt $dst, $tmp\t# convert half to single precision"
14744 %}
14745 effect(TEMP tmp);
14746 ins_encode %{
14747 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14748 %}
14749 ins_pipe(pipe_slow);
14750 %}
14751
14752 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14753 match(Set dst (ConvI2F src));
14754
14755 ins_cost(INSN_COST * 5);
14756 format %{ "scvtfws $dst, $src \t// i2f" %}
14757
14758 ins_encode %{
14759 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14760 %}
14761
14762 ins_pipe(fp_i2f);
14763 %}
14764
14765 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14766 match(Set dst (ConvL2F src));
14767
14768 ins_cost(INSN_COST * 5);
14769 format %{ "scvtfs $dst, $src \t// l2f" %}
14770
14771 ins_encode %{
14772 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14773 %}
14774
14775 ins_pipe(fp_l2f);
14776 %}
14777
14778 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14779 match(Set dst (ConvD2I src));
14780
14781 ins_cost(INSN_COST * 5);
14782 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14783
14784 ins_encode %{
14785 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14786 %}
14787
14788 ins_pipe(fp_d2i);
14789 %}
14790
14791 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14792 match(Set dst (ConvD2L src));
14793
14794 ins_cost(INSN_COST * 5);
14795 format %{ "fcvtzd $dst, $src \t// d2l" %}
14796
14797 ins_encode %{
14798 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14799 %}
14800
14801 ins_pipe(fp_d2l);
14802 %}
14803
14804 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14805 match(Set dst (ConvI2D src));
14806
14807 ins_cost(INSN_COST * 5);
14808 format %{ "scvtfwd $dst, $src \t// i2d" %}
14809
14810 ins_encode %{
14811 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14812 %}
14813
14814 ins_pipe(fp_i2d);
14815 %}
14816
14817 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14818 match(Set dst (ConvL2D src));
14819
14820 ins_cost(INSN_COST * 5);
14821 format %{ "scvtfd $dst, $src \t// l2d" %}
14822
14823 ins_encode %{
14824 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14825 %}
14826
14827 ins_pipe(fp_l2d);
14828 %}
14829
14830 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14831 %{
14832 match(Set dst (RoundD src));
14833 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14834 format %{ "java_round_double $dst,$src"%}
14835 ins_encode %{
14836 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14837 as_FloatRegister($ftmp$$reg));
14838 %}
14839 ins_pipe(pipe_slow);
14840 %}
14841
14842 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14843 %{
14844 match(Set dst (RoundF src));
14845 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14846 format %{ "java_round_float $dst,$src"%}
14847 ins_encode %{
14848 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14849 as_FloatRegister($ftmp$$reg));
14850 %}
14851 ins_pipe(pipe_slow);
14852 %}
14853
14854 // stack <-> reg and reg <-> reg shuffles with no conversion
14855
14856 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14857
14858 match(Set dst (MoveF2I src));
14859
14860 effect(DEF dst, USE src);
14861
14862 ins_cost(4 * INSN_COST);
14863
14864 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14865
14866 ins_encode %{
14867 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14868 %}
14869
14870 ins_pipe(iload_reg_reg);
14871
14872 %}
14873
14874 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14875
14876 match(Set dst (MoveI2F src));
14877
14878 effect(DEF dst, USE src);
14879
14880 ins_cost(4 * INSN_COST);
14881
14882 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14883
14884 ins_encode %{
14885 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14886 %}
14887
14888 ins_pipe(pipe_class_memory);
14889
14890 %}
14891
14892 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14893
14894 match(Set dst (MoveD2L src));
14895
14896 effect(DEF dst, USE src);
14897
14898 ins_cost(4 * INSN_COST);
14899
14900 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14901
14902 ins_encode %{
14903 __ ldr($dst$$Register, Address(sp, $src$$disp));
14904 %}
14905
14906 ins_pipe(iload_reg_reg);
14907
14908 %}
14909
14910 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14911
14912 match(Set dst (MoveL2D src));
14913
14914 effect(DEF dst, USE src);
14915
14916 ins_cost(4 * INSN_COST);
14917
14918 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14919
14920 ins_encode %{
14921 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14922 %}
14923
14924 ins_pipe(pipe_class_memory);
14925
14926 %}
14927
14928 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14929
14930 match(Set dst (MoveF2I src));
14931
14932 effect(DEF dst, USE src);
14933
14934 ins_cost(INSN_COST);
14935
14936 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14937
14938 ins_encode %{
14939 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14940 %}
14941
14942 ins_pipe(pipe_class_memory);
14943
14944 %}
14945
14946 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14947
14948 match(Set dst (MoveI2F src));
14949
14950 effect(DEF dst, USE src);
14951
14952 ins_cost(INSN_COST);
14953
14954 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14955
14956 ins_encode %{
14957 __ strw($src$$Register, Address(sp, $dst$$disp));
14958 %}
14959
14960 ins_pipe(istore_reg_reg);
14961
14962 %}
14963
14964 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14965
14966 match(Set dst (MoveD2L src));
14967
14968 effect(DEF dst, USE src);
14969
14970 ins_cost(INSN_COST);
14971
14972 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14973
14974 ins_encode %{
14975 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14976 %}
14977
14978 ins_pipe(pipe_class_memory);
14979
14980 %}
14981
14982 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14983
14984 match(Set dst (MoveL2D src));
14985
14986 effect(DEF dst, USE src);
14987
14988 ins_cost(INSN_COST);
14989
14990 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14991
14992 ins_encode %{
14993 __ str($src$$Register, Address(sp, $dst$$disp));
14994 %}
14995
14996 ins_pipe(istore_reg_reg);
14997
14998 %}
14999
15000 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15001
15002 match(Set dst (MoveF2I src));
15003
15004 effect(DEF dst, USE src);
15005
15006 ins_cost(INSN_COST);
15007
15008 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15009
15010 ins_encode %{
15011 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15012 %}
15013
15014 ins_pipe(fp_f2i);
15015
15016 %}
15017
15018 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15019
15020 match(Set dst (MoveI2F src));
15021
15022 effect(DEF dst, USE src);
15023
15024 ins_cost(INSN_COST);
15025
15026 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15027
15028 ins_encode %{
15029 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15030 %}
15031
15032 ins_pipe(fp_i2f);
15033
15034 %}
15035
15036 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15037
15038 match(Set dst (MoveD2L src));
15039
15040 effect(DEF dst, USE src);
15041
15042 ins_cost(INSN_COST);
15043
15044 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15045
15046 ins_encode %{
15047 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15048 %}
15049
15050 ins_pipe(fp_d2l);
15051
15052 %}
15053
15054 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15055
15056 match(Set dst (MoveL2D src));
15057
15058 effect(DEF dst, USE src);
15059
15060 ins_cost(INSN_COST);
15061
15062 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15063
15064 ins_encode %{
15065 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15066 %}
15067
15068 ins_pipe(fp_l2d);
15069
15070 %}
15071
15072 // ============================================================================
15073 // clearing of an array
15074
15075 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15076 %{
15077 match(Set dummy (ClearArray cnt base));
15078 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15079
15080 ins_cost(4 * INSN_COST);
15081 format %{ "ClearArray $cnt, $base" %}
15082
15083 ins_encode %{
15084 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15085 if (tpc == nullptr) {
15086 ciEnv::current()->record_failure("CodeCache is full");
15087 return;
15088 }
15089 %}
15090
15091 ins_pipe(pipe_class_memory);
15092 %}
15093
15094 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15095 %{
15096 predicate((uint64_t)n->in(2)->get_long()
15097 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15098 match(Set dummy (ClearArray cnt base));
15099 effect(TEMP temp, USE_KILL base, KILL cr);
15100
15101 ins_cost(4 * INSN_COST);
15102 format %{ "ClearArray $cnt, $base" %}
15103
15104 ins_encode %{
15105 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15106 if (tpc == nullptr) {
15107 ciEnv::current()->record_failure("CodeCache is full");
15108 return;
15109 }
15110 %}
15111
15112 ins_pipe(pipe_class_memory);
15113 %}
15114
15115 // ============================================================================
15116 // Overflow Math Instructions
15117
15118 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15119 %{
15120 match(Set cr (OverflowAddI op1 op2));
15121
15122 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15123 ins_cost(INSN_COST);
15124 ins_encode %{
15125 __ cmnw($op1$$Register, $op2$$Register);
15126 %}
15127
15128 ins_pipe(icmp_reg_reg);
15129 %}
15130
15131 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15132 %{
15133 match(Set cr (OverflowAddI op1 op2));
15134
15135 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15136 ins_cost(INSN_COST);
15137 ins_encode %{
15138 __ cmnw($op1$$Register, $op2$$constant);
15139 %}
15140
15141 ins_pipe(icmp_reg_imm);
15142 %}
15143
15144 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15145 %{
15146 match(Set cr (OverflowAddL op1 op2));
15147
15148 format %{ "cmn $op1, $op2\t# overflow check long" %}
15149 ins_cost(INSN_COST);
15150 ins_encode %{
15151 __ cmn($op1$$Register, $op2$$Register);
15152 %}
15153
15154 ins_pipe(icmp_reg_reg);
15155 %}
15156
15157 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15158 %{
15159 match(Set cr (OverflowAddL op1 op2));
15160
15161 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15162 ins_cost(INSN_COST);
15163 ins_encode %{
15164 __ adds(zr, $op1$$Register, $op2$$constant);
15165 %}
15166
15167 ins_pipe(icmp_reg_imm);
15168 %}
15169
15170 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15171 %{
15172 match(Set cr (OverflowSubI op1 op2));
15173
15174 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15175 ins_cost(INSN_COST);
15176 ins_encode %{
15177 __ cmpw($op1$$Register, $op2$$Register);
15178 %}
15179
15180 ins_pipe(icmp_reg_reg);
15181 %}
15182
15183 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15184 %{
15185 match(Set cr (OverflowSubI op1 op2));
15186
15187 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15188 ins_cost(INSN_COST);
15189 ins_encode %{
15190 __ cmpw($op1$$Register, $op2$$constant);
15191 %}
15192
15193 ins_pipe(icmp_reg_imm);
15194 %}
15195
15196 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15197 %{
15198 match(Set cr (OverflowSubL op1 op2));
15199
15200 format %{ "cmp $op1, $op2\t# overflow check long" %}
15201 ins_cost(INSN_COST);
15202 ins_encode %{
15203 __ cmp($op1$$Register, $op2$$Register);
15204 %}
15205
15206 ins_pipe(icmp_reg_reg);
15207 %}
15208
15209 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15210 %{
15211 match(Set cr (OverflowSubL op1 op2));
15212
15213 format %{ "cmp $op1, $op2\t# overflow check long" %}
15214 ins_cost(INSN_COST);
15215 ins_encode %{
15216 __ subs(zr, $op1$$Register, $op2$$constant);
15217 %}
15218
15219 ins_pipe(icmp_reg_imm);
15220 %}
15221
15222 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15223 %{
15224 match(Set cr (OverflowSubI zero op1));
15225
15226 format %{ "cmpw zr, $op1\t# overflow check int" %}
15227 ins_cost(INSN_COST);
15228 ins_encode %{
15229 __ cmpw(zr, $op1$$Register);
15230 %}
15231
15232 ins_pipe(icmp_reg_imm);
15233 %}
15234
15235 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15236 %{
15237 match(Set cr (OverflowSubL zero op1));
15238
15239 format %{ "cmp zr, $op1\t# overflow check long" %}
15240 ins_cost(INSN_COST);
15241 ins_encode %{
15242 __ cmp(zr, $op1$$Register);
15243 %}
15244
15245 ins_pipe(icmp_reg_imm);
15246 %}
15247
15248 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15249 %{
15250 match(Set cr (OverflowMulI op1 op2));
15251
15252 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15253 "cmp rscratch1, rscratch1, sxtw\n\t"
15254 "movw rscratch1, #0x80000000\n\t"
15255 "cselw rscratch1, rscratch1, zr, NE\n\t"
15256 "cmpw rscratch1, #1" %}
15257 ins_cost(5 * INSN_COST);
15258 ins_encode %{
15259 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15260 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15261 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15262 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15263 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15264 %}
15265
15266 ins_pipe(pipe_slow);
15267 %}
15268
15269 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15270 %{
15271 match(If cmp (OverflowMulI op1 op2));
15272 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15273 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15274 effect(USE labl, KILL cr);
15275
15276 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15277 "cmp rscratch1, rscratch1, sxtw\n\t"
15278 "b$cmp $labl" %}
15279 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15280 ins_encode %{
15281 Label* L = $labl$$label;
15282 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15283 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15284 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15285 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15286 %}
15287
15288 ins_pipe(pipe_serial);
15289 %}
15290
15291 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15292 %{
15293 match(Set cr (OverflowMulL op1 op2));
15294
15295 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15296 "smulh rscratch2, $op1, $op2\n\t"
15297 "cmp rscratch2, rscratch1, ASR #63\n\t"
15298 "movw rscratch1, #0x80000000\n\t"
15299 "cselw rscratch1, rscratch1, zr, NE\n\t"
15300 "cmpw rscratch1, #1" %}
15301 ins_cost(6 * INSN_COST);
15302 ins_encode %{
15303 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15304 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15305 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15306 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15307 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15308 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15309 %}
15310
15311 ins_pipe(pipe_slow);
15312 %}
15313
15314 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15315 %{
15316 match(If cmp (OverflowMulL op1 op2));
15317 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15318 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15319 effect(USE labl, KILL cr);
15320
15321 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15322 "smulh rscratch2, $op1, $op2\n\t"
15323 "cmp rscratch2, rscratch1, ASR #63\n\t"
15324 "b$cmp $labl" %}
15325 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15326 ins_encode %{
15327 Label* L = $labl$$label;
15328 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15329 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15330 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15331 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15332 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15333 %}
15334
15335 ins_pipe(pipe_serial);
15336 %}
15337
15338 // ============================================================================
15339 // Compare Instructions
15340
15341 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15342 %{
15343 match(Set cr (CmpI op1 op2));
15344
15345 effect(DEF cr, USE op1, USE op2);
15346
15347 ins_cost(INSN_COST);
15348 format %{ "cmpw $op1, $op2" %}
15349
15350 ins_encode(aarch64_enc_cmpw(op1, op2));
15351
15352 ins_pipe(icmp_reg_reg);
15353 %}
15354
15355 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15356 %{
15357 match(Set cr (CmpI op1 zero));
15358
15359 effect(DEF cr, USE op1);
15360
15361 ins_cost(INSN_COST);
15362 format %{ "cmpw $op1, 0" %}
15363
15364 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15365
15366 ins_pipe(icmp_reg_imm);
15367 %}
15368
15369 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15370 %{
15371 match(Set cr (CmpI op1 op2));
15372
15373 effect(DEF cr, USE op1);
15374
15375 ins_cost(INSN_COST);
15376 format %{ "cmpw $op1, $op2" %}
15377
15378 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15379
15380 ins_pipe(icmp_reg_imm);
15381 %}
15382
15383 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15384 %{
15385 match(Set cr (CmpI op1 op2));
15386
15387 effect(DEF cr, USE op1);
15388
15389 ins_cost(INSN_COST * 2);
15390 format %{ "cmpw $op1, $op2" %}
15391
15392 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15393
15394 ins_pipe(icmp_reg_imm);
15395 %}
15396
15397 // Unsigned compare Instructions; really, same as signed compare
15398 // except it should only be used to feed an If or a CMovI which takes a
15399 // cmpOpU.
15400
15401 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15402 %{
15403 match(Set cr (CmpU op1 op2));
15404
15405 effect(DEF cr, USE op1, USE op2);
15406
15407 ins_cost(INSN_COST);
15408 format %{ "cmpw $op1, $op2\t# unsigned" %}
15409
15410 ins_encode(aarch64_enc_cmpw(op1, op2));
15411
15412 ins_pipe(icmp_reg_reg);
15413 %}
15414
15415 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15416 %{
15417 match(Set cr (CmpU op1 zero));
15418
15419 effect(DEF cr, USE op1);
15420
15421 ins_cost(INSN_COST);
15422 format %{ "cmpw $op1, #0\t# unsigned" %}
15423
15424 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15425
15426 ins_pipe(icmp_reg_imm);
15427 %}
15428
15429 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15430 %{
15431 match(Set cr (CmpU op1 op2));
15432
15433 effect(DEF cr, USE op1);
15434
15435 ins_cost(INSN_COST);
15436 format %{ "cmpw $op1, $op2\t# unsigned" %}
15437
15438 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15439
15440 ins_pipe(icmp_reg_imm);
15441 %}
15442
15443 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15444 %{
15445 match(Set cr (CmpU op1 op2));
15446
15447 effect(DEF cr, USE op1);
15448
15449 ins_cost(INSN_COST * 2);
15450 format %{ "cmpw $op1, $op2\t# unsigned" %}
15451
15452 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15453
15454 ins_pipe(icmp_reg_imm);
15455 %}
15456
15457 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15458 %{
15459 match(Set cr (CmpL op1 op2));
15460
15461 effect(DEF cr, USE op1, USE op2);
15462
15463 ins_cost(INSN_COST);
15464 format %{ "cmp $op1, $op2" %}
15465
15466 ins_encode(aarch64_enc_cmp(op1, op2));
15467
15468 ins_pipe(icmp_reg_reg);
15469 %}
15470
15471 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15472 %{
15473 match(Set cr (CmpL op1 zero));
15474
15475 effect(DEF cr, USE op1);
15476
15477 ins_cost(INSN_COST);
15478 format %{ "tst $op1" %}
15479
15480 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15481
15482 ins_pipe(icmp_reg_imm);
15483 %}
15484
15485 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15486 %{
15487 match(Set cr (CmpL op1 op2));
15488
15489 effect(DEF cr, USE op1);
15490
15491 ins_cost(INSN_COST);
15492 format %{ "cmp $op1, $op2" %}
15493
15494 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15495
15496 ins_pipe(icmp_reg_imm);
15497 %}
15498
15499 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15500 %{
15501 match(Set cr (CmpL op1 op2));
15502
15503 effect(DEF cr, USE op1);
15504
15505 ins_cost(INSN_COST * 2);
15506 format %{ "cmp $op1, $op2" %}
15507
15508 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15509
15510 ins_pipe(icmp_reg_imm);
15511 %}
15512
15513 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15514 %{
15515 match(Set cr (CmpUL op1 op2));
15516
15517 effect(DEF cr, USE op1, USE op2);
15518
15519 ins_cost(INSN_COST);
15520 format %{ "cmp $op1, $op2" %}
15521
15522 ins_encode(aarch64_enc_cmp(op1, op2));
15523
15524 ins_pipe(icmp_reg_reg);
15525 %}
15526
15527 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15528 %{
15529 match(Set cr (CmpUL op1 zero));
15530
15531 effect(DEF cr, USE op1);
15532
15533 ins_cost(INSN_COST);
15534 format %{ "tst $op1" %}
15535
15536 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15537
15538 ins_pipe(icmp_reg_imm);
15539 %}
15540
15541 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15542 %{
15543 match(Set cr (CmpUL op1 op2));
15544
15545 effect(DEF cr, USE op1);
15546
15547 ins_cost(INSN_COST);
15548 format %{ "cmp $op1, $op2" %}
15549
15550 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15551
15552 ins_pipe(icmp_reg_imm);
15553 %}
15554
15555 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15556 %{
15557 match(Set cr (CmpUL op1 op2));
15558
15559 effect(DEF cr, USE op1);
15560
15561 ins_cost(INSN_COST * 2);
15562 format %{ "cmp $op1, $op2" %}
15563
15564 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15565
15566 ins_pipe(icmp_reg_imm);
15567 %}
15568
15569 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15570 %{
15571 match(Set cr (CmpP op1 op2));
15572
15573 effect(DEF cr, USE op1, USE op2);
15574
15575 ins_cost(INSN_COST);
15576 format %{ "cmp $op1, $op2\t // ptr" %}
15577
15578 ins_encode(aarch64_enc_cmpp(op1, op2));
15579
15580 ins_pipe(icmp_reg_reg);
15581 %}
15582
15583 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15584 %{
15585 match(Set cr (CmpN op1 op2));
15586
15587 effect(DEF cr, USE op1, USE op2);
15588
15589 ins_cost(INSN_COST);
15590 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15591
15592 ins_encode(aarch64_enc_cmpn(op1, op2));
15593
15594 ins_pipe(icmp_reg_reg);
15595 %}
15596
15597 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15598 %{
15599 match(Set cr (CmpP op1 zero));
15600
15601 effect(DEF cr, USE op1, USE zero);
15602
15603 ins_cost(INSN_COST);
15604 format %{ "cmp $op1, 0\t // ptr" %}
15605
15606 ins_encode(aarch64_enc_testp(op1));
15607
15608 ins_pipe(icmp_reg_imm);
15609 %}
15610
15611 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15612 %{
15613 match(Set cr (CmpN op1 zero));
15614
15615 effect(DEF cr, USE op1, USE zero);
15616
15617 ins_cost(INSN_COST);
15618 format %{ "cmp $op1, 0\t // compressed ptr" %}
15619
15620 ins_encode(aarch64_enc_testn(op1));
15621
15622 ins_pipe(icmp_reg_imm);
15623 %}
15624
15625 // FP comparisons
15626 //
15627 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15628 // using normal cmpOp. See declaration of rFlagsReg for details.
15629
15630 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15631 %{
15632 match(Set cr (CmpF src1 src2));
15633
15634 ins_cost(3 * INSN_COST);
15635 format %{ "fcmps $src1, $src2" %}
15636
15637 ins_encode %{
15638 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15639 %}
15640
15641 ins_pipe(pipe_class_compare);
15642 %}
15643
15644 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15645 %{
15646 match(Set cr (CmpF src1 src2));
15647
15648 ins_cost(3 * INSN_COST);
15649 format %{ "fcmps $src1, 0.0" %}
15650
15651 ins_encode %{
15652 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15653 %}
15654
15655 ins_pipe(pipe_class_compare);
15656 %}
15657 // FROM HERE
15658
15659 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15660 %{
15661 match(Set cr (CmpD src1 src2));
15662
15663 ins_cost(3 * INSN_COST);
15664 format %{ "fcmpd $src1, $src2" %}
15665
15666 ins_encode %{
15667 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15668 %}
15669
15670 ins_pipe(pipe_class_compare);
15671 %}
15672
15673 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15674 %{
15675 match(Set cr (CmpD src1 src2));
15676
15677 ins_cost(3 * INSN_COST);
15678 format %{ "fcmpd $src1, 0.0" %}
15679
15680 ins_encode %{
15681 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15682 %}
15683
15684 ins_pipe(pipe_class_compare);
15685 %}
15686
15687 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15688 %{
15689 match(Set dst (CmpF3 src1 src2));
15690 effect(KILL cr);
15691
15692 ins_cost(5 * INSN_COST);
15693 format %{ "fcmps $src1, $src2\n\t"
15694 "csinvw($dst, zr, zr, eq\n\t"
15695 "csnegw($dst, $dst, $dst, lt)"
15696 %}
15697
15698 ins_encode %{
15699 Label done;
15700 FloatRegister s1 = as_FloatRegister($src1$$reg);
15701 FloatRegister s2 = as_FloatRegister($src2$$reg);
15702 Register d = as_Register($dst$$reg);
15703 __ fcmps(s1, s2);
15704 // installs 0 if EQ else -1
15705 __ csinvw(d, zr, zr, Assembler::EQ);
15706 // keeps -1 if less or unordered else installs 1
15707 __ csnegw(d, d, d, Assembler::LT);
15708 __ bind(done);
15709 %}
15710
15711 ins_pipe(pipe_class_default);
15712
15713 %}
15714
15715 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15716 %{
15717 match(Set dst (CmpD3 src1 src2));
15718 effect(KILL cr);
15719
15720 ins_cost(5 * INSN_COST);
15721 format %{ "fcmpd $src1, $src2\n\t"
15722 "csinvw($dst, zr, zr, eq\n\t"
15723 "csnegw($dst, $dst, $dst, lt)"
15724 %}
15725
15726 ins_encode %{
15727 Label done;
15728 FloatRegister s1 = as_FloatRegister($src1$$reg);
15729 FloatRegister s2 = as_FloatRegister($src2$$reg);
15730 Register d = as_Register($dst$$reg);
15731 __ fcmpd(s1, s2);
15732 // installs 0 if EQ else -1
15733 __ csinvw(d, zr, zr, Assembler::EQ);
15734 // keeps -1 if less or unordered else installs 1
15735 __ csnegw(d, d, d, Assembler::LT);
15736 __ bind(done);
15737 %}
15738 ins_pipe(pipe_class_default);
15739
15740 %}
15741
15742 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15743 %{
15744 match(Set dst (CmpF3 src1 zero));
15745 effect(KILL cr);
15746
15747 ins_cost(5 * INSN_COST);
15748 format %{ "fcmps $src1, 0.0\n\t"
15749 "csinvw($dst, zr, zr, eq\n\t"
15750 "csnegw($dst, $dst, $dst, lt)"
15751 %}
15752
15753 ins_encode %{
15754 Label done;
15755 FloatRegister s1 = as_FloatRegister($src1$$reg);
15756 Register d = as_Register($dst$$reg);
15757 __ fcmps(s1, 0.0);
15758 // installs 0 if EQ else -1
15759 __ csinvw(d, zr, zr, Assembler::EQ);
15760 // keeps -1 if less or unordered else installs 1
15761 __ csnegw(d, d, d, Assembler::LT);
15762 __ bind(done);
15763 %}
15764
15765 ins_pipe(pipe_class_default);
15766
15767 %}
15768
15769 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15770 %{
15771 match(Set dst (CmpD3 src1 zero));
15772 effect(KILL cr);
15773
15774 ins_cost(5 * INSN_COST);
15775 format %{ "fcmpd $src1, 0.0\n\t"
15776 "csinvw($dst, zr, zr, eq\n\t"
15777 "csnegw($dst, $dst, $dst, lt)"
15778 %}
15779
15780 ins_encode %{
15781 Label done;
15782 FloatRegister s1 = as_FloatRegister($src1$$reg);
15783 Register d = as_Register($dst$$reg);
15784 __ fcmpd(s1, 0.0);
15785 // installs 0 if EQ else -1
15786 __ csinvw(d, zr, zr, Assembler::EQ);
15787 // keeps -1 if less or unordered else installs 1
15788 __ csnegw(d, d, d, Assembler::LT);
15789 __ bind(done);
15790 %}
15791 ins_pipe(pipe_class_default);
15792
15793 %}
15794
15795 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15796 %{
15797 match(Set dst (CmpLTMask p q));
15798 effect(KILL cr);
15799
15800 ins_cost(3 * INSN_COST);
15801
15802 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15803 "csetw $dst, lt\n\t"
15804 "subw $dst, zr, $dst"
15805 %}
15806
15807 ins_encode %{
15808 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15809 __ csetw(as_Register($dst$$reg), Assembler::LT);
15810 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15811 %}
15812
15813 ins_pipe(ialu_reg_reg);
15814 %}
15815
15816 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15817 %{
15818 match(Set dst (CmpLTMask src zero));
15819 effect(KILL cr);
15820
15821 ins_cost(INSN_COST);
15822
15823 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15824
15825 ins_encode %{
15826 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15827 %}
15828
15829 ins_pipe(ialu_reg_shift);
15830 %}
15831
15832 // ============================================================================
15833 // Max and Min
15834
15835 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15836
15837 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15838 %{
15839 effect(DEF cr, USE src);
15840 ins_cost(INSN_COST);
15841 format %{ "cmpw $src, 0" %}
15842
15843 ins_encode %{
15844 __ cmpw($src$$Register, 0);
15845 %}
15846 ins_pipe(icmp_reg_imm);
15847 %}
15848
15849 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15850 %{
15851 match(Set dst (MinI src1 src2));
15852 ins_cost(INSN_COST * 3);
15853
15854 expand %{
15855 rFlagsReg cr;
15856 compI_reg_reg(cr, src1, src2);
15857 cmovI_reg_reg_lt(dst, src1, src2, cr);
15858 %}
15859 %}
15860
15861 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15862 %{
15863 match(Set dst (MaxI src1 src2));
15864 ins_cost(INSN_COST * 3);
15865
15866 expand %{
15867 rFlagsReg cr;
15868 compI_reg_reg(cr, src1, src2);
15869 cmovI_reg_reg_gt(dst, src1, src2, cr);
15870 %}
15871 %}
15872
15873
15874 // ============================================================================
15875 // Branch Instructions
15876
15877 // Direct Branch.
15878 instruct branch(label lbl)
15879 %{
15880 match(Goto);
15881
15882 effect(USE lbl);
15883
15884 ins_cost(BRANCH_COST);
15885 format %{ "b $lbl" %}
15886
15887 ins_encode(aarch64_enc_b(lbl));
15888
15889 ins_pipe(pipe_branch);
15890 %}
15891
15892 // Conditional Near Branch
15893 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15894 %{
15895 // Same match rule as `branchConFar'.
15896 match(If cmp cr);
15897
15898 effect(USE lbl);
15899
15900 ins_cost(BRANCH_COST);
15901 // If set to 1 this indicates that the current instruction is a
15902 // short variant of a long branch. This avoids using this
15903 // instruction in first-pass matching. It will then only be used in
15904 // the `Shorten_branches' pass.
15905 // ins_short_branch(1);
15906 format %{ "b$cmp $lbl" %}
15907
15908 ins_encode(aarch64_enc_br_con(cmp, lbl));
15909
15910 ins_pipe(pipe_branch_cond);
15911 %}
15912
15913 // Conditional Near Branch Unsigned
15914 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15915 %{
15916 // Same match rule as `branchConFar'.
15917 match(If cmp cr);
15918
15919 effect(USE lbl);
15920
15921 ins_cost(BRANCH_COST);
15922 // If set to 1 this indicates that the current instruction is a
15923 // short variant of a long branch. This avoids using this
15924 // instruction in first-pass matching. It will then only be used in
15925 // the `Shorten_branches' pass.
15926 // ins_short_branch(1);
15927 format %{ "b$cmp $lbl\t# unsigned" %}
15928
15929 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15930
15931 ins_pipe(pipe_branch_cond);
15932 %}
15933
15934 // Make use of CBZ and CBNZ. These instructions, as well as being
15935 // shorter than (cmp; branch), have the additional benefit of not
15936 // killing the flags.
15937
15938 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15939 match(If cmp (CmpI op1 op2));
15940 effect(USE labl);
15941
15942 ins_cost(BRANCH_COST);
15943 format %{ "cbw$cmp $op1, $labl" %}
15944 ins_encode %{
15945 Label* L = $labl$$label;
15946 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15947 if (cond == Assembler::EQ)
15948 __ cbzw($op1$$Register, *L);
15949 else
15950 __ cbnzw($op1$$Register, *L);
15951 %}
15952 ins_pipe(pipe_cmp_branch);
15953 %}
15954
15955 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15956 match(If cmp (CmpL op1 op2));
15957 effect(USE labl);
15958
15959 ins_cost(BRANCH_COST);
15960 format %{ "cb$cmp $op1, $labl" %}
15961 ins_encode %{
15962 Label* L = $labl$$label;
15963 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15964 if (cond == Assembler::EQ)
15965 __ cbz($op1$$Register, *L);
15966 else
15967 __ cbnz($op1$$Register, *L);
15968 %}
15969 ins_pipe(pipe_cmp_branch);
15970 %}
15971
15972 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15973 match(If cmp (CmpP op1 op2));
15974 effect(USE labl);
15975
15976 ins_cost(BRANCH_COST);
15977 format %{ "cb$cmp $op1, $labl" %}
15978 ins_encode %{
15979 Label* L = $labl$$label;
15980 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15981 if (cond == Assembler::EQ)
15982 __ cbz($op1$$Register, *L);
15983 else
15984 __ cbnz($op1$$Register, *L);
15985 %}
15986 ins_pipe(pipe_cmp_branch);
15987 %}
15988
15989 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15990 match(If cmp (CmpN op1 op2));
15991 effect(USE labl);
15992
15993 ins_cost(BRANCH_COST);
15994 format %{ "cbw$cmp $op1, $labl" %}
15995 ins_encode %{
15996 Label* L = $labl$$label;
15997 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15998 if (cond == Assembler::EQ)
15999 __ cbzw($op1$$Register, *L);
16000 else
16001 __ cbnzw($op1$$Register, *L);
16002 %}
16003 ins_pipe(pipe_cmp_branch);
16004 %}
16005
16006 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16007 match(If cmp (CmpP (DecodeN oop) zero));
16008 effect(USE labl);
16009
16010 ins_cost(BRANCH_COST);
16011 format %{ "cb$cmp $oop, $labl" %}
16012 ins_encode %{
16013 Label* L = $labl$$label;
16014 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16015 if (cond == Assembler::EQ)
16016 __ cbzw($oop$$Register, *L);
16017 else
16018 __ cbnzw($oop$$Register, *L);
16019 %}
16020 ins_pipe(pipe_cmp_branch);
16021 %}
16022
16023 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16024 match(If cmp (CmpU op1 op2));
16025 effect(USE labl);
16026
16027 ins_cost(BRANCH_COST);
16028 format %{ "cbw$cmp $op1, $labl" %}
16029 ins_encode %{
16030 Label* L = $labl$$label;
16031 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16032 if (cond == Assembler::EQ || cond == Assembler::LS) {
16033 __ cbzw($op1$$Register, *L);
16034 } else {
16035 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16036 __ cbnzw($op1$$Register, *L);
16037 }
16038 %}
16039 ins_pipe(pipe_cmp_branch);
16040 %}
16041
16042 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16043 match(If cmp (CmpUL op1 op2));
16044 effect(USE labl);
16045
16046 ins_cost(BRANCH_COST);
16047 format %{ "cb$cmp $op1, $labl" %}
16048 ins_encode %{
16049 Label* L = $labl$$label;
16050 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16051 if (cond == Assembler::EQ || cond == Assembler::LS) {
16052 __ cbz($op1$$Register, *L);
16053 } else {
16054 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16055 __ cbnz($op1$$Register, *L);
16056 }
16057 %}
16058 ins_pipe(pipe_cmp_branch);
16059 %}
16060
16061 // Test bit and Branch
16062
16063 // Patterns for short (< 32KiB) variants
16064 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16065 match(If cmp (CmpL op1 op2));
16066 effect(USE labl);
16067
16068 ins_cost(BRANCH_COST);
16069 format %{ "cb$cmp $op1, $labl # long" %}
16070 ins_encode %{
16071 Label* L = $labl$$label;
16072 Assembler::Condition cond =
16073 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16074 __ tbr(cond, $op1$$Register, 63, *L);
16075 %}
16076 ins_pipe(pipe_cmp_branch);
16077 ins_short_branch(1);
16078 %}
16079
16080 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16081 match(If cmp (CmpI op1 op2));
16082 effect(USE labl);
16083
16084 ins_cost(BRANCH_COST);
16085 format %{ "cb$cmp $op1, $labl # int" %}
16086 ins_encode %{
16087 Label* L = $labl$$label;
16088 Assembler::Condition cond =
16089 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16090 __ tbr(cond, $op1$$Register, 31, *L);
16091 %}
16092 ins_pipe(pipe_cmp_branch);
16093 ins_short_branch(1);
16094 %}
16095
16096 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16097 match(If cmp (CmpL (AndL op1 op2) op3));
16098 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16099 effect(USE labl);
16100
16101 ins_cost(BRANCH_COST);
16102 format %{ "tb$cmp $op1, $op2, $labl" %}
16103 ins_encode %{
16104 Label* L = $labl$$label;
16105 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16106 int bit = exact_log2_long($op2$$constant);
16107 __ tbr(cond, $op1$$Register, bit, *L);
16108 %}
16109 ins_pipe(pipe_cmp_branch);
16110 ins_short_branch(1);
16111 %}
16112
16113 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16114 match(If cmp (CmpI (AndI op1 op2) op3));
16115 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16116 effect(USE labl);
16117
16118 ins_cost(BRANCH_COST);
16119 format %{ "tb$cmp $op1, $op2, $labl" %}
16120 ins_encode %{
16121 Label* L = $labl$$label;
16122 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16123 int bit = exact_log2((juint)$op2$$constant);
16124 __ tbr(cond, $op1$$Register, bit, *L);
16125 %}
16126 ins_pipe(pipe_cmp_branch);
16127 ins_short_branch(1);
16128 %}
16129
16130 // And far variants
16131 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16132 match(If cmp (CmpL op1 op2));
16133 effect(USE labl);
16134
16135 ins_cost(BRANCH_COST);
16136 format %{ "cb$cmp $op1, $labl # long" %}
16137 ins_encode %{
16138 Label* L = $labl$$label;
16139 Assembler::Condition cond =
16140 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16141 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16142 %}
16143 ins_pipe(pipe_cmp_branch);
16144 %}
16145
16146 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16147 match(If cmp (CmpI op1 op2));
16148 effect(USE labl);
16149
16150 ins_cost(BRANCH_COST);
16151 format %{ "cb$cmp $op1, $labl # int" %}
16152 ins_encode %{
16153 Label* L = $labl$$label;
16154 Assembler::Condition cond =
16155 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16156 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16157 %}
16158 ins_pipe(pipe_cmp_branch);
16159 %}
16160
16161 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16162 match(If cmp (CmpL (AndL op1 op2) op3));
16163 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16164 effect(USE labl);
16165
16166 ins_cost(BRANCH_COST);
16167 format %{ "tb$cmp $op1, $op2, $labl" %}
16168 ins_encode %{
16169 Label* L = $labl$$label;
16170 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16171 int bit = exact_log2_long($op2$$constant);
16172 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16173 %}
16174 ins_pipe(pipe_cmp_branch);
16175 %}
16176
16177 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16178 match(If cmp (CmpI (AndI op1 op2) op3));
16179 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16180 effect(USE labl);
16181
16182 ins_cost(BRANCH_COST);
16183 format %{ "tb$cmp $op1, $op2, $labl" %}
16184 ins_encode %{
16185 Label* L = $labl$$label;
16186 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16187 int bit = exact_log2((juint)$op2$$constant);
16188 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16189 %}
16190 ins_pipe(pipe_cmp_branch);
16191 %}
16192
16193 // Test bits
16194
16195 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16196 match(Set cr (CmpL (AndL op1 op2) op3));
16197 predicate(Assembler::operand_valid_for_logical_immediate
16198 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16199
16200 ins_cost(INSN_COST);
16201 format %{ "tst $op1, $op2 # long" %}
16202 ins_encode %{
16203 __ tst($op1$$Register, $op2$$constant);
16204 %}
16205 ins_pipe(ialu_reg_reg);
16206 %}
16207
16208 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16209 match(Set cr (CmpI (AndI op1 op2) op3));
16210 predicate(Assembler::operand_valid_for_logical_immediate
16211 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16212
16213 ins_cost(INSN_COST);
16214 format %{ "tst $op1, $op2 # int" %}
16215 ins_encode %{
16216 __ tstw($op1$$Register, $op2$$constant);
16217 %}
16218 ins_pipe(ialu_reg_reg);
16219 %}
16220
16221 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16222 match(Set cr (CmpL (AndL op1 op2) op3));
16223
16224 ins_cost(INSN_COST);
16225 format %{ "tst $op1, $op2 # long" %}
16226 ins_encode %{
16227 __ tst($op1$$Register, $op2$$Register);
16228 %}
16229 ins_pipe(ialu_reg_reg);
16230 %}
16231
16232 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16233 match(Set cr (CmpI (AndI op1 op2) op3));
16234
16235 ins_cost(INSN_COST);
16236 format %{ "tstw $op1, $op2 # int" %}
16237 ins_encode %{
16238 __ tstw($op1$$Register, $op2$$Register);
16239 %}
16240 ins_pipe(ialu_reg_reg);
16241 %}
16242
16243
16244 // Conditional Far Branch
16245 // Conditional Far Branch Unsigned
16246 // TODO: fixme
16247
16248 // counted loop end branch near
16249 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16250 %{
16251 match(CountedLoopEnd cmp cr);
16252
16253 effect(USE lbl);
16254
16255 ins_cost(BRANCH_COST);
16256 // short variant.
16257 // ins_short_branch(1);
16258 format %{ "b$cmp $lbl \t// counted loop end" %}
16259
16260 ins_encode(aarch64_enc_br_con(cmp, lbl));
16261
16262 ins_pipe(pipe_branch);
16263 %}
16264
16265 // counted loop end branch far
16266 // TODO: fixme
16267
16268 // ============================================================================
16269 // inlined locking and unlocking
16270
16271 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16272 %{
16273 match(Set cr (FastLock object box));
16274 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16275
16276 ins_cost(5 * INSN_COST);
16277 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16278
16279 ins_encode %{
16280 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16281 %}
16282
16283 ins_pipe(pipe_serial);
16284 %}
16285
16286 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16287 %{
16288 match(Set cr (FastUnlock object box));
16289 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16290
16291 ins_cost(5 * INSN_COST);
16292 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16293
16294 ins_encode %{
16295 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16296 %}
16297
16298 ins_pipe(pipe_serial);
16299 %}
16300
16301 // ============================================================================
16302 // Safepoint Instructions
16303
16304 // TODO
16305 // provide a near and far version of this code
16306
16307 instruct safePoint(rFlagsReg cr, iRegP poll)
16308 %{
16309 match(SafePoint poll);
16310 effect(KILL cr);
16311
16312 format %{
16313 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16314 %}
16315 ins_encode %{
16316 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16317 %}
16318 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16319 %}
16320
16321
16322 // ============================================================================
16323 // Procedure Call/Return Instructions
16324
16325 // Call Java Static Instruction
16326
16327 instruct CallStaticJavaDirect(method meth)
16328 %{
16329 match(CallStaticJava);
16330
16331 effect(USE meth);
16332
16333 ins_cost(CALL_COST);
16334
16335 format %{ "call,static $meth \t// ==> " %}
16336
16337 ins_encode(aarch64_enc_java_static_call(meth),
16338 aarch64_enc_call_epilog);
16339
16340 ins_pipe(pipe_class_call);
16341 %}
16342
16343 // TO HERE
16344
16345 // Call Java Dynamic Instruction
16346 instruct CallDynamicJavaDirect(method meth)
16347 %{
16348 match(CallDynamicJava);
16349
16350 effect(USE meth);
16351
16352 ins_cost(CALL_COST);
16353
16354 format %{ "CALL,dynamic $meth \t// ==> " %}
16355
16356 ins_encode(aarch64_enc_java_dynamic_call(meth),
16357 aarch64_enc_call_epilog);
16358
16359 ins_pipe(pipe_class_call);
16360 %}
16361
16362 // Call Runtime Instruction
16363
16364 instruct CallRuntimeDirect(method meth)
16365 %{
16366 match(CallRuntime);
16367
16368 effect(USE meth);
16369
16370 ins_cost(CALL_COST);
16371
16372 format %{ "CALL, runtime $meth" %}
16373
16374 ins_encode( aarch64_enc_java_to_runtime(meth) );
16375
16376 ins_pipe(pipe_class_call);
16377 %}
16378
16379 // Call Runtime Instruction
16380
16381 instruct CallLeafDirect(method meth)
16382 %{
16383 match(CallLeaf);
16384
16385 effect(USE meth);
16386
16387 ins_cost(CALL_COST);
16388
16389 format %{ "CALL, runtime leaf $meth" %}
16390
16391 ins_encode( aarch64_enc_java_to_runtime(meth) );
16392
16393 ins_pipe(pipe_class_call);
16394 %}
16395
16396 // Call Runtime Instruction without safepoint and with vector arguments
16397 instruct CallLeafDirectVector(method meth)
16398 %{
16399 match(CallLeafVector);
16400
16401 effect(USE meth);
16402
16403 ins_cost(CALL_COST);
16404
16405 format %{ "CALL, runtime leaf vector $meth" %}
16406
16407 ins_encode(aarch64_enc_java_to_runtime(meth));
16408
16409 ins_pipe(pipe_class_call);
16410 %}
16411
16412 // Call Runtime Instruction
16413
16414 instruct CallLeafNoFPDirect(method meth)
16415 %{
16416 match(CallLeafNoFP);
16417
16418 effect(USE meth);
16419
16420 ins_cost(CALL_COST);
16421
16422 format %{ "CALL, runtime leaf nofp $meth" %}
16423
16424 ins_encode( aarch64_enc_java_to_runtime(meth) );
16425
16426 ins_pipe(pipe_class_call);
16427 %}
16428
16429 // Tail Call; Jump from runtime stub to Java code.
16430 // Also known as an 'interprocedural jump'.
16431 // Target of jump will eventually return to caller.
16432 // TailJump below removes the return address.
16433 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16434 // emitted just above the TailCall which has reset rfp to the caller state.
16435 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16436 %{
16437 match(TailCall jump_target method_ptr);
16438
16439 ins_cost(CALL_COST);
16440
16441 format %{ "br $jump_target\t# $method_ptr holds method" %}
16442
16443 ins_encode(aarch64_enc_tail_call(jump_target));
16444
16445 ins_pipe(pipe_class_call);
16446 %}
16447
16448 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16449 %{
16450 match(TailJump jump_target ex_oop);
16451
16452 ins_cost(CALL_COST);
16453
16454 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16455
16456 ins_encode(aarch64_enc_tail_jmp(jump_target));
16457
16458 ins_pipe(pipe_class_call);
16459 %}
16460
16461 // Forward exception.
16462 instruct ForwardExceptionjmp()
16463 %{
16464 match(ForwardException);
16465 ins_cost(CALL_COST);
16466
16467 format %{ "b forward_exception_stub" %}
16468 ins_encode %{
16469 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16470 %}
16471 ins_pipe(pipe_class_call);
16472 %}
16473
16474 // Create exception oop: created by stack-crawling runtime code.
16475 // Created exception is now available to this handler, and is setup
16476 // just prior to jumping to this handler. No code emitted.
16477 // TODO check
16478 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16479 instruct CreateException(iRegP_R0 ex_oop)
16480 %{
16481 match(Set ex_oop (CreateEx));
16482
16483 format %{ " -- \t// exception oop; no code emitted" %}
16484
16485 size(0);
16486
16487 ins_encode( /*empty*/ );
16488
16489 ins_pipe(pipe_class_empty);
16490 %}
16491
16492 // Rethrow exception: The exception oop will come in the first
16493 // argument position. Then JUMP (not call) to the rethrow stub code.
16494 instruct RethrowException() %{
16495 match(Rethrow);
16496 ins_cost(CALL_COST);
16497
16498 format %{ "b rethrow_stub" %}
16499
16500 ins_encode( aarch64_enc_rethrow() );
16501
16502 ins_pipe(pipe_class_call);
16503 %}
16504
16505
16506 // Return Instruction
16507 // epilog node loads ret address into lr as part of frame pop
16508 instruct Ret()
16509 %{
16510 match(Return);
16511
16512 format %{ "ret\t// return register" %}
16513
16514 ins_encode( aarch64_enc_ret() );
16515
16516 ins_pipe(pipe_branch);
16517 %}
16518
16519 // Die now.
16520 instruct ShouldNotReachHere() %{
16521 match(Halt);
16522
16523 ins_cost(CALL_COST);
16524 format %{ "ShouldNotReachHere" %}
16525
16526 ins_encode %{
16527 if (is_reachable()) {
16528 const char* str = __ code_string(_halt_reason);
16529 __ stop(str);
16530 }
16531 %}
16532
16533 ins_pipe(pipe_class_default);
16534 %}
16535
16536 // ============================================================================
16537 // Partial Subtype Check
16538 //
16539 // superklass array for an instance of the superklass. Set a hidden
16540 // internal cache on a hit (cache is checked with exposed code in
16541 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16542 // encoding ALSO sets flags.
16543
16544 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16545 %{
16546 match(Set result (PartialSubtypeCheck sub super));
16547 predicate(!UseSecondarySupersTable);
16548 effect(KILL cr, KILL temp);
16549
16550 ins_cost(20 * INSN_COST); // slightly larger than the next version
16551 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16552
16553 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16554
16555 opcode(0x1); // Force zero of result reg on hit
16556
16557 ins_pipe(pipe_class_memory);
16558 %}
16559
16560 // Two versions of partialSubtypeCheck, both used when we need to
16561 // search for a super class in the secondary supers array. The first
16562 // is used when we don't know _a priori_ the class being searched
16563 // for. The second, far more common, is used when we do know: this is
16564 // used for instanceof, checkcast, and any case where C2 can determine
16565 // it by constant propagation.
16566
16567 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16568 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16569 rFlagsReg cr)
16570 %{
16571 match(Set result (PartialSubtypeCheck sub super));
16572 predicate(UseSecondarySupersTable);
16573 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16574
16575 ins_cost(10 * INSN_COST); // slightly larger than the next version
16576 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16577
16578 ins_encode %{
16579 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16580 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16581 $vtemp$$FloatRegister,
16582 $result$$Register, /*L_success*/nullptr);
16583 %}
16584
16585 ins_pipe(pipe_class_memory);
16586 %}
16587
16588 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16589 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16590 rFlagsReg cr)
16591 %{
16592 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16593 predicate(UseSecondarySupersTable);
16594 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16595
16596 ins_cost(5 * INSN_COST); // smaller than the next version
16597 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16598
16599 ins_encode %{
16600 bool success = false;
16601 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16602 if (InlineSecondarySupersTest) {
16603 success =
16604 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16605 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16606 $vtemp$$FloatRegister,
16607 $result$$Register,
16608 super_klass_slot);
16609 } else {
16610 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16611 success = (call != nullptr);
16612 }
16613 if (!success) {
16614 ciEnv::current()->record_failure("CodeCache is full");
16615 return;
16616 }
16617 %}
16618
16619 ins_pipe(pipe_class_memory);
16620 %}
16621
16622 // Intrisics for String.compareTo()
16623
16624 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16625 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16626 %{
16627 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16628 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16629 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16630
16631 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16632 ins_encode %{
16633 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16634 __ string_compare($str1$$Register, $str2$$Register,
16635 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16636 $tmp1$$Register, $tmp2$$Register,
16637 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16638 %}
16639 ins_pipe(pipe_class_memory);
16640 %}
16641
16642 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16643 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16644 %{
16645 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16646 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16647 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16648
16649 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16650 ins_encode %{
16651 __ string_compare($str1$$Register, $str2$$Register,
16652 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16653 $tmp1$$Register, $tmp2$$Register,
16654 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16655 %}
16656 ins_pipe(pipe_class_memory);
16657 %}
16658
16659 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16660 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16661 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16662 %{
16663 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16664 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16665 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16666 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16667
16668 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16669 ins_encode %{
16670 __ string_compare($str1$$Register, $str2$$Register,
16671 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16672 $tmp1$$Register, $tmp2$$Register,
16673 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16674 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16675 %}
16676 ins_pipe(pipe_class_memory);
16677 %}
16678
16679 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16680 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16681 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16682 %{
16683 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16684 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16685 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16686 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16687
16688 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16689 ins_encode %{
16690 __ string_compare($str1$$Register, $str2$$Register,
16691 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16692 $tmp1$$Register, $tmp2$$Register,
16693 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16694 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16695 %}
16696 ins_pipe(pipe_class_memory);
16697 %}
16698
16699 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16700 // these string_compare variants as NEON register type for convenience so that the prototype of
16701 // string_compare can be shared with all variants.
16702
16703 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16704 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16705 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16706 pRegGov_P1 pgtmp2, rFlagsReg cr)
16707 %{
16708 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16709 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16710 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16711 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16712
16713 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16714 ins_encode %{
16715 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16716 __ string_compare($str1$$Register, $str2$$Register,
16717 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16718 $tmp1$$Register, $tmp2$$Register,
16719 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16720 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16721 StrIntrinsicNode::LL);
16722 %}
16723 ins_pipe(pipe_class_memory);
16724 %}
16725
16726 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16727 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16728 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16729 pRegGov_P1 pgtmp2, rFlagsReg cr)
16730 %{
16731 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16732 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16733 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16734 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16735
16736 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16737 ins_encode %{
16738 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16739 __ string_compare($str1$$Register, $str2$$Register,
16740 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16741 $tmp1$$Register, $tmp2$$Register,
16742 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16743 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16744 StrIntrinsicNode::LU);
16745 %}
16746 ins_pipe(pipe_class_memory);
16747 %}
16748
16749 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16750 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16751 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16752 pRegGov_P1 pgtmp2, rFlagsReg cr)
16753 %{
16754 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16755 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16756 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16757 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16758
16759 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16760 ins_encode %{
16761 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16762 __ string_compare($str1$$Register, $str2$$Register,
16763 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16764 $tmp1$$Register, $tmp2$$Register,
16765 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16766 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16767 StrIntrinsicNode::UL);
16768 %}
16769 ins_pipe(pipe_class_memory);
16770 %}
16771
16772 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16773 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16774 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16775 pRegGov_P1 pgtmp2, rFlagsReg cr)
16776 %{
16777 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16778 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16779 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16780 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16781
16782 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16783 ins_encode %{
16784 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16785 __ string_compare($str1$$Register, $str2$$Register,
16786 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16787 $tmp1$$Register, $tmp2$$Register,
16788 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16789 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16790 StrIntrinsicNode::UU);
16791 %}
16792 ins_pipe(pipe_class_memory);
16793 %}
16794
16795 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16796 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16797 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16798 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16799 %{
16800 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16801 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16802 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16803 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16804 TEMP vtmp0, TEMP vtmp1, KILL cr);
16805 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16806 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16807
16808 ins_encode %{
16809 __ string_indexof($str1$$Register, $str2$$Register,
16810 $cnt1$$Register, $cnt2$$Register,
16811 $tmp1$$Register, $tmp2$$Register,
16812 $tmp3$$Register, $tmp4$$Register,
16813 $tmp5$$Register, $tmp6$$Register,
16814 -1, $result$$Register, StrIntrinsicNode::UU);
16815 %}
16816 ins_pipe(pipe_class_memory);
16817 %}
16818
16819 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16820 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16821 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16822 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16823 %{
16824 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16825 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16826 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16827 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16828 TEMP vtmp0, TEMP vtmp1, KILL cr);
16829 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16830 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16831
16832 ins_encode %{
16833 __ string_indexof($str1$$Register, $str2$$Register,
16834 $cnt1$$Register, $cnt2$$Register,
16835 $tmp1$$Register, $tmp2$$Register,
16836 $tmp3$$Register, $tmp4$$Register,
16837 $tmp5$$Register, $tmp6$$Register,
16838 -1, $result$$Register, StrIntrinsicNode::LL);
16839 %}
16840 ins_pipe(pipe_class_memory);
16841 %}
16842
16843 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16844 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16845 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16846 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16847 %{
16848 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16849 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16850 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16851 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16852 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16853 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16854 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16855
16856 ins_encode %{
16857 __ string_indexof($str1$$Register, $str2$$Register,
16858 $cnt1$$Register, $cnt2$$Register,
16859 $tmp1$$Register, $tmp2$$Register,
16860 $tmp3$$Register, $tmp4$$Register,
16861 $tmp5$$Register, $tmp6$$Register,
16862 -1, $result$$Register, StrIntrinsicNode::UL);
16863 %}
16864 ins_pipe(pipe_class_memory);
16865 %}
16866
16867 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16868 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16869 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16870 %{
16871 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16872 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16873 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16874 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16875 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16876 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16877
16878 ins_encode %{
16879 int icnt2 = (int)$int_cnt2$$constant;
16880 __ string_indexof($str1$$Register, $str2$$Register,
16881 $cnt1$$Register, zr,
16882 $tmp1$$Register, $tmp2$$Register,
16883 $tmp3$$Register, $tmp4$$Register, zr, zr,
16884 icnt2, $result$$Register, StrIntrinsicNode::UU);
16885 %}
16886 ins_pipe(pipe_class_memory);
16887 %}
16888
16889 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16890 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16891 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16892 %{
16893 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16894 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16895 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16896 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16897 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16898 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16899
16900 ins_encode %{
16901 int icnt2 = (int)$int_cnt2$$constant;
16902 __ string_indexof($str1$$Register, $str2$$Register,
16903 $cnt1$$Register, zr,
16904 $tmp1$$Register, $tmp2$$Register,
16905 $tmp3$$Register, $tmp4$$Register, zr, zr,
16906 icnt2, $result$$Register, StrIntrinsicNode::LL);
16907 %}
16908 ins_pipe(pipe_class_memory);
16909 %}
16910
16911 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16912 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16913 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16914 %{
16915 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16916 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16917 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16918 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16919 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16920 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16921
16922 ins_encode %{
16923 int icnt2 = (int)$int_cnt2$$constant;
16924 __ string_indexof($str1$$Register, $str2$$Register,
16925 $cnt1$$Register, zr,
16926 $tmp1$$Register, $tmp2$$Register,
16927 $tmp3$$Register, $tmp4$$Register, zr, zr,
16928 icnt2, $result$$Register, StrIntrinsicNode::UL);
16929 %}
16930 ins_pipe(pipe_class_memory);
16931 %}
16932
16933 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16934 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16935 iRegINoSp tmp3, rFlagsReg cr)
16936 %{
16937 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16938 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16939 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16940 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16941
16942 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16943
16944 ins_encode %{
16945 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16946 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16947 $tmp3$$Register);
16948 %}
16949 ins_pipe(pipe_class_memory);
16950 %}
16951
16952 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16953 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16954 iRegINoSp tmp3, rFlagsReg cr)
16955 %{
16956 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16957 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16958 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16959 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16960
16961 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16962
16963 ins_encode %{
16964 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16965 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16966 $tmp3$$Register);
16967 %}
16968 ins_pipe(pipe_class_memory);
16969 %}
16970
16971 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16972 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16973 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16974 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16975 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16976 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16977 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16978 ins_encode %{
16979 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16980 $result$$Register, $ztmp1$$FloatRegister,
16981 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16982 $ptmp$$PRegister, true /* isL */);
16983 %}
16984 ins_pipe(pipe_class_memory);
16985 %}
16986
16987 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16988 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16989 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16990 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16991 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16992 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16993 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16994 ins_encode %{
16995 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16996 $result$$Register, $ztmp1$$FloatRegister,
16997 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16998 $ptmp$$PRegister, false /* isL */);
16999 %}
17000 ins_pipe(pipe_class_memory);
17001 %}
17002
17003 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17004 iRegI_R0 result, rFlagsReg cr)
17005 %{
17006 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17007 match(Set result (StrEquals (Binary str1 str2) cnt));
17008 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17009
17010 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17011 ins_encode %{
17012 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17013 __ string_equals($str1$$Register, $str2$$Register,
17014 $result$$Register, $cnt$$Register);
17015 %}
17016 ins_pipe(pipe_class_memory);
17017 %}
17018
17019 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17020 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17021 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17022 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17023 iRegP_R10 tmp, rFlagsReg cr)
17024 %{
17025 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17026 match(Set result (AryEq ary1 ary2));
17027 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17028 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17029 TEMP vtmp6, TEMP vtmp7, KILL cr);
17030
17031 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17032 ins_encode %{
17033 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17034 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17035 $result$$Register, $tmp$$Register, 1);
17036 if (tpc == nullptr) {
17037 ciEnv::current()->record_failure("CodeCache is full");
17038 return;
17039 }
17040 %}
17041 ins_pipe(pipe_class_memory);
17042 %}
17043
17044 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17045 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17046 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17047 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17048 iRegP_R10 tmp, rFlagsReg cr)
17049 %{
17050 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17051 match(Set result (AryEq ary1 ary2));
17052 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17053 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17054 TEMP vtmp6, TEMP vtmp7, KILL cr);
17055
17056 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17057 ins_encode %{
17058 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17059 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17060 $result$$Register, $tmp$$Register, 2);
17061 if (tpc == nullptr) {
17062 ciEnv::current()->record_failure("CodeCache is full");
17063 return;
17064 }
17065 %}
17066 ins_pipe(pipe_class_memory);
17067 %}
17068
17069 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17070 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17071 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17072 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17073 %{
17074 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17075 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17076 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17077
17078 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
17079 ins_encode %{
17080 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17081 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17082 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17083 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17084 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17085 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17086 (BasicType)$basic_type$$constant);
17087 if (tpc == nullptr) {
17088 ciEnv::current()->record_failure("CodeCache is full");
17089 return;
17090 }
17091 %}
17092 ins_pipe(pipe_class_memory);
17093 %}
17094
17095 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17096 %{
17097 match(Set result (CountPositives ary1 len));
17098 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17099 format %{ "count positives byte[] $ary1,$len -> $result" %}
17100 ins_encode %{
17101 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17102 if (tpc == nullptr) {
17103 ciEnv::current()->record_failure("CodeCache is full");
17104 return;
17105 }
17106 %}
17107 ins_pipe( pipe_slow );
17108 %}
17109
17110 // fast char[] to byte[] compression
17111 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17112 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17113 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17114 iRegI_R0 result, rFlagsReg cr)
17115 %{
17116 match(Set result (StrCompressedCopy src (Binary dst len)));
17117 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17118 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17119
17120 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17121 ins_encode %{
17122 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17123 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17124 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17125 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17126 %}
17127 ins_pipe(pipe_slow);
17128 %}
17129
17130 // fast byte[] to char[] inflation
17131 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17132 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17133 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17134 %{
17135 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17136 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17137 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17138 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17139
17140 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17141 ins_encode %{
17142 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17143 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17144 $vtmp2$$FloatRegister, $tmp$$Register);
17145 if (tpc == nullptr) {
17146 ciEnv::current()->record_failure("CodeCache is full");
17147 return;
17148 }
17149 %}
17150 ins_pipe(pipe_class_memory);
17151 %}
17152
17153 // encode char[] to byte[] in ISO_8859_1
17154 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17155 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17156 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17157 iRegI_R0 result, rFlagsReg cr)
17158 %{
17159 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17160 match(Set result (EncodeISOArray src (Binary dst len)));
17161 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17162 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17163
17164 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17165 ins_encode %{
17166 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17167 $result$$Register, false,
17168 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17169 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17170 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17171 %}
17172 ins_pipe(pipe_class_memory);
17173 %}
17174
17175 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17176 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17177 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17178 iRegI_R0 result, rFlagsReg cr)
17179 %{
17180 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17181 match(Set result (EncodeISOArray src (Binary dst len)));
17182 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17183 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17184
17185 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17186 ins_encode %{
17187 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17188 $result$$Register, true,
17189 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17190 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17191 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17192 %}
17193 ins_pipe(pipe_class_memory);
17194 %}
17195
17196 //----------------------------- CompressBits/ExpandBits ------------------------
17197
17198 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17199 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17200 match(Set dst (CompressBits src mask));
17201 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17202 format %{ "mov $tsrc, $src\n\t"
17203 "mov $tmask, $mask\n\t"
17204 "bext $tdst, $tsrc, $tmask\n\t"
17205 "mov $dst, $tdst"
17206 %}
17207 ins_encode %{
17208 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17209 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17210 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17211 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17212 %}
17213 ins_pipe(pipe_slow);
17214 %}
17215
17216 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17217 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17218 match(Set dst (CompressBits (LoadI mem) mask));
17219 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17220 format %{ "ldrs $tsrc, $mem\n\t"
17221 "ldrs $tmask, $mask\n\t"
17222 "bext $tdst, $tsrc, $tmask\n\t"
17223 "mov $dst, $tdst"
17224 %}
17225 ins_encode %{
17226 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17227 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17228 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17229 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17230 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17231 %}
17232 ins_pipe(pipe_slow);
17233 %}
17234
17235 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17236 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17237 match(Set dst (CompressBits src mask));
17238 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17239 format %{ "mov $tsrc, $src\n\t"
17240 "mov $tmask, $mask\n\t"
17241 "bext $tdst, $tsrc, $tmask\n\t"
17242 "mov $dst, $tdst"
17243 %}
17244 ins_encode %{
17245 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17246 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17247 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17248 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17249 %}
17250 ins_pipe(pipe_slow);
17251 %}
17252
17253 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17254 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17255 match(Set dst (CompressBits (LoadL mem) mask));
17256 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17257 format %{ "ldrd $tsrc, $mem\n\t"
17258 "ldrd $tmask, $mask\n\t"
17259 "bext $tdst, $tsrc, $tmask\n\t"
17260 "mov $dst, $tdst"
17261 %}
17262 ins_encode %{
17263 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17264 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17265 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17266 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17267 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17268 %}
17269 ins_pipe(pipe_slow);
17270 %}
17271
17272 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17273 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17274 match(Set dst (ExpandBits src mask));
17275 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17276 format %{ "mov $tsrc, $src\n\t"
17277 "mov $tmask, $mask\n\t"
17278 "bdep $tdst, $tsrc, $tmask\n\t"
17279 "mov $dst, $tdst"
17280 %}
17281 ins_encode %{
17282 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17283 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17284 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17285 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17286 %}
17287 ins_pipe(pipe_slow);
17288 %}
17289
17290 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17291 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17292 match(Set dst (ExpandBits (LoadI mem) mask));
17293 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17294 format %{ "ldrs $tsrc, $mem\n\t"
17295 "ldrs $tmask, $mask\n\t"
17296 "bdep $tdst, $tsrc, $tmask\n\t"
17297 "mov $dst, $tdst"
17298 %}
17299 ins_encode %{
17300 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17301 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17302 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17303 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17304 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17305 %}
17306 ins_pipe(pipe_slow);
17307 %}
17308
17309 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17310 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17311 match(Set dst (ExpandBits src mask));
17312 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17313 format %{ "mov $tsrc, $src\n\t"
17314 "mov $tmask, $mask\n\t"
17315 "bdep $tdst, $tsrc, $tmask\n\t"
17316 "mov $dst, $tdst"
17317 %}
17318 ins_encode %{
17319 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17320 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17321 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17322 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17323 %}
17324 ins_pipe(pipe_slow);
17325 %}
17326
17327
17328 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17329 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17330 match(Set dst (ExpandBits (LoadL mem) mask));
17331 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17332 format %{ "ldrd $tsrc, $mem\n\t"
17333 "ldrd $tmask, $mask\n\t"
17334 "bdep $tdst, $tsrc, $tmask\n\t"
17335 "mov $dst, $tdst"
17336 %}
17337 ins_encode %{
17338 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17339 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17340 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17341 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17342 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17343 %}
17344 ins_pipe(pipe_slow);
17345 %}
17346
17347 //----------------------------- Reinterpret ----------------------------------
17348 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17349 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17350 match(Set dst (ReinterpretHF2S src));
17351 format %{ "reinterpretHF2S $dst, $src" %}
17352 ins_encode %{
17353 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17354 %}
17355 ins_pipe(pipe_slow);
17356 %}
17357
17358 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17359 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17360 match(Set dst (ReinterpretS2HF src));
17361 format %{ "reinterpretS2HF $dst, $src" %}
17362 ins_encode %{
17363 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17364 %}
17365 ins_pipe(pipe_slow);
17366 %}
17367
17368 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17369 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17370 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
17371 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
17372 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
17373 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17374 // can be omitted in this pattern, resulting in -
17375 // fcvt $dst, $src // Convert float to half-precision float
17376 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17377 %{
17378 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17379 format %{ "convF2HFAndS2HF $dst, $src" %}
17380 ins_encode %{
17381 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17382 %}
17383 ins_pipe(pipe_slow);
17384 %}
17385
17386 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17387 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17388 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
17389 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17390 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
17391 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17392 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17393 // resulting in -
17394 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
17395 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17396 %{
17397 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17398 format %{ "convHF2SAndHF2F $dst, $src" %}
17399 ins_encode %{
17400 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17401 %}
17402 ins_pipe(pipe_slow);
17403 %}
17404
17405 // ============================================================================
17406 // This name is KNOWN by the ADLC and cannot be changed.
17407 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17408 // for this guy.
17409 instruct tlsLoadP(thread_RegP dst)
17410 %{
17411 match(Set dst (ThreadLocal));
17412
17413 ins_cost(0);
17414
17415 format %{ " -- \t// $dst=Thread::current(), empty" %}
17416
17417 size(0);
17418
17419 ins_encode( /*empty*/ );
17420
17421 ins_pipe(pipe_class_empty);
17422 %}
17423
17424 //----------PEEPHOLE RULES-----------------------------------------------------
17425 // These must follow all instruction definitions as they use the names
17426 // defined in the instructions definitions.
17427 //
17428 // peepmatch ( root_instr_name [preceding_instruction]* );
17429 //
17430 // peepconstraint %{
17431 // (instruction_number.operand_name relational_op instruction_number.operand_name
17432 // [, ...] );
17433 // // instruction numbers are zero-based using left to right order in peepmatch
17434 //
17435 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17436 // // provide an instruction_number.operand_name for each operand that appears
17437 // // in the replacement instruction's match rule
17438 //
17439 // ---------VM FLAGS---------------------------------------------------------
17440 //
17441 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17442 //
17443 // Each peephole rule is given an identifying number starting with zero and
17444 // increasing by one in the order seen by the parser. An individual peephole
17445 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17446 // on the command-line.
17447 //
17448 // ---------CURRENT LIMITATIONS----------------------------------------------
17449 //
17450 // Only match adjacent instructions in same basic block
17451 // Only equality constraints
17452 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17453 // Only one replacement instruction
17454 //
17455 // ---------EXAMPLE----------------------------------------------------------
17456 //
17457 // // pertinent parts of existing instructions in architecture description
17458 // instruct movI(iRegINoSp dst, iRegI src)
17459 // %{
17460 // match(Set dst (CopyI src));
17461 // %}
17462 //
17463 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17464 // %{
17465 // match(Set dst (AddI dst src));
17466 // effect(KILL cr);
17467 // %}
17468 //
17469 // // Change (inc mov) to lea
17470 // peephole %{
17471 // // increment preceded by register-register move
17472 // peepmatch ( incI_iReg movI );
17473 // // require that the destination register of the increment
17474 // // match the destination register of the move
17475 // peepconstraint ( 0.dst == 1.dst );
17476 // // construct a replacement instruction that sets
17477 // // the destination to ( move's source register + one )
17478 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17479 // %}
17480 //
17481
17482 // Implementation no longer uses movX instructions since
17483 // machine-independent system no longer uses CopyX nodes.
17484 //
17485 // peephole
17486 // %{
17487 // peepmatch (incI_iReg movI);
17488 // peepconstraint (0.dst == 1.dst);
17489 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17490 // %}
17491
17492 // peephole
17493 // %{
17494 // peepmatch (decI_iReg movI);
17495 // peepconstraint (0.dst == 1.dst);
17496 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17497 // %}
17498
17499 // peephole
17500 // %{
17501 // peepmatch (addI_iReg_imm movI);
17502 // peepconstraint (0.dst == 1.dst);
17503 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17504 // %}
17505
17506 // peephole
17507 // %{
17508 // peepmatch (incL_iReg movL);
17509 // peepconstraint (0.dst == 1.dst);
17510 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17511 // %}
17512
17513 // peephole
17514 // %{
17515 // peepmatch (decL_iReg movL);
17516 // peepconstraint (0.dst == 1.dst);
17517 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17518 // %}
17519
17520 // peephole
17521 // %{
17522 // peepmatch (addL_iReg_imm movL);
17523 // peepconstraint (0.dst == 1.dst);
17524 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17525 // %}
17526
17527 // peephole
17528 // %{
17529 // peepmatch (addP_iReg_imm movP);
17530 // peepconstraint (0.dst == 1.dst);
17531 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17532 // %}
17533
17534 // // Change load of spilled value to only a spill
17535 // instruct storeI(memory mem, iRegI src)
17536 // %{
17537 // match(Set mem (StoreI mem src));
17538 // %}
17539 //
17540 // instruct loadI(iRegINoSp dst, memory mem)
17541 // %{
17542 // match(Set dst (LoadI mem));
17543 // %}
17544 //
17545
17546 //----------SMARTSPILL RULES---------------------------------------------------
17547 // These must follow all instruction definitions as they use the names
17548 // defined in the instructions definitions.
17549
17550 // Local Variables:
17551 // mode: c++
17552 // End: