1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BootTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else if (_entry_point == nullptr) {
1694 // See CallLeafNoFPIndirect
1695 return 1 * NativeInstruction::instruction_size;
1696 } else {
1697 return 6 * NativeInstruction::instruction_size;
1698 }
1699 }
1700
1701 //=============================================================================
1702
1703 #ifndef PRODUCT
1704 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1705 st->print("BREAKPOINT");
1706 }
1707 #endif
1708
1709 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1710 __ brk(0);
1711 }
1712
1713 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1714 return MachNode::size(ra_);
1715 }
1716
1717 //=============================================================================
1718
1719 #ifndef PRODUCT
1720 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1721 st->print("nop \t# %d bytes pad for loops and calls", _count);
1722 }
1723 #endif
1724
1725 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1726 for (int i = 0; i < _count; i++) {
1727 __ nop();
1728 }
1729 }
1730
1731 uint MachNopNode::size(PhaseRegAlloc*) const {
1732 return _count * NativeInstruction::instruction_size;
1733 }
1734
1735 //=============================================================================
1736 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1737
1738 int ConstantTable::calculate_table_base_offset() const {
1739 return 0; // absolute addressing, no offset
1740 }
1741
1742 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1743 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1744 ShouldNotReachHere();
1745 }
1746
1747 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1748 // Empty encoding
1749 }
1750
1751 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1752 return 0;
1753 }
1754
1755 #ifndef PRODUCT
1756 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1757 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1758 }
1759 #endif
1760
1761 #ifndef PRODUCT
1762 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1763 Compile* C = ra_->C;
1764
1765 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1766
1767 if (C->output()->need_stack_bang(framesize))
1768 st->print("# stack bang size=%d\n\t", framesize);
1769
1770 if (VM_Version::use_rop_protection()) {
1771 st->print("ldr zr, [lr]\n\t");
1772 st->print("paciaz\n\t");
1773 }
1774 if (framesize < ((1 << 9) + 2 * wordSize)) {
1775 st->print("sub sp, sp, #%d\n\t", framesize);
1776 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1777 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1778 } else {
1779 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1780 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1781 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1782 st->print("sub sp, sp, rscratch1");
1783 }
1784 if (C->stub_function() == nullptr) {
1785 st->print("\n\t");
1786 st->print("ldr rscratch1, [guard]\n\t");
1787 st->print("dmb ishld\n\t");
1788 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1789 st->print("cmp rscratch1, rscratch2\n\t");
1790 st->print("b.eq skip");
1791 st->print("\n\t");
1792 st->print("blr #nmethod_entry_barrier_stub\n\t");
1793 st->print("b skip\n\t");
1794 st->print("guard: int\n\t");
1795 st->print("\n\t");
1796 st->print("skip:\n\t");
1797 }
1798 }
1799 #endif
1800
1801 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1802 Compile* C = ra_->C;
1803
1804
1805 __ verified_entry(C, 0);
1806
1807 if (C->stub_function() == nullptr) {
1808 __ entry_barrier();
1809 }
1810
1811 if (!Compile::current()->output()->in_scratch_emit_size()) {
1812 __ bind(*_verified_entry);
1813 }
1814
1815 if (VerifyStackAtCalls) {
1816 Unimplemented();
1817 }
1818
1819 C->output()->set_frame_complete(__ offset());
1820
1821 if (C->has_mach_constant_base_node()) {
1822 // NOTE: We set the table base offset here because users might be
1823 // emitted before MachConstantBaseNode.
1824 ConstantTable& constant_table = C->output()->constant_table();
1825 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1826 }
1827 }
1828
1829 int MachPrologNode::reloc() const
1830 {
1831 return 0;
1832 }
1833
1834 //=============================================================================
1835
1836 #ifndef PRODUCT
1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1838 Compile* C = ra_->C;
1839 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1840
1841 st->print("# pop frame %d\n\t",framesize);
1842
1843 if (framesize == 0) {
1844 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1845 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1846 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1847 st->print("add sp, sp, #%d\n\t", framesize);
1848 } else {
1849 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1850 st->print("add sp, sp, rscratch1\n\t");
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 }
1853 if (VM_Version::use_rop_protection()) {
1854 st->print("autiaz\n\t");
1855 st->print("ldr zr, [lr]\n\t");
1856 }
1857
1858 if (do_polling() && C->is_method_compilation()) {
1859 st->print("# test polling word\n\t");
1860 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1861 st->print("cmp sp, rscratch1\n\t");
1862 st->print("bhi #slow_path");
1863 }
1864 }
1865 #endif
1866
1867 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1868 Compile* C = ra_->C;
1869 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1870
1871 __ remove_frame(framesize, C->needs_stack_repair());
1872
1873 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1874 __ reserved_stack_check();
1875 }
1876
1877 if (do_polling() && C->is_method_compilation()) {
1878 Label dummy_label;
1879 Label* code_stub = &dummy_label;
1880 if (!C->output()->in_scratch_emit_size()) {
1881 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1882 C->output()->add_stub(stub);
1883 code_stub = &stub->entry();
1884 }
1885 __ relocate(relocInfo::poll_return_type);
1886 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1887 }
1888 }
1889
1890 int MachEpilogNode::reloc() const {
1891 // Return number of relocatable values contained in this instruction.
1892 return 1; // 1 for polling page.
1893 }
1894
1895 const Pipeline * MachEpilogNode::pipeline() const {
1896 return MachNode::pipeline_class();
1897 }
1898
1899 //=============================================================================
1900
1901 static enum RC rc_class(OptoReg::Name reg) {
1902
1903 if (reg == OptoReg::Bad) {
1904 return rc_bad;
1905 }
1906
1907 // we have 32 int registers * 2 halves
1908 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1909
1910 if (reg < slots_of_int_registers) {
1911 return rc_int;
1912 }
1913
1914 // we have 32 float register * 8 halves
1915 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1916 if (reg < slots_of_int_registers + slots_of_float_registers) {
1917 return rc_float;
1918 }
1919
1920 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1921 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1922 return rc_predicate;
1923 }
1924
1925 // Between predicate regs & stack is the flags.
1926 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1927
1928 return rc_stack;
1929 }
1930
1931 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1932 Compile* C = ra_->C;
1933
1934 // Get registers to move.
1935 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1936 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1937 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1938 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1939
1940 enum RC src_hi_rc = rc_class(src_hi);
1941 enum RC src_lo_rc = rc_class(src_lo);
1942 enum RC dst_hi_rc = rc_class(dst_hi);
1943 enum RC dst_lo_rc = rc_class(dst_lo);
1944
1945 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1946
1947 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1948 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1949 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1950 "expected aligned-adjacent pairs");
1951 }
1952
1953 if (src_lo == dst_lo && src_hi == dst_hi) {
1954 return 0; // Self copy, no move.
1955 }
1956
1957 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1958 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1959 int src_offset = ra_->reg2offset(src_lo);
1960 int dst_offset = ra_->reg2offset(dst_lo);
1961
1962 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1963 uint ireg = ideal_reg();
1964 if (ireg == Op_VecA && masm) {
1965 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1966 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1967 // stack->stack
1968 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1969 sve_vector_reg_size_in_bytes);
1970 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1971 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1972 sve_vector_reg_size_in_bytes);
1973 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1974 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1975 sve_vector_reg_size_in_bytes);
1976 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1977 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1978 as_FloatRegister(Matcher::_regEncode[src_lo]),
1979 as_FloatRegister(Matcher::_regEncode[src_lo]));
1980 } else {
1981 ShouldNotReachHere();
1982 }
1983 } else if (masm) {
1984 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1985 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1986 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1987 // stack->stack
1988 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
1989 if (ireg == Op_VecD) {
1990 __ unspill(rscratch1, true, src_offset);
1991 __ spill(rscratch1, true, dst_offset);
1992 } else {
1993 __ spill_copy128(src_offset, dst_offset);
1994 }
1995 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1996 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1997 ireg == Op_VecD ? __ T8B : __ T16B,
1998 as_FloatRegister(Matcher::_regEncode[src_lo]));
1999 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2000 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2001 ireg == Op_VecD ? __ D : __ Q,
2002 ra_->reg2offset(dst_lo));
2003 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2004 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2005 ireg == Op_VecD ? __ D : __ Q,
2006 ra_->reg2offset(src_lo));
2007 } else {
2008 ShouldNotReachHere();
2009 }
2010 }
2011 } else if (masm) {
2012 switch (src_lo_rc) {
2013 case rc_int:
2014 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2015 if (is64) {
2016 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2017 as_Register(Matcher::_regEncode[src_lo]));
2018 } else {
2019 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2020 as_Register(Matcher::_regEncode[src_lo]));
2021 }
2022 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2023 if (is64) {
2024 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2025 as_Register(Matcher::_regEncode[src_lo]));
2026 } else {
2027 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2028 as_Register(Matcher::_regEncode[src_lo]));
2029 }
2030 } else { // gpr --> stack spill
2031 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2032 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2033 }
2034 break;
2035 case rc_float:
2036 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2037 if (is64) {
2038 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2039 as_FloatRegister(Matcher::_regEncode[src_lo]));
2040 } else {
2041 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2042 as_FloatRegister(Matcher::_regEncode[src_lo]));
2043 }
2044 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2045 if (is64) {
2046 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2047 as_FloatRegister(Matcher::_regEncode[src_lo]));
2048 } else {
2049 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2050 as_FloatRegister(Matcher::_regEncode[src_lo]));
2051 }
2052 } else { // fpr --> stack spill
2053 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2054 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2055 is64 ? __ D : __ S, dst_offset);
2056 }
2057 break;
2058 case rc_stack:
2059 if (dst_lo_rc == rc_int) { // stack --> gpr load
2060 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2061 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2062 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2063 is64 ? __ D : __ S, src_offset);
2064 } else if (dst_lo_rc == rc_predicate) {
2065 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2066 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2067 } else { // stack --> stack copy
2068 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2069 if (ideal_reg() == Op_RegVectMask) {
2070 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2071 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2072 } else {
2073 __ unspill(rscratch1, is64, src_offset);
2074 __ spill(rscratch1, is64, dst_offset);
2075 }
2076 }
2077 break;
2078 case rc_predicate:
2079 if (dst_lo_rc == rc_predicate) {
2080 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2081 } else if (dst_lo_rc == rc_stack) {
2082 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2083 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2084 } else {
2085 assert(false, "bad src and dst rc_class combination.");
2086 ShouldNotReachHere();
2087 }
2088 break;
2089 default:
2090 assert(false, "bad rc_class for spill");
2091 ShouldNotReachHere();
2092 }
2093 }
2094
2095 if (st) {
2096 st->print("spill ");
2097 if (src_lo_rc == rc_stack) {
2098 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2099 } else {
2100 st->print("%s -> ", Matcher::regName[src_lo]);
2101 }
2102 if (dst_lo_rc == rc_stack) {
2103 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2104 } else {
2105 st->print("%s", Matcher::regName[dst_lo]);
2106 }
2107 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2108 int vsize = 0;
2109 switch (ideal_reg()) {
2110 case Op_VecD:
2111 vsize = 64;
2112 break;
2113 case Op_VecX:
2114 vsize = 128;
2115 break;
2116 case Op_VecA:
2117 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2118 break;
2119 default:
2120 assert(false, "bad register type for spill");
2121 ShouldNotReachHere();
2122 }
2123 st->print("\t# vector spill size = %d", vsize);
2124 } else if (ideal_reg() == Op_RegVectMask) {
2125 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2126 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2127 st->print("\t# predicate spill size = %d", vsize);
2128 } else {
2129 st->print("\t# spill size = %d", is64 ? 64 : 32);
2130 }
2131 }
2132
2133 return 0;
2134
2135 }
2136
2137 #ifndef PRODUCT
2138 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2139 if (!ra_)
2140 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2141 else
2142 implementation(nullptr, ra_, false, st);
2143 }
2144 #endif
2145
2146 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2147 implementation(masm, ra_, false, nullptr);
2148 }
2149
2150 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2151 return MachNode::size(ra_);
2152 }
2153
2154 //=============================================================================
2155
2156 #ifndef PRODUCT
2157 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2158 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2159 int reg = ra_->get_reg_first(this);
2160 st->print("add %s, rsp, #%d]\t# box lock",
2161 Matcher::regName[reg], offset);
2162 }
2163 #endif
2164
2165 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2166 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2167 int reg = ra_->get_encode(this);
2168
2169 // This add will handle any 24-bit signed offset. 24 bits allows an
2170 // 8 megabyte stack frame.
2171 __ add(as_Register(reg), sp, offset);
2172 }
2173
2174 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2175 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2176 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2177
2178 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2179 return NativeInstruction::instruction_size;
2180 } else {
2181 return 2 * NativeInstruction::instruction_size;
2182 }
2183 }
2184
2185 ///=============================================================================
2186 #ifndef PRODUCT
2187 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2188 {
2189 st->print_cr("# MachVEPNode");
2190 if (!_verified) {
2191 st->print_cr("\t load_class");
2192 } else {
2193 st->print_cr("\t unpack_inline_arg");
2194 }
2195 }
2196 #endif
2197
2198 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2199 {
2200 if (!_verified) {
2201 __ ic_check(1);
2202 } else {
2203 // TODO 8284443 Avoid creation of temporary frame
2204 if (ra_->C->stub_function() == nullptr) {
2205 __ verified_entry(ra_->C, 0);
2206 __ entry_barrier();
2207 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2208 __ remove_frame(framesize, false);
2209 }
2210 // Unpack inline type args passed as oop and then jump to
2211 // the verified entry point (skipping the unverified entry).
2212 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2213 // Emit code for verified entry and save increment for stack repair on return
2214 __ verified_entry(ra_->C, sp_inc);
2215 if (Compile::current()->output()->in_scratch_emit_size()) {
2216 Label dummy_verified_entry;
2217 __ b(dummy_verified_entry);
2218 } else {
2219 __ b(*_verified_entry);
2220 }
2221 }
2222 }
2223
2224 //=============================================================================
2225 #ifndef PRODUCT
2226 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2227 {
2228 st->print_cr("# MachUEPNode");
2229 if (UseCompressedClassPointers) {
2230 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2231 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2232 st->print_cr("\tcmpw rscratch1, r10");
2233 } else {
2234 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2235 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2236 st->print_cr("\tcmp rscratch1, r10");
2237 }
2238 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2239 }
2240 #endif
2241
2242 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2243 {
2244 __ ic_check(InteriorEntryAlignment);
2245 }
2246
2247 // REQUIRED EMIT CODE
2248
2249 //=============================================================================
2250
2251 // Emit deopt handler code.
2252 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2253 {
2254 // Note that the code buffer's insts_mark is always relative to insts.
2255 // That's why we must use the macroassembler to generate a handler.
2256 address base = __ start_a_stub(size_deopt_handler());
2257 if (base == nullptr) {
2258 ciEnv::current()->record_failure("CodeCache is full");
2259 return 0; // CodeBuffer::expand failed
2260 }
2261
2262 int offset = __ offset();
2263 Label start;
2264 __ bind(start);
2265 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2266
2267 int entry_offset = __ offset();
2268 __ b(start);
2269
2270 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2271 __ end_a_stub();
2272 return entry_offset;
2273 }
2274
2275 // REQUIRED MATCHER CODE
2276
2277 //=============================================================================
2278
2279 bool Matcher::match_rule_supported(int opcode) {
2280 if (!has_match_rule(opcode))
2281 return false;
2282
2283 switch (opcode) {
2284 case Op_OnSpinWait:
2285 return VM_Version::supports_on_spin_wait();
2286 case Op_CacheWB:
2287 case Op_CacheWBPreSync:
2288 case Op_CacheWBPostSync:
2289 if (!VM_Version::supports_data_cache_line_flush()) {
2290 return false;
2291 }
2292 break;
2293 case Op_ExpandBits:
2294 case Op_CompressBits:
2295 if (!VM_Version::supports_svebitperm()) {
2296 return false;
2297 }
2298 break;
2299 case Op_FmaF:
2300 case Op_FmaD:
2301 case Op_FmaVF:
2302 case Op_FmaVD:
2303 if (!UseFMA) {
2304 return false;
2305 }
2306 break;
2307 case Op_FmaHF:
2308 // UseFMA flag also needs to be checked along with FEAT_FP16
2309 if (!UseFMA || !is_feat_fp16_supported()) {
2310 return false;
2311 }
2312 break;
2313 case Op_AddHF:
2314 case Op_SubHF:
2315 case Op_MulHF:
2316 case Op_DivHF:
2317 case Op_MinHF:
2318 case Op_MaxHF:
2319 case Op_SqrtHF:
2320 // Half-precision floating point scalar operations require FEAT_FP16
2321 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2322 // features are supported.
2323 if (!is_feat_fp16_supported()) {
2324 return false;
2325 }
2326 break;
2327 }
2328
2329 return true; // Per default match rules are supported.
2330 }
2331
2332 const RegMask* Matcher::predicate_reg_mask(void) {
2333 return &_PR_REG_mask;
2334 }
2335
2336 bool Matcher::supports_vector_calling_convention(void) {
2337 return EnableVectorSupport;
2338 }
2339
2340 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2341 assert(EnableVectorSupport, "sanity");
2342 int lo = V0_num;
2343 int hi = V0_H_num;
2344 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2345 hi = V0_K_num;
2346 }
2347 return OptoRegPair(hi, lo);
2348 }
2349
2350 // Is this branch offset short enough that a short branch can be used?
2351 //
2352 // NOTE: If the platform does not provide any short branch variants, then
2353 // this method should return false for offset 0.
2354 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2355 // The passed offset is relative to address of the branch.
2356
2357 return (-32768 <= offset && offset < 32768);
2358 }
2359
2360 // Vector width in bytes.
2361 int Matcher::vector_width_in_bytes(BasicType bt) {
2362 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2363 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2364 // Minimum 2 values in vector
2365 if (size < 2*type2aelembytes(bt)) size = 0;
2366 // But never < 4
2367 if (size < 4) size = 0;
2368 return size;
2369 }
2370
2371 // Limits on vector size (number of elements) loaded into vector.
2372 int Matcher::max_vector_size(const BasicType bt) {
2373 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2374 }
2375
2376 int Matcher::min_vector_size(const BasicType bt) {
2377 // Usually, the shortest vector length supported by AArch64 ISA and
2378 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2379 // vectors in a few special cases.
2380 int size;
2381 switch(bt) {
2382 case T_BOOLEAN:
2383 // Load/store a vector mask with only 2 elements for vector types
2384 // such as "2I/2F/2L/2D".
2385 size = 2;
2386 break;
2387 case T_BYTE:
2388 // Generate a "4B" vector, to support vector cast between "8B/16B"
2389 // and "4S/4I/4L/4F/4D".
2390 size = 4;
2391 break;
2392 case T_SHORT:
2393 // Generate a "2S" vector, to support vector cast between "4S/8S"
2394 // and "2I/2L/2F/2D".
2395 size = 2;
2396 break;
2397 default:
2398 // Limit the min vector length to 64-bit.
2399 size = 8 / type2aelembytes(bt);
2400 // The number of elements in a vector should be at least 2.
2401 size = MAX2(size, 2);
2402 }
2403
2404 int max_size = max_vector_size(bt);
2405 return MIN2(size, max_size);
2406 }
2407
2408 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2409 return Matcher::max_vector_size(bt);
2410 }
2411
2412 // Actual max scalable vector register length.
2413 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2414 return Matcher::max_vector_size(bt);
2415 }
2416
2417 // Vector ideal reg.
2418 uint Matcher::vector_ideal_reg(int len) {
2419 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2420 return Op_VecA;
2421 }
2422 switch(len) {
2423 // For 16-bit/32-bit mask vector, reuse VecD.
2424 case 2:
2425 case 4:
2426 case 8: return Op_VecD;
2427 case 16: return Op_VecX;
2428 }
2429 ShouldNotReachHere();
2430 return 0;
2431 }
2432
2433 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2434 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2435 switch (ideal_reg) {
2436 case Op_VecA: return new vecAOper();
2437 case Op_VecD: return new vecDOper();
2438 case Op_VecX: return new vecXOper();
2439 }
2440 ShouldNotReachHere();
2441 return nullptr;
2442 }
2443
2444 bool Matcher::is_reg2reg_move(MachNode* m) {
2445 return false;
2446 }
2447
2448 bool Matcher::is_generic_vector(MachOper* opnd) {
2449 return opnd->opcode() == VREG;
2450 }
2451
2452 // Return whether or not this register is ever used as an argument.
2453 // This function is used on startup to build the trampoline stubs in
2454 // generateOptoStub. Registers not mentioned will be killed by the VM
2455 // call in the trampoline, and arguments in those registers not be
2456 // available to the callee.
2457 bool Matcher::can_be_java_arg(int reg)
2458 {
2459 return
2460 reg == R0_num || reg == R0_H_num ||
2461 reg == R1_num || reg == R1_H_num ||
2462 reg == R2_num || reg == R2_H_num ||
2463 reg == R3_num || reg == R3_H_num ||
2464 reg == R4_num || reg == R4_H_num ||
2465 reg == R5_num || reg == R5_H_num ||
2466 reg == R6_num || reg == R6_H_num ||
2467 reg == R7_num || reg == R7_H_num ||
2468 reg == V0_num || reg == V0_H_num ||
2469 reg == V1_num || reg == V1_H_num ||
2470 reg == V2_num || reg == V2_H_num ||
2471 reg == V3_num || reg == V3_H_num ||
2472 reg == V4_num || reg == V4_H_num ||
2473 reg == V5_num || reg == V5_H_num ||
2474 reg == V6_num || reg == V6_H_num ||
2475 reg == V7_num || reg == V7_H_num;
2476 }
2477
2478 bool Matcher::is_spillable_arg(int reg)
2479 {
2480 return can_be_java_arg(reg);
2481 }
2482
2483 uint Matcher::int_pressure_limit()
2484 {
2485 // JDK-8183543: When taking the number of available registers as int
2486 // register pressure threshold, the jtreg test:
2487 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2488 // failed due to C2 compilation failure with
2489 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2490 //
2491 // A derived pointer is live at CallNode and then is flagged by RA
2492 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2493 // derived pointers and lastly fail to spill after reaching maximum
2494 // number of iterations. Lowering the default pressure threshold to
2495 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2496 // a high register pressure area of the code so that split_DEF can
2497 // generate DefinitionSpillCopy for the derived pointer.
2498 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2499 if (!PreserveFramePointer) {
2500 // When PreserveFramePointer is off, frame pointer is allocatable,
2501 // but different from other SOC registers, it is excluded from
2502 // fatproj's mask because its save type is No-Save. Decrease 1 to
2503 // ensure high pressure at fatproj when PreserveFramePointer is off.
2504 // See check_pressure_at_fatproj().
2505 default_int_pressure_threshold--;
2506 }
2507 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2508 }
2509
2510 uint Matcher::float_pressure_limit()
2511 {
2512 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2513 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2514 }
2515
2516 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2517 return false;
2518 }
2519
2520 const RegMask& Matcher::divI_proj_mask() {
2521 ShouldNotReachHere();
2522 return RegMask::EMPTY;
2523 }
2524
2525 // Register for MODI projection of divmodI.
2526 const RegMask& Matcher::modI_proj_mask() {
2527 ShouldNotReachHere();
2528 return RegMask::EMPTY;
2529 }
2530
2531 // Register for DIVL projection of divmodL.
2532 const RegMask& Matcher::divL_proj_mask() {
2533 ShouldNotReachHere();
2534 return RegMask::EMPTY;
2535 }
2536
2537 // Register for MODL projection of divmodL.
2538 const RegMask& Matcher::modL_proj_mask() {
2539 ShouldNotReachHere();
2540 return RegMask::EMPTY;
2541 }
2542
2543 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2544 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2545 Node* u = addp->fast_out(i);
2546 if (u->is_LoadStore()) {
2547 // On AArch64, LoadStoreNodes (i.e. compare and swap
2548 // instructions) only take register indirect as an operand, so
2549 // any attempt to use an AddPNode as an input to a LoadStoreNode
2550 // must fail.
2551 return false;
2552 }
2553 if (u->is_Mem()) {
2554 int opsize = u->as_Mem()->memory_size();
2555 assert(opsize > 0, "unexpected memory operand size");
2556 if (u->as_Mem()->memory_size() != (1<<shift)) {
2557 return false;
2558 }
2559 }
2560 }
2561 return true;
2562 }
2563
2564 // Convert BootTest condition to Assembler condition.
2565 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2566 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2567 Assembler::Condition result;
2568 switch(cond) {
2569 case BoolTest::eq:
2570 result = Assembler::EQ; break;
2571 case BoolTest::ne:
2572 result = Assembler::NE; break;
2573 case BoolTest::le:
2574 result = Assembler::LE; break;
2575 case BoolTest::ge:
2576 result = Assembler::GE; break;
2577 case BoolTest::lt:
2578 result = Assembler::LT; break;
2579 case BoolTest::gt:
2580 result = Assembler::GT; break;
2581 case BoolTest::ule:
2582 result = Assembler::LS; break;
2583 case BoolTest::uge:
2584 result = Assembler::HS; break;
2585 case BoolTest::ult:
2586 result = Assembler::LO; break;
2587 case BoolTest::ugt:
2588 result = Assembler::HI; break;
2589 case BoolTest::overflow:
2590 result = Assembler::VS; break;
2591 case BoolTest::no_overflow:
2592 result = Assembler::VC; break;
2593 default:
2594 ShouldNotReachHere();
2595 return Assembler::Condition(-1);
2596 }
2597
2598 // Check conversion
2599 if (cond & BoolTest::unsigned_compare) {
2600 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2601 } else {
2602 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2603 }
2604
2605 return result;
2606 }
2607
2608 // Binary src (Replicate con)
2609 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2610 if (n == nullptr || m == nullptr) {
2611 return false;
2612 }
2613
2614 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2615 return false;
2616 }
2617
2618 Node* imm_node = m->in(1);
2619 if (!imm_node->is_Con()) {
2620 return false;
2621 }
2622
2623 const Type* t = imm_node->bottom_type();
2624 if (!(t->isa_int() || t->isa_long())) {
2625 return false;
2626 }
2627
2628 switch (n->Opcode()) {
2629 case Op_AndV:
2630 case Op_OrV:
2631 case Op_XorV: {
2632 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2633 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2634 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2635 }
2636 case Op_AddVB:
2637 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2638 case Op_AddVS:
2639 case Op_AddVI:
2640 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2641 case Op_AddVL:
2642 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2643 default:
2644 return false;
2645 }
2646 }
2647
2648 // (XorV src (Replicate m1))
2649 // (XorVMask src (MaskAll m1))
2650 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2651 if (n != nullptr && m != nullptr) {
2652 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2653 VectorNode::is_all_ones_vector(m);
2654 }
2655 return false;
2656 }
2657
2658 // Should the matcher clone input 'm' of node 'n'?
2659 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2660 if (is_vshift_con_pattern(n, m) ||
2661 is_vector_bitwise_not_pattern(n, m) ||
2662 is_valid_sve_arith_imm_pattern(n, m) ||
2663 is_encode_and_store_pattern(n, m)) {
2664 mstack.push(m, Visit);
2665 return true;
2666 }
2667 return false;
2668 }
2669
2670 // Should the Matcher clone shifts on addressing modes, expecting them
2671 // to be subsumed into complex addressing expressions or compute them
2672 // into registers?
2673 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2674
2675 // Loads and stores with indirect memory input (e.g., volatile loads and
2676 // stores) do not subsume the input into complex addressing expressions. If
2677 // the addressing expression is input to at least one such load or store, do
2678 // not clone the addressing expression. Query needs_acquiring_load and
2679 // needs_releasing_store as a proxy for indirect memory input, as it is not
2680 // possible to directly query for indirect memory input at this stage.
2681 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2682 Node* n = m->fast_out(i);
2683 if (n->is_Load() && needs_acquiring_load(n)) {
2684 return false;
2685 }
2686 if (n->is_Store() && needs_releasing_store(n)) {
2687 return false;
2688 }
2689 }
2690
2691 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2692 return true;
2693 }
2694
2695 Node *off = m->in(AddPNode::Offset);
2696 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2697 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2698 // Are there other uses besides address expressions?
2699 !is_visited(off)) {
2700 address_visited.set(off->_idx); // Flag as address_visited
2701 mstack.push(off->in(2), Visit);
2702 Node *conv = off->in(1);
2703 if (conv->Opcode() == Op_ConvI2L &&
2704 // Are there other uses besides address expressions?
2705 !is_visited(conv)) {
2706 address_visited.set(conv->_idx); // Flag as address_visited
2707 mstack.push(conv->in(1), Pre_Visit);
2708 } else {
2709 mstack.push(conv, Pre_Visit);
2710 }
2711 address_visited.test_set(m->_idx); // Flag as address_visited
2712 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2713 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2714 return true;
2715 } else if (off->Opcode() == Op_ConvI2L &&
2716 // Are there other uses besides address expressions?
2717 !is_visited(off)) {
2718 address_visited.test_set(m->_idx); // Flag as address_visited
2719 address_visited.set(off->_idx); // Flag as address_visited
2720 mstack.push(off->in(1), Pre_Visit);
2721 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2722 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2723 return true;
2724 }
2725 return false;
2726 }
2727
2728 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2729 { \
2730 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2731 guarantee(DISP == 0, "mode not permitted for volatile"); \
2732 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2733 __ INSN(REG, as_Register(BASE)); \
2734 }
2735
2736
2737 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2738 {
2739 Address::extend scale;
2740
2741 // Hooboy, this is fugly. We need a way to communicate to the
2742 // encoder that the index needs to be sign extended, so we have to
2743 // enumerate all the cases.
2744 switch (opcode) {
2745 case INDINDEXSCALEDI2L:
2746 case INDINDEXSCALEDI2LN:
2747 case INDINDEXI2L:
2748 case INDINDEXI2LN:
2749 scale = Address::sxtw(size);
2750 break;
2751 default:
2752 scale = Address::lsl(size);
2753 }
2754
2755 if (index == -1) {
2756 return Address(base, disp);
2757 } else {
2758 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2759 return Address(base, as_Register(index), scale);
2760 }
2761 }
2762
2763
2764 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2765 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2766 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2767 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2768 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2769
2770 // Used for all non-volatile memory accesses. The use of
2771 // $mem->opcode() to discover whether this pattern uses sign-extended
2772 // offsets is something of a kludge.
2773 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2774 Register reg, int opcode,
2775 Register base, int index, int scale, int disp,
2776 int size_in_memory)
2777 {
2778 Address addr = mem2address(opcode, base, index, scale, disp);
2779 if (addr.getMode() == Address::base_plus_offset) {
2780 /* Fix up any out-of-range offsets. */
2781 assert_different_registers(rscratch1, base);
2782 assert_different_registers(rscratch1, reg);
2783 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2784 }
2785 (masm->*insn)(reg, addr);
2786 }
2787
2788 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2789 FloatRegister reg, int opcode,
2790 Register base, int index, int size, int disp,
2791 int size_in_memory)
2792 {
2793 Address::extend scale;
2794
2795 switch (opcode) {
2796 case INDINDEXSCALEDI2L:
2797 case INDINDEXSCALEDI2LN:
2798 scale = Address::sxtw(size);
2799 break;
2800 default:
2801 scale = Address::lsl(size);
2802 }
2803
2804 if (index == -1) {
2805 // Fix up any out-of-range offsets.
2806 assert_different_registers(rscratch1, base);
2807 Address addr = Address(base, disp);
2808 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2809 (masm->*insn)(reg, addr);
2810 } else {
2811 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2812 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2813 }
2814 }
2815
2816 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2817 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2818 int opcode, Register base, int index, int size, int disp)
2819 {
2820 if (index == -1) {
2821 (masm->*insn)(reg, T, Address(base, disp));
2822 } else {
2823 assert(disp == 0, "unsupported address mode");
2824 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2825 }
2826 }
2827
2828 %}
2829
2830
2831
2832 //----------ENCODING BLOCK-----------------------------------------------------
2833 // This block specifies the encoding classes used by the compiler to
2834 // output byte streams. Encoding classes are parameterized macros
2835 // used by Machine Instruction Nodes in order to generate the bit
2836 // encoding of the instruction. Operands specify their base encoding
2837 // interface with the interface keyword. There are currently
2838 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2839 // COND_INTER. REG_INTER causes an operand to generate a function
2840 // which returns its register number when queried. CONST_INTER causes
2841 // an operand to generate a function which returns the value of the
2842 // constant when queried. MEMORY_INTER causes an operand to generate
2843 // four functions which return the Base Register, the Index Register,
2844 // the Scale Value, and the Offset Value of the operand when queried.
2845 // COND_INTER causes an operand to generate six functions which return
2846 // the encoding code (ie - encoding bits for the instruction)
2847 // associated with each basic boolean condition for a conditional
2848 // instruction.
2849 //
2850 // Instructions specify two basic values for encoding. Again, a
2851 // function is available to check if the constant displacement is an
2852 // oop. They use the ins_encode keyword to specify their encoding
2853 // classes (which must be a sequence of enc_class names, and their
2854 // parameters, specified in the encoding block), and they use the
2855 // opcode keyword to specify, in order, their primary, secondary, and
2856 // tertiary opcode. Only the opcode sections which a particular
2857 // instruction needs for encoding need to be specified.
2858 encode %{
2859 // Build emit functions for each basic byte or larger field in the
2860 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2861 // from C++ code in the enc_class source block. Emit functions will
2862 // live in the main source block for now. In future, we can
2863 // generalize this by adding a syntax that specifies the sizes of
2864 // fields in an order, so that the adlc can build the emit functions
2865 // automagically
2866
2867 // catch all for unimplemented encodings
2868 enc_class enc_unimplemented %{
2869 __ unimplemented("C2 catch all");
2870 %}
2871
2872 // BEGIN Non-volatile memory access
2873
2874 // This encoding class is generated automatically from ad_encode.m4.
2875 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2876 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2877 Register dst_reg = as_Register($dst$$reg);
2878 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2879 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2880 %}
2881
2882 // This encoding class is generated automatically from ad_encode.m4.
2883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2884 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2885 Register dst_reg = as_Register($dst$$reg);
2886 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2887 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2888 %}
2889
2890 // This encoding class is generated automatically from ad_encode.m4.
2891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2892 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2893 Register dst_reg = as_Register($dst$$reg);
2894 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2895 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2896 %}
2897
2898 // This encoding class is generated automatically from ad_encode.m4.
2899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2900 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2901 Register dst_reg = as_Register($dst$$reg);
2902 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2903 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2904 %}
2905
2906 // This encoding class is generated automatically from ad_encode.m4.
2907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2908 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2909 Register dst_reg = as_Register($dst$$reg);
2910 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2911 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2912 %}
2913
2914 // This encoding class is generated automatically from ad_encode.m4.
2915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2916 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2917 Register dst_reg = as_Register($dst$$reg);
2918 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2919 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2920 %}
2921
2922 // This encoding class is generated automatically from ad_encode.m4.
2923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2924 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2925 Register dst_reg = as_Register($dst$$reg);
2926 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2927 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2928 %}
2929
2930 // This encoding class is generated automatically from ad_encode.m4.
2931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2932 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2933 Register dst_reg = as_Register($dst$$reg);
2934 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2935 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2936 %}
2937
2938 // This encoding class is generated automatically from ad_encode.m4.
2939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2940 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2941 Register dst_reg = as_Register($dst$$reg);
2942 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2943 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2944 %}
2945
2946 // This encoding class is generated automatically from ad_encode.m4.
2947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2948 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2949 Register dst_reg = as_Register($dst$$reg);
2950 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2951 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2952 %}
2953
2954 // This encoding class is generated automatically from ad_encode.m4.
2955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2956 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2957 Register dst_reg = as_Register($dst$$reg);
2958 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2959 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2960 %}
2961
2962 // This encoding class is generated automatically from ad_encode.m4.
2963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2964 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2965 Register dst_reg = as_Register($dst$$reg);
2966 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2967 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2968 %}
2969
2970 // This encoding class is generated automatically from ad_encode.m4.
2971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2972 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2973 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2974 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2975 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2976 %}
2977
2978 // This encoding class is generated automatically from ad_encode.m4.
2979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2980 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2981 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2982 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2983 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2984 %}
2985
2986 // This encoding class is generated automatically from ad_encode.m4.
2987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2988 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2989 Register src_reg = as_Register($src$$reg);
2990 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2991 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2992 %}
2993
2994 // This encoding class is generated automatically from ad_encode.m4.
2995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2996 enc_class aarch64_enc_strb0(memory1 mem) %{
2997 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2998 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2999 %}
3000
3001 // This encoding class is generated automatically from ad_encode.m4.
3002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3003 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3004 Register src_reg = as_Register($src$$reg);
3005 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3006 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3007 %}
3008
3009 // This encoding class is generated automatically from ad_encode.m4.
3010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3011 enc_class aarch64_enc_strh0(memory2 mem) %{
3012 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3013 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3014 %}
3015
3016 // This encoding class is generated automatically from ad_encode.m4.
3017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3018 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3019 Register src_reg = as_Register($src$$reg);
3020 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3021 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3022 %}
3023
3024 // This encoding class is generated automatically from ad_encode.m4.
3025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3026 enc_class aarch64_enc_strw0(memory4 mem) %{
3027 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3028 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3029 %}
3030
3031 // This encoding class is generated automatically from ad_encode.m4.
3032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3033 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3034 Register src_reg = as_Register($src$$reg);
3035 // we sometimes get asked to store the stack pointer into the
3036 // current thread -- we cannot do that directly on AArch64
3037 if (src_reg == r31_sp) {
3038 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3039 __ mov(rscratch2, sp);
3040 src_reg = rscratch2;
3041 }
3042 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3043 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3044 %}
3045
3046 // This encoding class is generated automatically from ad_encode.m4.
3047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3048 enc_class aarch64_enc_str0(memory8 mem) %{
3049 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3050 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3051 %}
3052
3053 // This encoding class is generated automatically from ad_encode.m4.
3054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3055 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3056 FloatRegister src_reg = as_FloatRegister($src$$reg);
3057 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3058 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3059 %}
3060
3061 // This encoding class is generated automatically from ad_encode.m4.
3062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3063 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3064 FloatRegister src_reg = as_FloatRegister($src$$reg);
3065 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3066 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3067 %}
3068
3069 // This encoding class is generated automatically from ad_encode.m4.
3070 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3071 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3072 __ membar(Assembler::StoreStore);
3073 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3074 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3075 %}
3076
3077 // END Non-volatile memory access
3078
3079 // Vector loads and stores
3080 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3081 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3082 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3083 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3084 %}
3085
3086 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3087 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3088 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3089 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3090 %}
3091
3092 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3093 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3094 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3095 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3096 %}
3097
3098 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3099 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3100 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3101 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3102 %}
3103
3104 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3105 FloatRegister src_reg = as_FloatRegister($src$$reg);
3106 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3107 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3108 %}
3109
3110 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3111 FloatRegister src_reg = as_FloatRegister($src$$reg);
3112 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3113 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3114 %}
3115
3116 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3117 FloatRegister src_reg = as_FloatRegister($src$$reg);
3118 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3119 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3120 %}
3121
3122 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3123 FloatRegister src_reg = as_FloatRegister($src$$reg);
3124 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3125 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3126 %}
3127
3128 // volatile loads and stores
3129
3130 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3131 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3132 rscratch1, stlrb);
3133 %}
3134
3135 enc_class aarch64_enc_stlrb0(memory mem) %{
3136 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3137 rscratch1, stlrb);
3138 %}
3139
3140 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3141 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3142 rscratch1, stlrh);
3143 %}
3144
3145 enc_class aarch64_enc_stlrh0(memory mem) %{
3146 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3147 rscratch1, stlrh);
3148 %}
3149
3150 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3151 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3152 rscratch1, stlrw);
3153 %}
3154
3155 enc_class aarch64_enc_stlrw0(memory mem) %{
3156 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3157 rscratch1, stlrw);
3158 %}
3159
3160 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3161 Register dst_reg = as_Register($dst$$reg);
3162 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3163 rscratch1, ldarb);
3164 __ sxtbw(dst_reg, dst_reg);
3165 %}
3166
3167 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3168 Register dst_reg = as_Register($dst$$reg);
3169 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3170 rscratch1, ldarb);
3171 __ sxtb(dst_reg, dst_reg);
3172 %}
3173
3174 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3175 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3176 rscratch1, ldarb);
3177 %}
3178
3179 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3180 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, ldarb);
3182 %}
3183
3184 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3185 Register dst_reg = as_Register($dst$$reg);
3186 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3187 rscratch1, ldarh);
3188 __ sxthw(dst_reg, dst_reg);
3189 %}
3190
3191 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3192 Register dst_reg = as_Register($dst$$reg);
3193 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3194 rscratch1, ldarh);
3195 __ sxth(dst_reg, dst_reg);
3196 %}
3197
3198 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3199 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3200 rscratch1, ldarh);
3201 %}
3202
3203 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3204 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3205 rscratch1, ldarh);
3206 %}
3207
3208 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3209 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3210 rscratch1, ldarw);
3211 %}
3212
3213 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3214 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3215 rscratch1, ldarw);
3216 %}
3217
3218 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3219 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3220 rscratch1, ldar);
3221 %}
3222
3223 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3224 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3225 rscratch1, ldarw);
3226 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3227 %}
3228
3229 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3230 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3231 rscratch1, ldar);
3232 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3233 %}
3234
3235 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3236 Register src_reg = as_Register($src$$reg);
3237 // we sometimes get asked to store the stack pointer into the
3238 // current thread -- we cannot do that directly on AArch64
3239 if (src_reg == r31_sp) {
3240 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3241 __ mov(rscratch2, sp);
3242 src_reg = rscratch2;
3243 }
3244 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3245 rscratch1, stlr);
3246 %}
3247
3248 enc_class aarch64_enc_stlr0(memory mem) %{
3249 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3250 rscratch1, stlr);
3251 %}
3252
3253 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3254 {
3255 FloatRegister src_reg = as_FloatRegister($src$$reg);
3256 __ fmovs(rscratch2, src_reg);
3257 }
3258 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3259 rscratch1, stlrw);
3260 %}
3261
3262 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3263 {
3264 FloatRegister src_reg = as_FloatRegister($src$$reg);
3265 __ fmovd(rscratch2, src_reg);
3266 }
3267 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3268 rscratch1, stlr);
3269 %}
3270
3271 // synchronized read/update encodings
3272
3273 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3274 Register dst_reg = as_Register($dst$$reg);
3275 Register base = as_Register($mem$$base);
3276 int index = $mem$$index;
3277 int scale = $mem$$scale;
3278 int disp = $mem$$disp;
3279 if (index == -1) {
3280 if (disp != 0) {
3281 __ lea(rscratch1, Address(base, disp));
3282 __ ldaxr(dst_reg, rscratch1);
3283 } else {
3284 // TODO
3285 // should we ever get anything other than this case?
3286 __ ldaxr(dst_reg, base);
3287 }
3288 } else {
3289 Register index_reg = as_Register(index);
3290 if (disp == 0) {
3291 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3292 __ ldaxr(dst_reg, rscratch1);
3293 } else {
3294 __ lea(rscratch1, Address(base, disp));
3295 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3296 __ ldaxr(dst_reg, rscratch1);
3297 }
3298 }
3299 %}
3300
3301 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3302 Register src_reg = as_Register($src$$reg);
3303 Register base = as_Register($mem$$base);
3304 int index = $mem$$index;
3305 int scale = $mem$$scale;
3306 int disp = $mem$$disp;
3307 if (index == -1) {
3308 if (disp != 0) {
3309 __ lea(rscratch2, Address(base, disp));
3310 __ stlxr(rscratch1, src_reg, rscratch2);
3311 } else {
3312 // TODO
3313 // should we ever get anything other than this case?
3314 __ stlxr(rscratch1, src_reg, base);
3315 }
3316 } else {
3317 Register index_reg = as_Register(index);
3318 if (disp == 0) {
3319 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3320 __ stlxr(rscratch1, src_reg, rscratch2);
3321 } else {
3322 __ lea(rscratch2, Address(base, disp));
3323 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3324 __ stlxr(rscratch1, src_reg, rscratch2);
3325 }
3326 }
3327 __ cmpw(rscratch1, zr);
3328 %}
3329
3330 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3331 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3332 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3333 Assembler::xword, /*acquire*/ false, /*release*/ true,
3334 /*weak*/ false, noreg);
3335 %}
3336
3337 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3338 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3339 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3340 Assembler::word, /*acquire*/ false, /*release*/ true,
3341 /*weak*/ false, noreg);
3342 %}
3343
3344 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3345 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3346 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3347 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3348 /*weak*/ false, noreg);
3349 %}
3350
3351 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3352 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3353 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3354 Assembler::byte, /*acquire*/ false, /*release*/ true,
3355 /*weak*/ false, noreg);
3356 %}
3357
3358
3359 // The only difference between aarch64_enc_cmpxchg and
3360 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3361 // CompareAndSwap sequence to serve as a barrier on acquiring a
3362 // lock.
3363 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3364 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3365 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3366 Assembler::xword, /*acquire*/ true, /*release*/ true,
3367 /*weak*/ false, noreg);
3368 %}
3369
3370 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3371 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3372 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3373 Assembler::word, /*acquire*/ true, /*release*/ true,
3374 /*weak*/ false, noreg);
3375 %}
3376
3377 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3378 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3379 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3380 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3381 /*weak*/ false, noreg);
3382 %}
3383
3384 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3385 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3386 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3387 Assembler::byte, /*acquire*/ true, /*release*/ true,
3388 /*weak*/ false, noreg);
3389 %}
3390
3391 // auxiliary used for CompareAndSwapX to set result register
3392 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3393 Register res_reg = as_Register($res$$reg);
3394 __ cset(res_reg, Assembler::EQ);
3395 %}
3396
3397 // prefetch encodings
3398
3399 enc_class aarch64_enc_prefetchw(memory mem) %{
3400 Register base = as_Register($mem$$base);
3401 int index = $mem$$index;
3402 int scale = $mem$$scale;
3403 int disp = $mem$$disp;
3404 if (index == -1) {
3405 // Fix up any out-of-range offsets.
3406 assert_different_registers(rscratch1, base);
3407 Address addr = Address(base, disp);
3408 addr = __ legitimize_address(addr, 8, rscratch1);
3409 __ prfm(addr, PSTL1KEEP);
3410 } else {
3411 Register index_reg = as_Register(index);
3412 if (disp == 0) {
3413 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3414 } else {
3415 __ lea(rscratch1, Address(base, disp));
3416 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3417 }
3418 }
3419 %}
3420
3421 // mov encodings
3422
3423 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3424 uint32_t con = (uint32_t)$src$$constant;
3425 Register dst_reg = as_Register($dst$$reg);
3426 if (con == 0) {
3427 __ movw(dst_reg, zr);
3428 } else {
3429 __ movw(dst_reg, con);
3430 }
3431 %}
3432
3433 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3434 Register dst_reg = as_Register($dst$$reg);
3435 uint64_t con = (uint64_t)$src$$constant;
3436 if (con == 0) {
3437 __ mov(dst_reg, zr);
3438 } else {
3439 __ mov(dst_reg, con);
3440 }
3441 %}
3442
3443 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3444 Register dst_reg = as_Register($dst$$reg);
3445 address con = (address)$src$$constant;
3446 if (con == nullptr || con == (address)1) {
3447 ShouldNotReachHere();
3448 } else {
3449 relocInfo::relocType rtype = $src->constant_reloc();
3450 if (rtype == relocInfo::oop_type) {
3451 __ movoop(dst_reg, (jobject)con);
3452 } else if (rtype == relocInfo::metadata_type) {
3453 __ mov_metadata(dst_reg, (Metadata*)con);
3454 } else {
3455 assert(rtype == relocInfo::none, "unexpected reloc type");
3456 if (! __ is_valid_AArch64_address(con) ||
3457 con < (address)(uintptr_t)os::vm_page_size()) {
3458 __ mov(dst_reg, con);
3459 } else {
3460 uint64_t offset;
3461 __ adrp(dst_reg, con, offset);
3462 __ add(dst_reg, dst_reg, offset);
3463 }
3464 }
3465 }
3466 %}
3467
3468 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3469 Register dst_reg = as_Register($dst$$reg);
3470 __ mov(dst_reg, zr);
3471 %}
3472
3473 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3474 Register dst_reg = as_Register($dst$$reg);
3475 __ mov(dst_reg, (uint64_t)1);
3476 %}
3477
3478 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3479 Register dst_reg = as_Register($dst$$reg);
3480 address con = (address)$src$$constant;
3481 if (con == nullptr) {
3482 ShouldNotReachHere();
3483 } else {
3484 relocInfo::relocType rtype = $src->constant_reloc();
3485 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3486 __ set_narrow_oop(dst_reg, (jobject)con);
3487 }
3488 %}
3489
3490 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3491 Register dst_reg = as_Register($dst$$reg);
3492 __ mov(dst_reg, zr);
3493 %}
3494
3495 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3496 Register dst_reg = as_Register($dst$$reg);
3497 address con = (address)$src$$constant;
3498 if (con == nullptr) {
3499 ShouldNotReachHere();
3500 } else {
3501 relocInfo::relocType rtype = $src->constant_reloc();
3502 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3503 __ set_narrow_klass(dst_reg, (Klass *)con);
3504 }
3505 %}
3506
3507 // arithmetic encodings
3508
3509 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3510 Register dst_reg = as_Register($dst$$reg);
3511 Register src_reg = as_Register($src1$$reg);
3512 int32_t con = (int32_t)$src2$$constant;
3513 // add has primary == 0, subtract has primary == 1
3514 if ($primary) { con = -con; }
3515 if (con < 0) {
3516 __ subw(dst_reg, src_reg, -con);
3517 } else {
3518 __ addw(dst_reg, src_reg, con);
3519 }
3520 %}
3521
3522 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3523 Register dst_reg = as_Register($dst$$reg);
3524 Register src_reg = as_Register($src1$$reg);
3525 int32_t con = (int32_t)$src2$$constant;
3526 // add has primary == 0, subtract has primary == 1
3527 if ($primary) { con = -con; }
3528 if (con < 0) {
3529 __ sub(dst_reg, src_reg, -con);
3530 } else {
3531 __ add(dst_reg, src_reg, con);
3532 }
3533 %}
3534
3535 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3536 Register dst_reg = as_Register($dst$$reg);
3537 Register src1_reg = as_Register($src1$$reg);
3538 Register src2_reg = as_Register($src2$$reg);
3539 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3540 %}
3541
3542 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3543 Register dst_reg = as_Register($dst$$reg);
3544 Register src1_reg = as_Register($src1$$reg);
3545 Register src2_reg = as_Register($src2$$reg);
3546 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3547 %}
3548
3549 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3550 Register dst_reg = as_Register($dst$$reg);
3551 Register src1_reg = as_Register($src1$$reg);
3552 Register src2_reg = as_Register($src2$$reg);
3553 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3554 %}
3555
3556 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3557 Register dst_reg = as_Register($dst$$reg);
3558 Register src1_reg = as_Register($src1$$reg);
3559 Register src2_reg = as_Register($src2$$reg);
3560 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3561 %}
3562
3563 // compare instruction encodings
3564
3565 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3566 Register reg1 = as_Register($src1$$reg);
3567 Register reg2 = as_Register($src2$$reg);
3568 __ cmpw(reg1, reg2);
3569 %}
3570
3571 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3572 Register reg = as_Register($src1$$reg);
3573 int32_t val = $src2$$constant;
3574 if (val >= 0) {
3575 __ subsw(zr, reg, val);
3576 } else {
3577 __ addsw(zr, reg, -val);
3578 }
3579 %}
3580
3581 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3582 Register reg1 = as_Register($src1$$reg);
3583 uint32_t val = (uint32_t)$src2$$constant;
3584 __ movw(rscratch1, val);
3585 __ cmpw(reg1, rscratch1);
3586 %}
3587
3588 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3589 Register reg1 = as_Register($src1$$reg);
3590 Register reg2 = as_Register($src2$$reg);
3591 __ cmp(reg1, reg2);
3592 %}
3593
3594 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3595 Register reg = as_Register($src1$$reg);
3596 int64_t val = $src2$$constant;
3597 if (val >= 0) {
3598 __ subs(zr, reg, val);
3599 } else if (val != -val) {
3600 __ adds(zr, reg, -val);
3601 } else {
3602 // aargh, Long.MIN_VALUE is a special case
3603 __ orr(rscratch1, zr, (uint64_t)val);
3604 __ subs(zr, reg, rscratch1);
3605 }
3606 %}
3607
3608 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3609 Register reg1 = as_Register($src1$$reg);
3610 uint64_t val = (uint64_t)$src2$$constant;
3611 __ mov(rscratch1, val);
3612 __ cmp(reg1, rscratch1);
3613 %}
3614
3615 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3616 Register reg1 = as_Register($src1$$reg);
3617 Register reg2 = as_Register($src2$$reg);
3618 __ cmp(reg1, reg2);
3619 %}
3620
3621 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3622 Register reg1 = as_Register($src1$$reg);
3623 Register reg2 = as_Register($src2$$reg);
3624 __ cmpw(reg1, reg2);
3625 %}
3626
3627 enc_class aarch64_enc_testp(iRegP src) %{
3628 Register reg = as_Register($src$$reg);
3629 __ cmp(reg, zr);
3630 %}
3631
3632 enc_class aarch64_enc_testn(iRegN src) %{
3633 Register reg = as_Register($src$$reg);
3634 __ cmpw(reg, zr);
3635 %}
3636
3637 enc_class aarch64_enc_b(label lbl) %{
3638 Label *L = $lbl$$label;
3639 __ b(*L);
3640 %}
3641
3642 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3643 Label *L = $lbl$$label;
3644 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3645 %}
3646
3647 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3648 Label *L = $lbl$$label;
3649 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3650 %}
3651
3652 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3653 %{
3654 Register sub_reg = as_Register($sub$$reg);
3655 Register super_reg = as_Register($super$$reg);
3656 Register temp_reg = as_Register($temp$$reg);
3657 Register result_reg = as_Register($result$$reg);
3658
3659 Label miss;
3660 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3661 nullptr, &miss,
3662 /*set_cond_codes:*/ true);
3663 if ($primary) {
3664 __ mov(result_reg, zr);
3665 }
3666 __ bind(miss);
3667 %}
3668
3669 enc_class aarch64_enc_java_static_call(method meth) %{
3670 address addr = (address)$meth$$method;
3671 address call;
3672 if (!_method) {
3673 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3674 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3675 if (call == nullptr) {
3676 ciEnv::current()->record_failure("CodeCache is full");
3677 return;
3678 }
3679 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3680 // The NOP here is purely to ensure that eliding a call to
3681 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3682 __ nop();
3683 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3684 } else {
3685 int method_index = resolved_method_index(masm);
3686 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3687 : static_call_Relocation::spec(method_index);
3688 call = __ trampoline_call(Address(addr, rspec));
3689 if (call == nullptr) {
3690 ciEnv::current()->record_failure("CodeCache is full");
3691 return;
3692 }
3693 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3694 // Calls of the same statically bound method can share
3695 // a stub to the interpreter.
3696 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3697 } else {
3698 // Emit stub for static call
3699 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3700 if (stub == nullptr) {
3701 ciEnv::current()->record_failure("CodeCache is full");
3702 return;
3703 }
3704 }
3705 }
3706
3707 __ post_call_nop();
3708
3709 // Only non uncommon_trap calls need to reinitialize ptrue.
3710 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3711 __ reinitialize_ptrue();
3712 }
3713 %}
3714
3715 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3716 int method_index = resolved_method_index(masm);
3717 address call = __ ic_call((address)$meth$$method, method_index);
3718 if (call == nullptr) {
3719 ciEnv::current()->record_failure("CodeCache is full");
3720 return;
3721 }
3722 __ post_call_nop();
3723 if (Compile::current()->max_vector_size() > 0) {
3724 __ reinitialize_ptrue();
3725 }
3726 %}
3727
3728 enc_class aarch64_enc_call_epilog() %{
3729 if (VerifyStackAtCalls) {
3730 // Check that stack depth is unchanged: find majik cookie on stack
3731 __ call_Unimplemented();
3732 }
3733 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
3734 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3735 // Search for the corresponding projection, get the register and emit code that initialized it.
3736 uint con = (tf()->range_cc()->cnt() - 1);
3737 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3738 ProjNode* proj = fast_out(i)->as_Proj();
3739 if (proj->_con == con) {
3740 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3741 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3742 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3743 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3744 __ cmp(r0, zr);
3745 __ cset(toReg, Assembler::NE);
3746 if (reg->is_stack()) {
3747 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3748 __ str(toReg, Address(sp, st_off));
3749 }
3750 break;
3751 }
3752 }
3753 if (return_value_is_used()) {
3754 // An inline type is returned as fields in multiple registers.
3755 // R0 either contains an oop if the inline type is buffered or a pointer
3756 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3757 // if the lowest bit is set to allow C2 to use the oop after null checking.
3758 // r0 &= (r0 & 1) - 1
3759 __ andr(rscratch1, r0, 0x1);
3760 __ sub(rscratch1, rscratch1, 0x1);
3761 __ andr(r0, r0, rscratch1);
3762 }
3763 }
3764 %}
3765
3766 enc_class aarch64_enc_java_to_runtime(method meth) %{
3767 // some calls to generated routines (arraycopy code) are scheduled
3768 // by C2 as runtime calls. if so we can call them using a br (they
3769 // will be in a reachable segment) otherwise we have to use a blr
3770 // which loads the absolute address into a register.
3771 address entry = (address)$meth$$method;
3772 CodeBlob *cb = CodeCache::find_blob(entry);
3773 if (cb) {
3774 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3775 if (call == nullptr) {
3776 ciEnv::current()->record_failure("CodeCache is full");
3777 return;
3778 }
3779 __ post_call_nop();
3780 } else {
3781 Label retaddr;
3782 // Make the anchor frame walkable
3783 __ adr(rscratch2, retaddr);
3784 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3785 __ lea(rscratch1, RuntimeAddress(entry));
3786 __ blr(rscratch1);
3787 __ bind(retaddr);
3788 __ post_call_nop();
3789 }
3790 if (Compile::current()->max_vector_size() > 0) {
3791 __ reinitialize_ptrue();
3792 }
3793 %}
3794
3795 enc_class aarch64_enc_rethrow() %{
3796 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3797 %}
3798
3799 enc_class aarch64_enc_ret() %{
3800 #ifdef ASSERT
3801 if (Compile::current()->max_vector_size() > 0) {
3802 __ verify_ptrue();
3803 }
3804 #endif
3805 __ ret(lr);
3806 %}
3807
3808 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3809 Register target_reg = as_Register($jump_target$$reg);
3810 __ br(target_reg);
3811 %}
3812
3813 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3814 Register target_reg = as_Register($jump_target$$reg);
3815 // exception oop should be in r0
3816 // ret addr has been popped into lr
3817 // callee expects it in r3
3818 __ mov(r3, lr);
3819 __ br(target_reg);
3820 %}
3821
3822 %}
3823
3824 //----------FRAME--------------------------------------------------------------
3825 // Definition of frame structure and management information.
3826 //
3827 // S T A C K L A Y O U T Allocators stack-slot number
3828 // | (to get allocators register number
3829 // G Owned by | | v add OptoReg::stack0())
3830 // r CALLER | |
3831 // o | +--------+ pad to even-align allocators stack-slot
3832 // w V | pad0 | numbers; owned by CALLER
3833 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3834 // h ^ | in | 5
3835 // | | args | 4 Holes in incoming args owned by SELF
3836 // | | | | 3
3837 // | | +--------+
3838 // V | | old out| Empty on Intel, window on Sparc
3839 // | old |preserve| Must be even aligned.
3840 // | SP-+--------+----> Matcher::_old_SP, even aligned
3841 // | | in | 3 area for Intel ret address
3842 // Owned by |preserve| Empty on Sparc.
3843 // SELF +--------+
3844 // | | pad2 | 2 pad to align old SP
3845 // | +--------+ 1
3846 // | | locks | 0
3847 // | +--------+----> OptoReg::stack0(), even aligned
3848 // | | pad1 | 11 pad to align new SP
3849 // | +--------+
3850 // | | | 10
3851 // | | spills | 9 spills
3852 // V | | 8 (pad0 slot for callee)
3853 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3854 // ^ | out | 7
3855 // | | args | 6 Holes in outgoing args owned by CALLEE
3856 // Owned by +--------+
3857 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3858 // | new |preserve| Must be even-aligned.
3859 // | SP-+--------+----> Matcher::_new_SP, even aligned
3860 // | | |
3861 //
3862 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3863 // known from SELF's arguments and the Java calling convention.
3864 // Region 6-7 is determined per call site.
3865 // Note 2: If the calling convention leaves holes in the incoming argument
3866 // area, those holes are owned by SELF. Holes in the outgoing area
3867 // are owned by the CALLEE. Holes should not be necessary in the
3868 // incoming area, as the Java calling convention is completely under
3869 // the control of the AD file. Doubles can be sorted and packed to
3870 // avoid holes. Holes in the outgoing arguments may be necessary for
3871 // varargs C calling conventions.
3872 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3873 // even aligned with pad0 as needed.
3874 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3875 // (the latter is true on Intel but is it false on AArch64?)
3876 // region 6-11 is even aligned; it may be padded out more so that
3877 // the region from SP to FP meets the minimum stack alignment.
3878 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3879 // alignment. Region 11, pad1, may be dynamically extended so that
3880 // SP meets the minimum alignment.
3881
3882 frame %{
3883 // These three registers define part of the calling convention
3884 // between compiled code and the interpreter.
3885
3886 // Inline Cache Register or Method for I2C.
3887 inline_cache_reg(R12);
3888
3889 // Number of stack slots consumed by locking an object
3890 sync_stack_slots(2);
3891
3892 // Compiled code's Frame Pointer
3893 frame_pointer(R31);
3894
3895 // Interpreter stores its frame pointer in a register which is
3896 // stored to the stack by I2CAdaptors.
3897 // I2CAdaptors convert from interpreted java to compiled java.
3898 interpreter_frame_pointer(R29);
3899
3900 // Stack alignment requirement
3901 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3902
3903 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3904 // for calls to C. Supports the var-args backing area for register parms.
3905 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3906
3907 // The after-PROLOG location of the return address. Location of
3908 // return address specifies a type (REG or STACK) and a number
3909 // representing the register number (i.e. - use a register name) or
3910 // stack slot.
3911 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3912 // Otherwise, it is above the locks and verification slot and alignment word
3913 // TODO this may well be correct but need to check why that - 2 is there
3914 // ppc port uses 0 but we definitely need to allow for fixed_slots
3915 // which folds in the space used for monitors
3916 return_addr(STACK - 2 +
3917 align_up((Compile::current()->in_preserve_stack_slots() +
3918 Compile::current()->fixed_slots()),
3919 stack_alignment_in_slots()));
3920
3921 // Location of compiled Java return values. Same as C for now.
3922 return_value
3923 %{
3924 // TODO do we allow ideal_reg == Op_RegN???
3925 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3926 "only return normal values");
3927
3928 static const int lo[Op_RegL + 1] = { // enum name
3929 0, // Op_Node
3930 0, // Op_Set
3931 R0_num, // Op_RegN
3932 R0_num, // Op_RegI
3933 R0_num, // Op_RegP
3934 V0_num, // Op_RegF
3935 V0_num, // Op_RegD
3936 R0_num // Op_RegL
3937 };
3938
3939 static const int hi[Op_RegL + 1] = { // enum name
3940 0, // Op_Node
3941 0, // Op_Set
3942 OptoReg::Bad, // Op_RegN
3943 OptoReg::Bad, // Op_RegI
3944 R0_H_num, // Op_RegP
3945 OptoReg::Bad, // Op_RegF
3946 V0_H_num, // Op_RegD
3947 R0_H_num // Op_RegL
3948 };
3949
3950 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3951 %}
3952 %}
3953
3954 //----------ATTRIBUTES---------------------------------------------------------
3955 //----------Operand Attributes-------------------------------------------------
3956 op_attrib op_cost(1); // Required cost attribute
3957
3958 //----------Instruction Attributes---------------------------------------------
3959 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3960 ins_attrib ins_size(32); // Required size attribute (in bits)
3961 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3962 // a non-matching short branch variant
3963 // of some long branch?
3964 ins_attrib ins_alignment(4); // Required alignment attribute (must
3965 // be a power of 2) specifies the
3966 // alignment that some part of the
3967 // instruction (not necessarily the
3968 // start) requires. If > 1, a
3969 // compute_padding() function must be
3970 // provided for the instruction
3971
3972 // Whether this node is expanded during code emission into a sequence of
3973 // instructions and the first instruction can perform an implicit null check.
3974 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3975
3976 //----------OPERANDS-----------------------------------------------------------
3977 // Operand definitions must precede instruction definitions for correct parsing
3978 // in the ADLC because operands constitute user defined types which are used in
3979 // instruction definitions.
3980
3981 //----------Simple Operands----------------------------------------------------
3982
3983 // Integer operands 32 bit
3984 // 32 bit immediate
3985 operand immI()
3986 %{
3987 match(ConI);
3988
3989 op_cost(0);
3990 format %{ %}
3991 interface(CONST_INTER);
3992 %}
3993
3994 // 32 bit zero
3995 operand immI0()
3996 %{
3997 predicate(n->get_int() == 0);
3998 match(ConI);
3999
4000 op_cost(0);
4001 format %{ %}
4002 interface(CONST_INTER);
4003 %}
4004
4005 // 32 bit unit increment
4006 operand immI_1()
4007 %{
4008 predicate(n->get_int() == 1);
4009 match(ConI);
4010
4011 op_cost(0);
4012 format %{ %}
4013 interface(CONST_INTER);
4014 %}
4015
4016 // 32 bit unit decrement
4017 operand immI_M1()
4018 %{
4019 predicate(n->get_int() == -1);
4020 match(ConI);
4021
4022 op_cost(0);
4023 format %{ %}
4024 interface(CONST_INTER);
4025 %}
4026
4027 // Shift values for add/sub extension shift
4028 operand immIExt()
4029 %{
4030 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4031 match(ConI);
4032
4033 op_cost(0);
4034 format %{ %}
4035 interface(CONST_INTER);
4036 %}
4037
4038 operand immI_gt_1()
4039 %{
4040 predicate(n->get_int() > 1);
4041 match(ConI);
4042
4043 op_cost(0);
4044 format %{ %}
4045 interface(CONST_INTER);
4046 %}
4047
4048 operand immI_le_4()
4049 %{
4050 predicate(n->get_int() <= 4);
4051 match(ConI);
4052
4053 op_cost(0);
4054 format %{ %}
4055 interface(CONST_INTER);
4056 %}
4057
4058 operand immI_16()
4059 %{
4060 predicate(n->get_int() == 16);
4061 match(ConI);
4062
4063 op_cost(0);
4064 format %{ %}
4065 interface(CONST_INTER);
4066 %}
4067
4068 operand immI_24()
4069 %{
4070 predicate(n->get_int() == 24);
4071 match(ConI);
4072
4073 op_cost(0);
4074 format %{ %}
4075 interface(CONST_INTER);
4076 %}
4077
4078 operand immI_32()
4079 %{
4080 predicate(n->get_int() == 32);
4081 match(ConI);
4082
4083 op_cost(0);
4084 format %{ %}
4085 interface(CONST_INTER);
4086 %}
4087
4088 operand immI_48()
4089 %{
4090 predicate(n->get_int() == 48);
4091 match(ConI);
4092
4093 op_cost(0);
4094 format %{ %}
4095 interface(CONST_INTER);
4096 %}
4097
4098 operand immI_56()
4099 %{
4100 predicate(n->get_int() == 56);
4101 match(ConI);
4102
4103 op_cost(0);
4104 format %{ %}
4105 interface(CONST_INTER);
4106 %}
4107
4108 operand immI_255()
4109 %{
4110 predicate(n->get_int() == 255);
4111 match(ConI);
4112
4113 op_cost(0);
4114 format %{ %}
4115 interface(CONST_INTER);
4116 %}
4117
4118 operand immI_65535()
4119 %{
4120 predicate(n->get_int() == 65535);
4121 match(ConI);
4122
4123 op_cost(0);
4124 format %{ %}
4125 interface(CONST_INTER);
4126 %}
4127
4128 operand immI_positive()
4129 %{
4130 predicate(n->get_int() > 0);
4131 match(ConI);
4132
4133 op_cost(0);
4134 format %{ %}
4135 interface(CONST_INTER);
4136 %}
4137
4138 // BoolTest condition for signed compare
4139 operand immI_cmp_cond()
4140 %{
4141 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4142 match(ConI);
4143
4144 op_cost(0);
4145 format %{ %}
4146 interface(CONST_INTER);
4147 %}
4148
4149 // BoolTest condition for unsigned compare
4150 operand immI_cmpU_cond()
4151 %{
4152 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4153 match(ConI);
4154
4155 op_cost(0);
4156 format %{ %}
4157 interface(CONST_INTER);
4158 %}
4159
4160 operand immL_255()
4161 %{
4162 predicate(n->get_long() == 255L);
4163 match(ConL);
4164
4165 op_cost(0);
4166 format %{ %}
4167 interface(CONST_INTER);
4168 %}
4169
4170 operand immL_65535()
4171 %{
4172 predicate(n->get_long() == 65535L);
4173 match(ConL);
4174
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 operand immL_4294967295()
4181 %{
4182 predicate(n->get_long() == 4294967295L);
4183 match(ConL);
4184
4185 op_cost(0);
4186 format %{ %}
4187 interface(CONST_INTER);
4188 %}
4189
4190 operand immL_bitmask()
4191 %{
4192 predicate((n->get_long() != 0)
4193 && ((n->get_long() & 0xc000000000000000l) == 0)
4194 && is_power_of_2(n->get_long() + 1));
4195 match(ConL);
4196
4197 op_cost(0);
4198 format %{ %}
4199 interface(CONST_INTER);
4200 %}
4201
4202 operand immI_bitmask()
4203 %{
4204 predicate((n->get_int() != 0)
4205 && ((n->get_int() & 0xc0000000) == 0)
4206 && is_power_of_2(n->get_int() + 1));
4207 match(ConI);
4208
4209 op_cost(0);
4210 format %{ %}
4211 interface(CONST_INTER);
4212 %}
4213
4214 operand immL_positive_bitmaskI()
4215 %{
4216 predicate((n->get_long() != 0)
4217 && ((julong)n->get_long() < 0x80000000ULL)
4218 && is_power_of_2(n->get_long() + 1));
4219 match(ConL);
4220
4221 op_cost(0);
4222 format %{ %}
4223 interface(CONST_INTER);
4224 %}
4225
4226 // Scale values for scaled offset addressing modes (up to long but not quad)
4227 operand immIScale()
4228 %{
4229 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4230 match(ConI);
4231
4232 op_cost(0);
4233 format %{ %}
4234 interface(CONST_INTER);
4235 %}
4236
4237 // 5 bit signed integer
4238 operand immI5()
4239 %{
4240 predicate(Assembler::is_simm(n->get_int(), 5));
4241 match(ConI);
4242
4243 op_cost(0);
4244 format %{ %}
4245 interface(CONST_INTER);
4246 %}
4247
4248 // 7 bit unsigned integer
4249 operand immIU7()
4250 %{
4251 predicate(Assembler::is_uimm(n->get_int(), 7));
4252 match(ConI);
4253
4254 op_cost(0);
4255 format %{ %}
4256 interface(CONST_INTER);
4257 %}
4258
4259 // Offset for scaled or unscaled immediate loads and stores
4260 operand immIOffset()
4261 %{
4262 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4263 match(ConI);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immIOffset1()
4271 %{
4272 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4273 match(ConI);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immIOffset2()
4281 %{
4282 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4283 match(ConI);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 operand immIOffset4()
4291 %{
4292 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4293 match(ConI);
4294
4295 op_cost(0);
4296 format %{ %}
4297 interface(CONST_INTER);
4298 %}
4299
4300 operand immIOffset8()
4301 %{
4302 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4303 match(ConI);
4304
4305 op_cost(0);
4306 format %{ %}
4307 interface(CONST_INTER);
4308 %}
4309
4310 operand immIOffset16()
4311 %{
4312 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4313 match(ConI);
4314
4315 op_cost(0);
4316 format %{ %}
4317 interface(CONST_INTER);
4318 %}
4319
4320 operand immLOffset()
4321 %{
4322 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4323 match(ConL);
4324
4325 op_cost(0);
4326 format %{ %}
4327 interface(CONST_INTER);
4328 %}
4329
4330 operand immLoffset1()
4331 %{
4332 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4333 match(ConL);
4334
4335 op_cost(0);
4336 format %{ %}
4337 interface(CONST_INTER);
4338 %}
4339
4340 operand immLoffset2()
4341 %{
4342 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4343 match(ConL);
4344
4345 op_cost(0);
4346 format %{ %}
4347 interface(CONST_INTER);
4348 %}
4349
4350 operand immLoffset4()
4351 %{
4352 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4353 match(ConL);
4354
4355 op_cost(0);
4356 format %{ %}
4357 interface(CONST_INTER);
4358 %}
4359
4360 operand immLoffset8()
4361 %{
4362 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4363 match(ConL);
4364
4365 op_cost(0);
4366 format %{ %}
4367 interface(CONST_INTER);
4368 %}
4369
4370 operand immLoffset16()
4371 %{
4372 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4373 match(ConL);
4374
4375 op_cost(0);
4376 format %{ %}
4377 interface(CONST_INTER);
4378 %}
4379
4380 // 5 bit signed long integer
4381 operand immL5()
4382 %{
4383 predicate(Assembler::is_simm(n->get_long(), 5));
4384 match(ConL);
4385
4386 op_cost(0);
4387 format %{ %}
4388 interface(CONST_INTER);
4389 %}
4390
4391 // 7 bit unsigned long integer
4392 operand immLU7()
4393 %{
4394 predicate(Assembler::is_uimm(n->get_long(), 7));
4395 match(ConL);
4396
4397 op_cost(0);
4398 format %{ %}
4399 interface(CONST_INTER);
4400 %}
4401
4402 // 8 bit signed value.
4403 operand immI8()
4404 %{
4405 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4406 match(ConI);
4407
4408 op_cost(0);
4409 format %{ %}
4410 interface(CONST_INTER);
4411 %}
4412
4413 // 8 bit signed value (simm8), or #simm8 LSL 8.
4414 operand immIDupV()
4415 %{
4416 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4417 match(ConI);
4418
4419 op_cost(0);
4420 format %{ %}
4421 interface(CONST_INTER);
4422 %}
4423
4424 // 8 bit signed value (simm8), or #simm8 LSL 8.
4425 operand immLDupV()
4426 %{
4427 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4428 match(ConL);
4429
4430 op_cost(0);
4431 format %{ %}
4432 interface(CONST_INTER);
4433 %}
4434
4435 // 8 bit signed value (simm8), or #simm8 LSL 8.
4436 operand immHDupV()
4437 %{
4438 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4439 match(ConH);
4440
4441 op_cost(0);
4442 format %{ %}
4443 interface(CONST_INTER);
4444 %}
4445
4446 // 8 bit integer valid for vector add sub immediate
4447 operand immBAddSubV()
4448 %{
4449 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4450 match(ConI);
4451
4452 op_cost(0);
4453 format %{ %}
4454 interface(CONST_INTER);
4455 %}
4456
4457 // 32 bit integer valid for add sub immediate
4458 operand immIAddSub()
4459 %{
4460 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4461 match(ConI);
4462 op_cost(0);
4463 format %{ %}
4464 interface(CONST_INTER);
4465 %}
4466
4467 // 32 bit integer valid for vector add sub immediate
4468 operand immIAddSubV()
4469 %{
4470 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4471 match(ConI);
4472
4473 op_cost(0);
4474 format %{ %}
4475 interface(CONST_INTER);
4476 %}
4477
4478 // 32 bit unsigned integer valid for logical immediate
4479
4480 operand immBLog()
4481 %{
4482 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4483 match(ConI);
4484
4485 op_cost(0);
4486 format %{ %}
4487 interface(CONST_INTER);
4488 %}
4489
4490 operand immSLog()
4491 %{
4492 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4493 match(ConI);
4494
4495 op_cost(0);
4496 format %{ %}
4497 interface(CONST_INTER);
4498 %}
4499
4500 operand immILog()
4501 %{
4502 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4503 match(ConI);
4504
4505 op_cost(0);
4506 format %{ %}
4507 interface(CONST_INTER);
4508 %}
4509
4510 // Integer operands 64 bit
4511 // 64 bit immediate
4512 operand immL()
4513 %{
4514 match(ConL);
4515
4516 op_cost(0);
4517 format %{ %}
4518 interface(CONST_INTER);
4519 %}
4520
4521 // 64 bit zero
4522 operand immL0()
4523 %{
4524 predicate(n->get_long() == 0);
4525 match(ConL);
4526
4527 op_cost(0);
4528 format %{ %}
4529 interface(CONST_INTER);
4530 %}
4531
4532 // 64 bit unit decrement
4533 operand immL_M1()
4534 %{
4535 predicate(n->get_long() == -1);
4536 match(ConL);
4537
4538 op_cost(0);
4539 format %{ %}
4540 interface(CONST_INTER);
4541 %}
4542
4543 // 64 bit integer valid for add sub immediate
4544 operand immLAddSub()
4545 %{
4546 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4547 match(ConL);
4548 op_cost(0);
4549 format %{ %}
4550 interface(CONST_INTER);
4551 %}
4552
4553 // 64 bit integer valid for addv subv immediate
4554 operand immLAddSubV()
4555 %{
4556 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4557 match(ConL);
4558
4559 op_cost(0);
4560 format %{ %}
4561 interface(CONST_INTER);
4562 %}
4563
4564 // 64 bit integer valid for logical immediate
4565 operand immLLog()
4566 %{
4567 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4568 match(ConL);
4569 op_cost(0);
4570 format %{ %}
4571 interface(CONST_INTER);
4572 %}
4573
4574 // Long Immediate: low 32-bit mask
4575 operand immL_32bits()
4576 %{
4577 predicate(n->get_long() == 0xFFFFFFFFL);
4578 match(ConL);
4579 op_cost(0);
4580 format %{ %}
4581 interface(CONST_INTER);
4582 %}
4583
4584 // Pointer operands
4585 // Pointer Immediate
4586 operand immP()
4587 %{
4588 match(ConP);
4589
4590 op_cost(0);
4591 format %{ %}
4592 interface(CONST_INTER);
4593 %}
4594
4595 // nullptr Pointer Immediate
4596 operand immP0()
4597 %{
4598 predicate(n->get_ptr() == 0);
4599 match(ConP);
4600
4601 op_cost(0);
4602 format %{ %}
4603 interface(CONST_INTER);
4604 %}
4605
4606 // Pointer Immediate One
4607 // this is used in object initialization (initial object header)
4608 operand immP_1()
4609 %{
4610 predicate(n->get_ptr() == 1);
4611 match(ConP);
4612
4613 op_cost(0);
4614 format %{ %}
4615 interface(CONST_INTER);
4616 %}
4617
4618 // Float and Double operands
4619 // Double Immediate
4620 operand immD()
4621 %{
4622 match(ConD);
4623 op_cost(0);
4624 format %{ %}
4625 interface(CONST_INTER);
4626 %}
4627
4628 // Double Immediate: +0.0d
4629 operand immD0()
4630 %{
4631 predicate(jlong_cast(n->getd()) == 0);
4632 match(ConD);
4633
4634 op_cost(0);
4635 format %{ %}
4636 interface(CONST_INTER);
4637 %}
4638
4639 // constant 'double +0.0'.
4640 operand immDPacked()
4641 %{
4642 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4643 match(ConD);
4644 op_cost(0);
4645 format %{ %}
4646 interface(CONST_INTER);
4647 %}
4648
4649 // Float Immediate
4650 operand immF()
4651 %{
4652 match(ConF);
4653 op_cost(0);
4654 format %{ %}
4655 interface(CONST_INTER);
4656 %}
4657
4658 // Float Immediate: +0.0f.
4659 operand immF0()
4660 %{
4661 predicate(jint_cast(n->getf()) == 0);
4662 match(ConF);
4663
4664 op_cost(0);
4665 format %{ %}
4666 interface(CONST_INTER);
4667 %}
4668
4669 // Half Float (FP16) Immediate
4670 operand immH()
4671 %{
4672 match(ConH);
4673 op_cost(0);
4674 format %{ %}
4675 interface(CONST_INTER);
4676 %}
4677
4678 //
4679 operand immFPacked()
4680 %{
4681 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4682 match(ConF);
4683 op_cost(0);
4684 format %{ %}
4685 interface(CONST_INTER);
4686 %}
4687
4688 // Narrow pointer operands
4689 // Narrow Pointer Immediate
4690 operand immN()
4691 %{
4692 match(ConN);
4693
4694 op_cost(0);
4695 format %{ %}
4696 interface(CONST_INTER);
4697 %}
4698
4699 // Narrow nullptr Pointer Immediate
4700 operand immN0()
4701 %{
4702 predicate(n->get_narrowcon() == 0);
4703 match(ConN);
4704
4705 op_cost(0);
4706 format %{ %}
4707 interface(CONST_INTER);
4708 %}
4709
4710 operand immNKlass()
4711 %{
4712 match(ConNKlass);
4713
4714 op_cost(0);
4715 format %{ %}
4716 interface(CONST_INTER);
4717 %}
4718
4719 // Integer 32 bit Register Operands
4720 // Integer 32 bitRegister (excludes SP)
4721 operand iRegI()
4722 %{
4723 constraint(ALLOC_IN_RC(any_reg32));
4724 match(RegI);
4725 match(iRegINoSp);
4726 op_cost(0);
4727 format %{ %}
4728 interface(REG_INTER);
4729 %}
4730
4731 // Integer 32 bit Register not Special
4732 operand iRegINoSp()
4733 %{
4734 constraint(ALLOC_IN_RC(no_special_reg32));
4735 match(RegI);
4736 op_cost(0);
4737 format %{ %}
4738 interface(REG_INTER);
4739 %}
4740
4741 // Integer 64 bit Register Operands
4742 // Integer 64 bit Register (includes SP)
4743 operand iRegL()
4744 %{
4745 constraint(ALLOC_IN_RC(any_reg));
4746 match(RegL);
4747 match(iRegLNoSp);
4748 op_cost(0);
4749 format %{ %}
4750 interface(REG_INTER);
4751 %}
4752
4753 // Integer 64 bit Register not Special
4754 operand iRegLNoSp()
4755 %{
4756 constraint(ALLOC_IN_RC(no_special_reg));
4757 match(RegL);
4758 match(iRegL_R0);
4759 format %{ %}
4760 interface(REG_INTER);
4761 %}
4762
4763 // Pointer Register Operands
4764 // Pointer Register
4765 operand iRegP()
4766 %{
4767 constraint(ALLOC_IN_RC(ptr_reg));
4768 match(RegP);
4769 match(iRegPNoSp);
4770 match(iRegP_R0);
4771 //match(iRegP_R2);
4772 //match(iRegP_R4);
4773 match(iRegP_R5);
4774 match(thread_RegP);
4775 op_cost(0);
4776 format %{ %}
4777 interface(REG_INTER);
4778 %}
4779
4780 // Pointer 64 bit Register not Special
4781 operand iRegPNoSp()
4782 %{
4783 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4784 match(RegP);
4785 // match(iRegP);
4786 // match(iRegP_R0);
4787 // match(iRegP_R2);
4788 // match(iRegP_R4);
4789 // match(iRegP_R5);
4790 // match(thread_RegP);
4791 op_cost(0);
4792 format %{ %}
4793 interface(REG_INTER);
4794 %}
4795
4796 // This operand is not allowed to use rfp even if
4797 // rfp is not used to hold the frame pointer.
4798 operand iRegPNoSpNoRfp()
4799 %{
4800 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4801 match(RegP);
4802 match(iRegPNoSp);
4803 op_cost(0);
4804 format %{ %}
4805 interface(REG_INTER);
4806 %}
4807
4808 // Pointer 64 bit Register R0 only
4809 operand iRegP_R0()
4810 %{
4811 constraint(ALLOC_IN_RC(r0_reg));
4812 match(RegP);
4813 // match(iRegP);
4814 match(iRegPNoSp);
4815 op_cost(0);
4816 format %{ %}
4817 interface(REG_INTER);
4818 %}
4819
4820 // Pointer 64 bit Register R1 only
4821 operand iRegP_R1()
4822 %{
4823 constraint(ALLOC_IN_RC(r1_reg));
4824 match(RegP);
4825 // match(iRegP);
4826 match(iRegPNoSp);
4827 op_cost(0);
4828 format %{ %}
4829 interface(REG_INTER);
4830 %}
4831
4832 // Pointer 64 bit Register R2 only
4833 operand iRegP_R2()
4834 %{
4835 constraint(ALLOC_IN_RC(r2_reg));
4836 match(RegP);
4837 // match(iRegP);
4838 match(iRegPNoSp);
4839 op_cost(0);
4840 format %{ %}
4841 interface(REG_INTER);
4842 %}
4843
4844 // Pointer 64 bit Register R3 only
4845 operand iRegP_R3()
4846 %{
4847 constraint(ALLOC_IN_RC(r3_reg));
4848 match(RegP);
4849 // match(iRegP);
4850 match(iRegPNoSp);
4851 op_cost(0);
4852 format %{ %}
4853 interface(REG_INTER);
4854 %}
4855
4856 // Pointer 64 bit Register R4 only
4857 operand iRegP_R4()
4858 %{
4859 constraint(ALLOC_IN_RC(r4_reg));
4860 match(RegP);
4861 // match(iRegP);
4862 match(iRegPNoSp);
4863 op_cost(0);
4864 format %{ %}
4865 interface(REG_INTER);
4866 %}
4867
4868 // Pointer 64 bit Register R5 only
4869 operand iRegP_R5()
4870 %{
4871 constraint(ALLOC_IN_RC(r5_reg));
4872 match(RegP);
4873 // match(iRegP);
4874 match(iRegPNoSp);
4875 op_cost(0);
4876 format %{ %}
4877 interface(REG_INTER);
4878 %}
4879
4880 // Pointer 64 bit Register R10 only
4881 operand iRegP_R10()
4882 %{
4883 constraint(ALLOC_IN_RC(r10_reg));
4884 match(RegP);
4885 // match(iRegP);
4886 match(iRegPNoSp);
4887 op_cost(0);
4888 format %{ %}
4889 interface(REG_INTER);
4890 %}
4891
4892 // Long 64 bit Register R0 only
4893 operand iRegL_R0()
4894 %{
4895 constraint(ALLOC_IN_RC(r0_reg));
4896 match(RegL);
4897 match(iRegLNoSp);
4898 op_cost(0);
4899 format %{ %}
4900 interface(REG_INTER);
4901 %}
4902
4903 // Long 64 bit Register R11 only
4904 operand iRegL_R11()
4905 %{
4906 constraint(ALLOC_IN_RC(r11_reg));
4907 match(RegL);
4908 match(iRegLNoSp);
4909 op_cost(0);
4910 format %{ %}
4911 interface(REG_INTER);
4912 %}
4913
4914 // Register R0 only
4915 operand iRegI_R0()
4916 %{
4917 constraint(ALLOC_IN_RC(int_r0_reg));
4918 match(RegI);
4919 match(iRegINoSp);
4920 op_cost(0);
4921 format %{ %}
4922 interface(REG_INTER);
4923 %}
4924
4925 // Register R2 only
4926 operand iRegI_R2()
4927 %{
4928 constraint(ALLOC_IN_RC(int_r2_reg));
4929 match(RegI);
4930 match(iRegINoSp);
4931 op_cost(0);
4932 format %{ %}
4933 interface(REG_INTER);
4934 %}
4935
4936 // Register R3 only
4937 operand iRegI_R3()
4938 %{
4939 constraint(ALLOC_IN_RC(int_r3_reg));
4940 match(RegI);
4941 match(iRegINoSp);
4942 op_cost(0);
4943 format %{ %}
4944 interface(REG_INTER);
4945 %}
4946
4947
4948 // Register R4 only
4949 operand iRegI_R4()
4950 %{
4951 constraint(ALLOC_IN_RC(int_r4_reg));
4952 match(RegI);
4953 match(iRegINoSp);
4954 op_cost(0);
4955 format %{ %}
4956 interface(REG_INTER);
4957 %}
4958
4959
4960 // Pointer Register Operands
4961 // Narrow Pointer Register
4962 operand iRegN()
4963 %{
4964 constraint(ALLOC_IN_RC(any_reg32));
4965 match(RegN);
4966 match(iRegNNoSp);
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 // Integer 64 bit Register not Special
4973 operand iRegNNoSp()
4974 %{
4975 constraint(ALLOC_IN_RC(no_special_reg32));
4976 match(RegN);
4977 op_cost(0);
4978 format %{ %}
4979 interface(REG_INTER);
4980 %}
4981
4982 // Float Register
4983 // Float register operands
4984 operand vRegF()
4985 %{
4986 constraint(ALLOC_IN_RC(float_reg));
4987 match(RegF);
4988
4989 op_cost(0);
4990 format %{ %}
4991 interface(REG_INTER);
4992 %}
4993
4994 // Double Register
4995 // Double register operands
4996 operand vRegD()
4997 %{
4998 constraint(ALLOC_IN_RC(double_reg));
4999 match(RegD);
5000
5001 op_cost(0);
5002 format %{ %}
5003 interface(REG_INTER);
5004 %}
5005
5006 // Generic vector class. This will be used for
5007 // all vector operands, including NEON and SVE.
5008 operand vReg()
5009 %{
5010 constraint(ALLOC_IN_RC(dynamic));
5011 match(VecA);
5012 match(VecD);
5013 match(VecX);
5014
5015 op_cost(0);
5016 format %{ %}
5017 interface(REG_INTER);
5018 %}
5019
5020 operand vReg_V10()
5021 %{
5022 constraint(ALLOC_IN_RC(v10_veca_reg));
5023 match(vReg);
5024
5025 op_cost(0);
5026 format %{ %}
5027 interface(REG_INTER);
5028 %}
5029
5030 operand vReg_V11()
5031 %{
5032 constraint(ALLOC_IN_RC(v11_veca_reg));
5033 match(vReg);
5034
5035 op_cost(0);
5036 format %{ %}
5037 interface(REG_INTER);
5038 %}
5039
5040 operand vReg_V12()
5041 %{
5042 constraint(ALLOC_IN_RC(v12_veca_reg));
5043 match(vReg);
5044
5045 op_cost(0);
5046 format %{ %}
5047 interface(REG_INTER);
5048 %}
5049
5050 operand vReg_V13()
5051 %{
5052 constraint(ALLOC_IN_RC(v13_veca_reg));
5053 match(vReg);
5054
5055 op_cost(0);
5056 format %{ %}
5057 interface(REG_INTER);
5058 %}
5059
5060 operand vReg_V17()
5061 %{
5062 constraint(ALLOC_IN_RC(v17_veca_reg));
5063 match(vReg);
5064
5065 op_cost(0);
5066 format %{ %}
5067 interface(REG_INTER);
5068 %}
5069
5070 operand vReg_V18()
5071 %{
5072 constraint(ALLOC_IN_RC(v18_veca_reg));
5073 match(vReg);
5074
5075 op_cost(0);
5076 format %{ %}
5077 interface(REG_INTER);
5078 %}
5079
5080 operand vReg_V23()
5081 %{
5082 constraint(ALLOC_IN_RC(v23_veca_reg));
5083 match(vReg);
5084
5085 op_cost(0);
5086 format %{ %}
5087 interface(REG_INTER);
5088 %}
5089
5090 operand vReg_V24()
5091 %{
5092 constraint(ALLOC_IN_RC(v24_veca_reg));
5093 match(vReg);
5094
5095 op_cost(0);
5096 format %{ %}
5097 interface(REG_INTER);
5098 %}
5099
5100 operand vecA()
5101 %{
5102 constraint(ALLOC_IN_RC(vectora_reg));
5103 match(VecA);
5104
5105 op_cost(0);
5106 format %{ %}
5107 interface(REG_INTER);
5108 %}
5109
5110 operand vecD()
5111 %{
5112 constraint(ALLOC_IN_RC(vectord_reg));
5113 match(VecD);
5114
5115 op_cost(0);
5116 format %{ %}
5117 interface(REG_INTER);
5118 %}
5119
5120 operand vecX()
5121 %{
5122 constraint(ALLOC_IN_RC(vectorx_reg));
5123 match(VecX);
5124
5125 op_cost(0);
5126 format %{ %}
5127 interface(REG_INTER);
5128 %}
5129
5130 operand vRegD_V0()
5131 %{
5132 constraint(ALLOC_IN_RC(v0_reg));
5133 match(RegD);
5134 op_cost(0);
5135 format %{ %}
5136 interface(REG_INTER);
5137 %}
5138
5139 operand vRegD_V1()
5140 %{
5141 constraint(ALLOC_IN_RC(v1_reg));
5142 match(RegD);
5143 op_cost(0);
5144 format %{ %}
5145 interface(REG_INTER);
5146 %}
5147
5148 operand vRegD_V2()
5149 %{
5150 constraint(ALLOC_IN_RC(v2_reg));
5151 match(RegD);
5152 op_cost(0);
5153 format %{ %}
5154 interface(REG_INTER);
5155 %}
5156
5157 operand vRegD_V3()
5158 %{
5159 constraint(ALLOC_IN_RC(v3_reg));
5160 match(RegD);
5161 op_cost(0);
5162 format %{ %}
5163 interface(REG_INTER);
5164 %}
5165
5166 operand vRegD_V4()
5167 %{
5168 constraint(ALLOC_IN_RC(v4_reg));
5169 match(RegD);
5170 op_cost(0);
5171 format %{ %}
5172 interface(REG_INTER);
5173 %}
5174
5175 operand vRegD_V5()
5176 %{
5177 constraint(ALLOC_IN_RC(v5_reg));
5178 match(RegD);
5179 op_cost(0);
5180 format %{ %}
5181 interface(REG_INTER);
5182 %}
5183
5184 operand vRegD_V6()
5185 %{
5186 constraint(ALLOC_IN_RC(v6_reg));
5187 match(RegD);
5188 op_cost(0);
5189 format %{ %}
5190 interface(REG_INTER);
5191 %}
5192
5193 operand vRegD_V7()
5194 %{
5195 constraint(ALLOC_IN_RC(v7_reg));
5196 match(RegD);
5197 op_cost(0);
5198 format %{ %}
5199 interface(REG_INTER);
5200 %}
5201
5202 operand vRegD_V12()
5203 %{
5204 constraint(ALLOC_IN_RC(v12_reg));
5205 match(RegD);
5206 op_cost(0);
5207 format %{ %}
5208 interface(REG_INTER);
5209 %}
5210
5211 operand vRegD_V13()
5212 %{
5213 constraint(ALLOC_IN_RC(v13_reg));
5214 match(RegD);
5215 op_cost(0);
5216 format %{ %}
5217 interface(REG_INTER);
5218 %}
5219
5220 operand pReg()
5221 %{
5222 constraint(ALLOC_IN_RC(pr_reg));
5223 match(RegVectMask);
5224 match(pRegGov);
5225 op_cost(0);
5226 format %{ %}
5227 interface(REG_INTER);
5228 %}
5229
5230 operand pRegGov()
5231 %{
5232 constraint(ALLOC_IN_RC(gov_pr));
5233 match(RegVectMask);
5234 match(pReg);
5235 op_cost(0);
5236 format %{ %}
5237 interface(REG_INTER);
5238 %}
5239
5240 operand pRegGov_P0()
5241 %{
5242 constraint(ALLOC_IN_RC(p0_reg));
5243 match(RegVectMask);
5244 op_cost(0);
5245 format %{ %}
5246 interface(REG_INTER);
5247 %}
5248
5249 operand pRegGov_P1()
5250 %{
5251 constraint(ALLOC_IN_RC(p1_reg));
5252 match(RegVectMask);
5253 op_cost(0);
5254 format %{ %}
5255 interface(REG_INTER);
5256 %}
5257
5258 // Flags register, used as output of signed compare instructions
5259
5260 // note that on AArch64 we also use this register as the output for
5261 // for floating point compare instructions (CmpF CmpD). this ensures
5262 // that ordered inequality tests use GT, GE, LT or LE none of which
5263 // pass through cases where the result is unordered i.e. one or both
5264 // inputs to the compare is a NaN. this means that the ideal code can
5265 // replace e.g. a GT with an LE and not end up capturing the NaN case
5266 // (where the comparison should always fail). EQ and NE tests are
5267 // always generated in ideal code so that unordered folds into the NE
5268 // case, matching the behaviour of AArch64 NE.
5269 //
5270 // This differs from x86 where the outputs of FP compares use a
5271 // special FP flags registers and where compares based on this
5272 // register are distinguished into ordered inequalities (cmpOpUCF) and
5273 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5274 // to explicitly handle the unordered case in branches. x86 also has
5275 // to include extra CMoveX rules to accept a cmpOpUCF input.
5276
5277 operand rFlagsReg()
5278 %{
5279 constraint(ALLOC_IN_RC(int_flags));
5280 match(RegFlags);
5281
5282 op_cost(0);
5283 format %{ "RFLAGS" %}
5284 interface(REG_INTER);
5285 %}
5286
5287 // Flags register, used as output of unsigned compare instructions
5288 operand rFlagsRegU()
5289 %{
5290 constraint(ALLOC_IN_RC(int_flags));
5291 match(RegFlags);
5292
5293 op_cost(0);
5294 format %{ "RFLAGSU" %}
5295 interface(REG_INTER);
5296 %}
5297
5298 // Special Registers
5299
5300 // Method Register
5301 operand inline_cache_RegP(iRegP reg)
5302 %{
5303 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5304 match(reg);
5305 match(iRegPNoSp);
5306 op_cost(0);
5307 format %{ %}
5308 interface(REG_INTER);
5309 %}
5310
5311 // Thread Register
5312 operand thread_RegP(iRegP reg)
5313 %{
5314 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5315 match(reg);
5316 op_cost(0);
5317 format %{ %}
5318 interface(REG_INTER);
5319 %}
5320
5321 //----------Memory Operands----------------------------------------------------
5322
5323 operand indirect(iRegP reg)
5324 %{
5325 constraint(ALLOC_IN_RC(ptr_reg));
5326 match(reg);
5327 op_cost(0);
5328 format %{ "[$reg]" %}
5329 interface(MEMORY_INTER) %{
5330 base($reg);
5331 index(0xffffffff);
5332 scale(0x0);
5333 disp(0x0);
5334 %}
5335 %}
5336
5337 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5338 %{
5339 constraint(ALLOC_IN_RC(ptr_reg));
5340 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5341 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5342 op_cost(0);
5343 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5344 interface(MEMORY_INTER) %{
5345 base($reg);
5346 index($ireg);
5347 scale($scale);
5348 disp(0x0);
5349 %}
5350 %}
5351
5352 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5353 %{
5354 constraint(ALLOC_IN_RC(ptr_reg));
5355 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5356 match(AddP reg (LShiftL lreg scale));
5357 op_cost(0);
5358 format %{ "$reg, $lreg lsl($scale)" %}
5359 interface(MEMORY_INTER) %{
5360 base($reg);
5361 index($lreg);
5362 scale($scale);
5363 disp(0x0);
5364 %}
5365 %}
5366
5367 operand indIndexI2L(iRegP reg, iRegI ireg)
5368 %{
5369 constraint(ALLOC_IN_RC(ptr_reg));
5370 match(AddP reg (ConvI2L ireg));
5371 op_cost(0);
5372 format %{ "$reg, $ireg, 0, I2L" %}
5373 interface(MEMORY_INTER) %{
5374 base($reg);
5375 index($ireg);
5376 scale(0x0);
5377 disp(0x0);
5378 %}
5379 %}
5380
5381 operand indIndex(iRegP reg, iRegL lreg)
5382 %{
5383 constraint(ALLOC_IN_RC(ptr_reg));
5384 match(AddP reg lreg);
5385 op_cost(0);
5386 format %{ "$reg, $lreg" %}
5387 interface(MEMORY_INTER) %{
5388 base($reg);
5389 index($lreg);
5390 scale(0x0);
5391 disp(0x0);
5392 %}
5393 %}
5394
5395 operand indOffI1(iRegP reg, immIOffset1 off)
5396 %{
5397 constraint(ALLOC_IN_RC(ptr_reg));
5398 match(AddP reg off);
5399 op_cost(0);
5400 format %{ "[$reg, $off]" %}
5401 interface(MEMORY_INTER) %{
5402 base($reg);
5403 index(0xffffffff);
5404 scale(0x0);
5405 disp($off);
5406 %}
5407 %}
5408
5409 operand indOffI2(iRegP reg, immIOffset2 off)
5410 %{
5411 constraint(ALLOC_IN_RC(ptr_reg));
5412 match(AddP reg off);
5413 op_cost(0);
5414 format %{ "[$reg, $off]" %}
5415 interface(MEMORY_INTER) %{
5416 base($reg);
5417 index(0xffffffff);
5418 scale(0x0);
5419 disp($off);
5420 %}
5421 %}
5422
5423 operand indOffI4(iRegP reg, immIOffset4 off)
5424 %{
5425 constraint(ALLOC_IN_RC(ptr_reg));
5426 match(AddP reg off);
5427 op_cost(0);
5428 format %{ "[$reg, $off]" %}
5429 interface(MEMORY_INTER) %{
5430 base($reg);
5431 index(0xffffffff);
5432 scale(0x0);
5433 disp($off);
5434 %}
5435 %}
5436
5437 operand indOffI8(iRegP reg, immIOffset8 off)
5438 %{
5439 constraint(ALLOC_IN_RC(ptr_reg));
5440 match(AddP reg off);
5441 op_cost(0);
5442 format %{ "[$reg, $off]" %}
5443 interface(MEMORY_INTER) %{
5444 base($reg);
5445 index(0xffffffff);
5446 scale(0x0);
5447 disp($off);
5448 %}
5449 %}
5450
5451 operand indOffI16(iRegP reg, immIOffset16 off)
5452 %{
5453 constraint(ALLOC_IN_RC(ptr_reg));
5454 match(AddP reg off);
5455 op_cost(0);
5456 format %{ "[$reg, $off]" %}
5457 interface(MEMORY_INTER) %{
5458 base($reg);
5459 index(0xffffffff);
5460 scale(0x0);
5461 disp($off);
5462 %}
5463 %}
5464
5465 operand indOffL1(iRegP reg, immLoffset1 off)
5466 %{
5467 constraint(ALLOC_IN_RC(ptr_reg));
5468 match(AddP reg off);
5469 op_cost(0);
5470 format %{ "[$reg, $off]" %}
5471 interface(MEMORY_INTER) %{
5472 base($reg);
5473 index(0xffffffff);
5474 scale(0x0);
5475 disp($off);
5476 %}
5477 %}
5478
5479 operand indOffL2(iRegP reg, immLoffset2 off)
5480 %{
5481 constraint(ALLOC_IN_RC(ptr_reg));
5482 match(AddP reg off);
5483 op_cost(0);
5484 format %{ "[$reg, $off]" %}
5485 interface(MEMORY_INTER) %{
5486 base($reg);
5487 index(0xffffffff);
5488 scale(0x0);
5489 disp($off);
5490 %}
5491 %}
5492
5493 operand indOffL4(iRegP reg, immLoffset4 off)
5494 %{
5495 constraint(ALLOC_IN_RC(ptr_reg));
5496 match(AddP reg off);
5497 op_cost(0);
5498 format %{ "[$reg, $off]" %}
5499 interface(MEMORY_INTER) %{
5500 base($reg);
5501 index(0xffffffff);
5502 scale(0x0);
5503 disp($off);
5504 %}
5505 %}
5506
5507 operand indOffL8(iRegP reg, immLoffset8 off)
5508 %{
5509 constraint(ALLOC_IN_RC(ptr_reg));
5510 match(AddP reg off);
5511 op_cost(0);
5512 format %{ "[$reg, $off]" %}
5513 interface(MEMORY_INTER) %{
5514 base($reg);
5515 index(0xffffffff);
5516 scale(0x0);
5517 disp($off);
5518 %}
5519 %}
5520
5521 operand indOffL16(iRegP reg, immLoffset16 off)
5522 %{
5523 constraint(ALLOC_IN_RC(ptr_reg));
5524 match(AddP reg off);
5525 op_cost(0);
5526 format %{ "[$reg, $off]" %}
5527 interface(MEMORY_INTER) %{
5528 base($reg);
5529 index(0xffffffff);
5530 scale(0x0);
5531 disp($off);
5532 %}
5533 %}
5534
5535 operand indirectX2P(iRegL reg)
5536 %{
5537 constraint(ALLOC_IN_RC(ptr_reg));
5538 match(CastX2P reg);
5539 op_cost(0);
5540 format %{ "[$reg]\t# long -> ptr" %}
5541 interface(MEMORY_INTER) %{
5542 base($reg);
5543 index(0xffffffff);
5544 scale(0x0);
5545 disp(0x0);
5546 %}
5547 %}
5548
5549 operand indOffX2P(iRegL reg, immLOffset off)
5550 %{
5551 constraint(ALLOC_IN_RC(ptr_reg));
5552 match(AddP (CastX2P reg) off);
5553 op_cost(0);
5554 format %{ "[$reg, $off]\t# long -> ptr" %}
5555 interface(MEMORY_INTER) %{
5556 base($reg);
5557 index(0xffffffff);
5558 scale(0x0);
5559 disp($off);
5560 %}
5561 %}
5562
5563 operand indirectN(iRegN reg)
5564 %{
5565 predicate(CompressedOops::shift() == 0);
5566 constraint(ALLOC_IN_RC(ptr_reg));
5567 match(DecodeN reg);
5568 op_cost(0);
5569 format %{ "[$reg]\t# narrow" %}
5570 interface(MEMORY_INTER) %{
5571 base($reg);
5572 index(0xffffffff);
5573 scale(0x0);
5574 disp(0x0);
5575 %}
5576 %}
5577
5578 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5579 %{
5580 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5581 constraint(ALLOC_IN_RC(ptr_reg));
5582 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5583 op_cost(0);
5584 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5585 interface(MEMORY_INTER) %{
5586 base($reg);
5587 index($ireg);
5588 scale($scale);
5589 disp(0x0);
5590 %}
5591 %}
5592
5593 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5594 %{
5595 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5596 constraint(ALLOC_IN_RC(ptr_reg));
5597 match(AddP (DecodeN reg) (LShiftL lreg scale));
5598 op_cost(0);
5599 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5600 interface(MEMORY_INTER) %{
5601 base($reg);
5602 index($lreg);
5603 scale($scale);
5604 disp(0x0);
5605 %}
5606 %}
5607
5608 operand indIndexI2LN(iRegN reg, iRegI ireg)
5609 %{
5610 predicate(CompressedOops::shift() == 0);
5611 constraint(ALLOC_IN_RC(ptr_reg));
5612 match(AddP (DecodeN reg) (ConvI2L ireg));
5613 op_cost(0);
5614 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5615 interface(MEMORY_INTER) %{
5616 base($reg);
5617 index($ireg);
5618 scale(0x0);
5619 disp(0x0);
5620 %}
5621 %}
5622
5623 operand indIndexN(iRegN reg, iRegL lreg)
5624 %{
5625 predicate(CompressedOops::shift() == 0);
5626 constraint(ALLOC_IN_RC(ptr_reg));
5627 match(AddP (DecodeN reg) lreg);
5628 op_cost(0);
5629 format %{ "$reg, $lreg\t# narrow" %}
5630 interface(MEMORY_INTER) %{
5631 base($reg);
5632 index($lreg);
5633 scale(0x0);
5634 disp(0x0);
5635 %}
5636 %}
5637
5638 operand indOffIN(iRegN reg, immIOffset off)
5639 %{
5640 predicate(CompressedOops::shift() == 0);
5641 constraint(ALLOC_IN_RC(ptr_reg));
5642 match(AddP (DecodeN reg) off);
5643 op_cost(0);
5644 format %{ "[$reg, $off]\t# narrow" %}
5645 interface(MEMORY_INTER) %{
5646 base($reg);
5647 index(0xffffffff);
5648 scale(0x0);
5649 disp($off);
5650 %}
5651 %}
5652
5653 operand indOffLN(iRegN reg, immLOffset off)
5654 %{
5655 predicate(CompressedOops::shift() == 0);
5656 constraint(ALLOC_IN_RC(ptr_reg));
5657 match(AddP (DecodeN reg) off);
5658 op_cost(0);
5659 format %{ "[$reg, $off]\t# narrow" %}
5660 interface(MEMORY_INTER) %{
5661 base($reg);
5662 index(0xffffffff);
5663 scale(0x0);
5664 disp($off);
5665 %}
5666 %}
5667
5668
5669 //----------Special Memory Operands--------------------------------------------
5670 // Stack Slot Operand - This operand is used for loading and storing temporary
5671 // values on the stack where a match requires a value to
5672 // flow through memory.
5673 operand stackSlotP(sRegP reg)
5674 %{
5675 constraint(ALLOC_IN_RC(stack_slots));
5676 op_cost(100);
5677 // No match rule because this operand is only generated in matching
5678 // match(RegP);
5679 format %{ "[$reg]" %}
5680 interface(MEMORY_INTER) %{
5681 base(0x1e); // RSP
5682 index(0x0); // No Index
5683 scale(0x0); // No Scale
5684 disp($reg); // Stack Offset
5685 %}
5686 %}
5687
5688 operand stackSlotI(sRegI reg)
5689 %{
5690 constraint(ALLOC_IN_RC(stack_slots));
5691 // No match rule because this operand is only generated in matching
5692 // match(RegI);
5693 format %{ "[$reg]" %}
5694 interface(MEMORY_INTER) %{
5695 base(0x1e); // RSP
5696 index(0x0); // No Index
5697 scale(0x0); // No Scale
5698 disp($reg); // Stack Offset
5699 %}
5700 %}
5701
5702 operand stackSlotF(sRegF reg)
5703 %{
5704 constraint(ALLOC_IN_RC(stack_slots));
5705 // No match rule because this operand is only generated in matching
5706 // match(RegF);
5707 format %{ "[$reg]" %}
5708 interface(MEMORY_INTER) %{
5709 base(0x1e); // RSP
5710 index(0x0); // No Index
5711 scale(0x0); // No Scale
5712 disp($reg); // Stack Offset
5713 %}
5714 %}
5715
5716 operand stackSlotD(sRegD reg)
5717 %{
5718 constraint(ALLOC_IN_RC(stack_slots));
5719 // No match rule because this operand is only generated in matching
5720 // match(RegD);
5721 format %{ "[$reg]" %}
5722 interface(MEMORY_INTER) %{
5723 base(0x1e); // RSP
5724 index(0x0); // No Index
5725 scale(0x0); // No Scale
5726 disp($reg); // Stack Offset
5727 %}
5728 %}
5729
5730 operand stackSlotL(sRegL reg)
5731 %{
5732 constraint(ALLOC_IN_RC(stack_slots));
5733 // No match rule because this operand is only generated in matching
5734 // match(RegL);
5735 format %{ "[$reg]" %}
5736 interface(MEMORY_INTER) %{
5737 base(0x1e); // RSP
5738 index(0x0); // No Index
5739 scale(0x0); // No Scale
5740 disp($reg); // Stack Offset
5741 %}
5742 %}
5743
5744 // Operands for expressing Control Flow
5745 // NOTE: Label is a predefined operand which should not be redefined in
5746 // the AD file. It is generically handled within the ADLC.
5747
5748 //----------Conditional Branch Operands----------------------------------------
5749 // Comparison Op - This is the operation of the comparison, and is limited to
5750 // the following set of codes:
5751 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5752 //
5753 // Other attributes of the comparison, such as unsignedness, are specified
5754 // by the comparison instruction that sets a condition code flags register.
5755 // That result is represented by a flags operand whose subtype is appropriate
5756 // to the unsignedness (etc.) of the comparison.
5757 //
5758 // Later, the instruction which matches both the Comparison Op (a Bool) and
5759 // the flags (produced by the Cmp) specifies the coding of the comparison op
5760 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5761
5762 // used for signed integral comparisons and fp comparisons
5763
5764 operand cmpOp()
5765 %{
5766 match(Bool);
5767
5768 format %{ "" %}
5769 interface(COND_INTER) %{
5770 equal(0x0, "eq");
5771 not_equal(0x1, "ne");
5772 less(0xb, "lt");
5773 greater_equal(0xa, "ge");
5774 less_equal(0xd, "le");
5775 greater(0xc, "gt");
5776 overflow(0x6, "vs");
5777 no_overflow(0x7, "vc");
5778 %}
5779 %}
5780
5781 // used for unsigned integral comparisons
5782
5783 operand cmpOpU()
5784 %{
5785 match(Bool);
5786
5787 format %{ "" %}
5788 interface(COND_INTER) %{
5789 equal(0x0, "eq");
5790 not_equal(0x1, "ne");
5791 less(0x3, "lo");
5792 greater_equal(0x2, "hs");
5793 less_equal(0x9, "ls");
5794 greater(0x8, "hi");
5795 overflow(0x6, "vs");
5796 no_overflow(0x7, "vc");
5797 %}
5798 %}
5799
5800 // used for certain integral comparisons which can be
5801 // converted to cbxx or tbxx instructions
5802
5803 operand cmpOpEqNe()
5804 %{
5805 match(Bool);
5806 op_cost(0);
5807 predicate(n->as_Bool()->_test._test == BoolTest::ne
5808 || n->as_Bool()->_test._test == BoolTest::eq);
5809
5810 format %{ "" %}
5811 interface(COND_INTER) %{
5812 equal(0x0, "eq");
5813 not_equal(0x1, "ne");
5814 less(0xb, "lt");
5815 greater_equal(0xa, "ge");
5816 less_equal(0xd, "le");
5817 greater(0xc, "gt");
5818 overflow(0x6, "vs");
5819 no_overflow(0x7, "vc");
5820 %}
5821 %}
5822
5823 // used for certain integral comparisons which can be
5824 // converted to cbxx or tbxx instructions
5825
5826 operand cmpOpLtGe()
5827 %{
5828 match(Bool);
5829 op_cost(0);
5830
5831 predicate(n->as_Bool()->_test._test == BoolTest::lt
5832 || n->as_Bool()->_test._test == BoolTest::ge);
5833
5834 format %{ "" %}
5835 interface(COND_INTER) %{
5836 equal(0x0, "eq");
5837 not_equal(0x1, "ne");
5838 less(0xb, "lt");
5839 greater_equal(0xa, "ge");
5840 less_equal(0xd, "le");
5841 greater(0xc, "gt");
5842 overflow(0x6, "vs");
5843 no_overflow(0x7, "vc");
5844 %}
5845 %}
5846
5847 // used for certain unsigned integral comparisons which can be
5848 // converted to cbxx or tbxx instructions
5849
5850 operand cmpOpUEqNeLeGt()
5851 %{
5852 match(Bool);
5853 op_cost(0);
5854
5855 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5856 n->as_Bool()->_test._test == BoolTest::ne ||
5857 n->as_Bool()->_test._test == BoolTest::le ||
5858 n->as_Bool()->_test._test == BoolTest::gt);
5859
5860 format %{ "" %}
5861 interface(COND_INTER) %{
5862 equal(0x0, "eq");
5863 not_equal(0x1, "ne");
5864 less(0x3, "lo");
5865 greater_equal(0x2, "hs");
5866 less_equal(0x9, "ls");
5867 greater(0x8, "hi");
5868 overflow(0x6, "vs");
5869 no_overflow(0x7, "vc");
5870 %}
5871 %}
5872
5873 // Special operand allowing long args to int ops to be truncated for free
5874
5875 operand iRegL2I(iRegL reg) %{
5876
5877 op_cost(0);
5878
5879 match(ConvL2I reg);
5880
5881 format %{ "l2i($reg)" %}
5882
5883 interface(REG_INTER)
5884 %}
5885
5886 operand iRegL2P(iRegL reg) %{
5887
5888 op_cost(0);
5889
5890 match(CastX2P reg);
5891
5892 format %{ "l2p($reg)" %}
5893
5894 interface(REG_INTER)
5895 %}
5896
5897 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5898 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5899 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5900 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5901
5902 //----------OPERAND CLASSES----------------------------------------------------
5903 // Operand Classes are groups of operands that are used as to simplify
5904 // instruction definitions by not requiring the AD writer to specify
5905 // separate instructions for every form of operand when the
5906 // instruction accepts multiple operand types with the same basic
5907 // encoding and format. The classic case of this is memory operands.
5908
5909 // memory is used to define read/write location for load/store
5910 // instruction defs. we can turn a memory op into an Address
5911
5912 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5913 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5914
5915 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5916 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5917
5918 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5919 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5920
5921 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5922 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5923
5924 // All of the memory operands. For the pipeline description.
5925 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5926 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5927 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5928
5929
5930 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5931 // operations. it allows the src to be either an iRegI or a (ConvL2I
5932 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5933 // can be elided because the 32-bit instruction will just employ the
5934 // lower 32 bits anyway.
5935 //
5936 // n.b. this does not elide all L2I conversions. if the truncated
5937 // value is consumed by more than one operation then the ConvL2I
5938 // cannot be bundled into the consuming nodes so an l2i gets planted
5939 // (actually a movw $dst $src) and the downstream instructions consume
5940 // the result of the l2i as an iRegI input. That's a shame since the
5941 // movw is actually redundant but its not too costly.
5942
5943 opclass iRegIorL2I(iRegI, iRegL2I);
5944 opclass iRegPorL2P(iRegP, iRegL2P);
5945
5946 //----------PIPELINE-----------------------------------------------------------
5947 // Rules which define the behavior of the target architectures pipeline.
5948
5949 // For specific pipelines, eg A53, define the stages of that pipeline
5950 //pipe_desc(ISS, EX1, EX2, WR);
5951 #define ISS S0
5952 #define EX1 S1
5953 #define EX2 S2
5954 #define WR S3
5955
5956 // Integer ALU reg operation
5957 pipeline %{
5958
5959 attributes %{
5960 // ARM instructions are of fixed length
5961 fixed_size_instructions; // Fixed size instructions TODO does
5962 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5963 // ARM instructions come in 32-bit word units
5964 instruction_unit_size = 4; // An instruction is 4 bytes long
5965 instruction_fetch_unit_size = 64; // The processor fetches one line
5966 instruction_fetch_units = 1; // of 64 bytes
5967 %}
5968
5969 // We don't use an actual pipeline model so don't care about resources
5970 // or description. we do use pipeline classes to introduce fixed
5971 // latencies
5972
5973 //----------RESOURCES----------------------------------------------------------
5974 // Resources are the functional units available to the machine
5975
5976 resources( INS0, INS1, INS01 = INS0 | INS1,
5977 ALU0, ALU1, ALU = ALU0 | ALU1,
5978 MAC,
5979 DIV,
5980 BRANCH,
5981 LDST,
5982 NEON_FP);
5983
5984 //----------PIPELINE DESCRIPTION-----------------------------------------------
5985 // Pipeline Description specifies the stages in the machine's pipeline
5986
5987 // Define the pipeline as a generic 6 stage pipeline
5988 pipe_desc(S0, S1, S2, S3, S4, S5);
5989
5990 //----------PIPELINE CLASSES---------------------------------------------------
5991 // Pipeline Classes describe the stages in which input and output are
5992 // referenced by the hardware pipeline.
5993
5994 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5995 %{
5996 single_instruction;
5997 src1 : S1(read);
5998 src2 : S2(read);
5999 dst : S5(write);
6000 INS01 : ISS;
6001 NEON_FP : S5;
6002 %}
6003
6004 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6005 %{
6006 single_instruction;
6007 src1 : S1(read);
6008 src2 : S2(read);
6009 dst : S5(write);
6010 INS01 : ISS;
6011 NEON_FP : S5;
6012 %}
6013
6014 pipe_class fp_uop_s(vRegF dst, vRegF src)
6015 %{
6016 single_instruction;
6017 src : S1(read);
6018 dst : S5(write);
6019 INS01 : ISS;
6020 NEON_FP : S5;
6021 %}
6022
6023 pipe_class fp_uop_d(vRegD dst, vRegD src)
6024 %{
6025 single_instruction;
6026 src : S1(read);
6027 dst : S5(write);
6028 INS01 : ISS;
6029 NEON_FP : S5;
6030 %}
6031
6032 pipe_class fp_d2f(vRegF dst, vRegD src)
6033 %{
6034 single_instruction;
6035 src : S1(read);
6036 dst : S5(write);
6037 INS01 : ISS;
6038 NEON_FP : S5;
6039 %}
6040
6041 pipe_class fp_f2d(vRegD dst, vRegF src)
6042 %{
6043 single_instruction;
6044 src : S1(read);
6045 dst : S5(write);
6046 INS01 : ISS;
6047 NEON_FP : S5;
6048 %}
6049
6050 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6051 %{
6052 single_instruction;
6053 src : S1(read);
6054 dst : S5(write);
6055 INS01 : ISS;
6056 NEON_FP : S5;
6057 %}
6058
6059 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6060 %{
6061 single_instruction;
6062 src : S1(read);
6063 dst : S5(write);
6064 INS01 : ISS;
6065 NEON_FP : S5;
6066 %}
6067
6068 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6069 %{
6070 single_instruction;
6071 src : S1(read);
6072 dst : S5(write);
6073 INS01 : ISS;
6074 NEON_FP : S5;
6075 %}
6076
6077 pipe_class fp_l2f(vRegF dst, iRegL src)
6078 %{
6079 single_instruction;
6080 src : S1(read);
6081 dst : S5(write);
6082 INS01 : ISS;
6083 NEON_FP : S5;
6084 %}
6085
6086 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6087 %{
6088 single_instruction;
6089 src : S1(read);
6090 dst : S5(write);
6091 INS01 : ISS;
6092 NEON_FP : S5;
6093 %}
6094
6095 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6096 %{
6097 single_instruction;
6098 src : S1(read);
6099 dst : S5(write);
6100 INS01 : ISS;
6101 NEON_FP : S5;
6102 %}
6103
6104 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6105 %{
6106 single_instruction;
6107 src : S1(read);
6108 dst : S5(write);
6109 INS01 : ISS;
6110 NEON_FP : S5;
6111 %}
6112
6113 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6114 %{
6115 single_instruction;
6116 src : S1(read);
6117 dst : S5(write);
6118 INS01 : ISS;
6119 NEON_FP : S5;
6120 %}
6121
6122 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6123 %{
6124 single_instruction;
6125 src1 : S1(read);
6126 src2 : S2(read);
6127 dst : S5(write);
6128 INS0 : ISS;
6129 NEON_FP : S5;
6130 %}
6131
6132 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6133 %{
6134 single_instruction;
6135 src1 : S1(read);
6136 src2 : S2(read);
6137 dst : S5(write);
6138 INS0 : ISS;
6139 NEON_FP : S5;
6140 %}
6141
6142 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6143 %{
6144 single_instruction;
6145 cr : S1(read);
6146 src1 : S1(read);
6147 src2 : S1(read);
6148 dst : S3(write);
6149 INS01 : ISS;
6150 NEON_FP : S3;
6151 %}
6152
6153 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6154 %{
6155 single_instruction;
6156 cr : S1(read);
6157 src1 : S1(read);
6158 src2 : S1(read);
6159 dst : S3(write);
6160 INS01 : ISS;
6161 NEON_FP : S3;
6162 %}
6163
6164 pipe_class fp_imm_s(vRegF dst)
6165 %{
6166 single_instruction;
6167 dst : S3(write);
6168 INS01 : ISS;
6169 NEON_FP : S3;
6170 %}
6171
6172 pipe_class fp_imm_d(vRegD dst)
6173 %{
6174 single_instruction;
6175 dst : S3(write);
6176 INS01 : ISS;
6177 NEON_FP : S3;
6178 %}
6179
6180 pipe_class fp_load_constant_s(vRegF dst)
6181 %{
6182 single_instruction;
6183 dst : S4(write);
6184 INS01 : ISS;
6185 NEON_FP : S4;
6186 %}
6187
6188 pipe_class fp_load_constant_d(vRegD dst)
6189 %{
6190 single_instruction;
6191 dst : S4(write);
6192 INS01 : ISS;
6193 NEON_FP : S4;
6194 %}
6195
6196 //------- Integer ALU operations --------------------------
6197
6198 // Integer ALU reg-reg operation
6199 // Operands needed in EX1, result generated in EX2
6200 // Eg. ADD x0, x1, x2
6201 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6202 %{
6203 single_instruction;
6204 dst : EX2(write);
6205 src1 : EX1(read);
6206 src2 : EX1(read);
6207 INS01 : ISS; // Dual issue as instruction 0 or 1
6208 ALU : EX2;
6209 %}
6210
6211 // Integer ALU reg-reg operation with constant shift
6212 // Shifted register must be available in LATE_ISS instead of EX1
6213 // Eg. ADD x0, x1, x2, LSL #2
6214 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6215 %{
6216 single_instruction;
6217 dst : EX2(write);
6218 src1 : EX1(read);
6219 src2 : ISS(read);
6220 INS01 : ISS;
6221 ALU : EX2;
6222 %}
6223
6224 // Integer ALU reg operation with constant shift
6225 // Eg. LSL x0, x1, #shift
6226 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6227 %{
6228 single_instruction;
6229 dst : EX2(write);
6230 src1 : ISS(read);
6231 INS01 : ISS;
6232 ALU : EX2;
6233 %}
6234
6235 // Integer ALU reg-reg operation with variable shift
6236 // Both operands must be available in LATE_ISS instead of EX1
6237 // Result is available in EX1 instead of EX2
6238 // Eg. LSLV x0, x1, x2
6239 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6240 %{
6241 single_instruction;
6242 dst : EX1(write);
6243 src1 : ISS(read);
6244 src2 : ISS(read);
6245 INS01 : ISS;
6246 ALU : EX1;
6247 %}
6248
6249 // Integer ALU reg-reg operation with extract
6250 // As for _vshift above, but result generated in EX2
6251 // Eg. EXTR x0, x1, x2, #N
6252 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6253 %{
6254 single_instruction;
6255 dst : EX2(write);
6256 src1 : ISS(read);
6257 src2 : ISS(read);
6258 INS1 : ISS; // Can only dual issue as Instruction 1
6259 ALU : EX1;
6260 %}
6261
6262 // Integer ALU reg operation
6263 // Eg. NEG x0, x1
6264 pipe_class ialu_reg(iRegI dst, iRegI src)
6265 %{
6266 single_instruction;
6267 dst : EX2(write);
6268 src : EX1(read);
6269 INS01 : ISS;
6270 ALU : EX2;
6271 %}
6272
6273 // Integer ALU reg mmediate operation
6274 // Eg. ADD x0, x1, #N
6275 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6276 %{
6277 single_instruction;
6278 dst : EX2(write);
6279 src1 : EX1(read);
6280 INS01 : ISS;
6281 ALU : EX2;
6282 %}
6283
6284 // Integer ALU immediate operation (no source operands)
6285 // Eg. MOV x0, #N
6286 pipe_class ialu_imm(iRegI dst)
6287 %{
6288 single_instruction;
6289 dst : EX1(write);
6290 INS01 : ISS;
6291 ALU : EX1;
6292 %}
6293
6294 //------- Compare operation -------------------------------
6295
6296 // Compare reg-reg
6297 // Eg. CMP x0, x1
6298 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6299 %{
6300 single_instruction;
6301 // fixed_latency(16);
6302 cr : EX2(write);
6303 op1 : EX1(read);
6304 op2 : EX1(read);
6305 INS01 : ISS;
6306 ALU : EX2;
6307 %}
6308
6309 // Compare reg-reg
6310 // Eg. CMP x0, #N
6311 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6312 %{
6313 single_instruction;
6314 // fixed_latency(16);
6315 cr : EX2(write);
6316 op1 : EX1(read);
6317 INS01 : ISS;
6318 ALU : EX2;
6319 %}
6320
6321 //------- Conditional instructions ------------------------
6322
6323 // Conditional no operands
6324 // Eg. CSINC x0, zr, zr, <cond>
6325 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6326 %{
6327 single_instruction;
6328 cr : EX1(read);
6329 dst : EX2(write);
6330 INS01 : ISS;
6331 ALU : EX2;
6332 %}
6333
6334 // Conditional 2 operand
6335 // EG. CSEL X0, X1, X2, <cond>
6336 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6337 %{
6338 single_instruction;
6339 cr : EX1(read);
6340 src1 : EX1(read);
6341 src2 : EX1(read);
6342 dst : EX2(write);
6343 INS01 : ISS;
6344 ALU : EX2;
6345 %}
6346
6347 // Conditional 2 operand
6348 // EG. CSEL X0, X1, X2, <cond>
6349 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6350 %{
6351 single_instruction;
6352 cr : EX1(read);
6353 src : EX1(read);
6354 dst : EX2(write);
6355 INS01 : ISS;
6356 ALU : EX2;
6357 %}
6358
6359 //------- Multiply pipeline operations --------------------
6360
6361 // Multiply reg-reg
6362 // Eg. MUL w0, w1, w2
6363 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6364 %{
6365 single_instruction;
6366 dst : WR(write);
6367 src1 : ISS(read);
6368 src2 : ISS(read);
6369 INS01 : ISS;
6370 MAC : WR;
6371 %}
6372
6373 // Multiply accumulate
6374 // Eg. MADD w0, w1, w2, w3
6375 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6376 %{
6377 single_instruction;
6378 dst : WR(write);
6379 src1 : ISS(read);
6380 src2 : ISS(read);
6381 src3 : ISS(read);
6382 INS01 : ISS;
6383 MAC : WR;
6384 %}
6385
6386 // Eg. MUL w0, w1, w2
6387 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6388 %{
6389 single_instruction;
6390 fixed_latency(3); // Maximum latency for 64 bit mul
6391 dst : WR(write);
6392 src1 : ISS(read);
6393 src2 : ISS(read);
6394 INS01 : ISS;
6395 MAC : WR;
6396 %}
6397
6398 // Multiply accumulate
6399 // Eg. MADD w0, w1, w2, w3
6400 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6401 %{
6402 single_instruction;
6403 fixed_latency(3); // Maximum latency for 64 bit mul
6404 dst : WR(write);
6405 src1 : ISS(read);
6406 src2 : ISS(read);
6407 src3 : ISS(read);
6408 INS01 : ISS;
6409 MAC : WR;
6410 %}
6411
6412 //------- Divide pipeline operations --------------------
6413
6414 // Eg. SDIV w0, w1, w2
6415 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6416 %{
6417 single_instruction;
6418 fixed_latency(8); // Maximum latency for 32 bit divide
6419 dst : WR(write);
6420 src1 : ISS(read);
6421 src2 : ISS(read);
6422 INS0 : ISS; // Can only dual issue as instruction 0
6423 DIV : WR;
6424 %}
6425
6426 // Eg. SDIV x0, x1, x2
6427 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6428 %{
6429 single_instruction;
6430 fixed_latency(16); // Maximum latency for 64 bit divide
6431 dst : WR(write);
6432 src1 : ISS(read);
6433 src2 : ISS(read);
6434 INS0 : ISS; // Can only dual issue as instruction 0
6435 DIV : WR;
6436 %}
6437
6438 //------- Load pipeline operations ------------------------
6439
6440 // Load - prefetch
6441 // Eg. PFRM <mem>
6442 pipe_class iload_prefetch(memory mem)
6443 %{
6444 single_instruction;
6445 mem : ISS(read);
6446 INS01 : ISS;
6447 LDST : WR;
6448 %}
6449
6450 // Load - reg, mem
6451 // Eg. LDR x0, <mem>
6452 pipe_class iload_reg_mem(iRegI dst, memory mem)
6453 %{
6454 single_instruction;
6455 dst : WR(write);
6456 mem : ISS(read);
6457 INS01 : ISS;
6458 LDST : WR;
6459 %}
6460
6461 // Load - reg, reg
6462 // Eg. LDR x0, [sp, x1]
6463 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6464 %{
6465 single_instruction;
6466 dst : WR(write);
6467 src : ISS(read);
6468 INS01 : ISS;
6469 LDST : WR;
6470 %}
6471
6472 //------- Store pipeline operations -----------------------
6473
6474 // Store - zr, mem
6475 // Eg. STR zr, <mem>
6476 pipe_class istore_mem(memory mem)
6477 %{
6478 single_instruction;
6479 mem : ISS(read);
6480 INS01 : ISS;
6481 LDST : WR;
6482 %}
6483
6484 // Store - reg, mem
6485 // Eg. STR x0, <mem>
6486 pipe_class istore_reg_mem(iRegI src, memory mem)
6487 %{
6488 single_instruction;
6489 mem : ISS(read);
6490 src : EX2(read);
6491 INS01 : ISS;
6492 LDST : WR;
6493 %}
6494
6495 // Store - reg, reg
6496 // Eg. STR x0, [sp, x1]
6497 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6498 %{
6499 single_instruction;
6500 dst : ISS(read);
6501 src : EX2(read);
6502 INS01 : ISS;
6503 LDST : WR;
6504 %}
6505
6506 //------- Store pipeline operations -----------------------
6507
6508 // Branch
6509 pipe_class pipe_branch()
6510 %{
6511 single_instruction;
6512 INS01 : ISS;
6513 BRANCH : EX1;
6514 %}
6515
6516 // Conditional branch
6517 pipe_class pipe_branch_cond(rFlagsReg cr)
6518 %{
6519 single_instruction;
6520 cr : EX1(read);
6521 INS01 : ISS;
6522 BRANCH : EX1;
6523 %}
6524
6525 // Compare & Branch
6526 // EG. CBZ/CBNZ
6527 pipe_class pipe_cmp_branch(iRegI op1)
6528 %{
6529 single_instruction;
6530 op1 : EX1(read);
6531 INS01 : ISS;
6532 BRANCH : EX1;
6533 %}
6534
6535 //------- Synchronisation operations ----------------------
6536
6537 // Any operation requiring serialization.
6538 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6539 pipe_class pipe_serial()
6540 %{
6541 single_instruction;
6542 force_serialization;
6543 fixed_latency(16);
6544 INS01 : ISS(2); // Cannot dual issue with any other instruction
6545 LDST : WR;
6546 %}
6547
6548 // Generic big/slow expanded idiom - also serialized
6549 pipe_class pipe_slow()
6550 %{
6551 instruction_count(10);
6552 multiple_bundles;
6553 force_serialization;
6554 fixed_latency(16);
6555 INS01 : ISS(2); // Cannot dual issue with any other instruction
6556 LDST : WR;
6557 %}
6558
6559 // Empty pipeline class
6560 pipe_class pipe_class_empty()
6561 %{
6562 single_instruction;
6563 fixed_latency(0);
6564 %}
6565
6566 // Default pipeline class.
6567 pipe_class pipe_class_default()
6568 %{
6569 single_instruction;
6570 fixed_latency(2);
6571 %}
6572
6573 // Pipeline class for compares.
6574 pipe_class pipe_class_compare()
6575 %{
6576 single_instruction;
6577 fixed_latency(16);
6578 %}
6579
6580 // Pipeline class for memory operations.
6581 pipe_class pipe_class_memory()
6582 %{
6583 single_instruction;
6584 fixed_latency(16);
6585 %}
6586
6587 // Pipeline class for call.
6588 pipe_class pipe_class_call()
6589 %{
6590 single_instruction;
6591 fixed_latency(100);
6592 %}
6593
6594 // Define the class for the Nop node.
6595 define %{
6596 MachNop = pipe_class_empty;
6597 %}
6598
6599 %}
6600 //----------INSTRUCTIONS-------------------------------------------------------
6601 //
6602 // match -- States which machine-independent subtree may be replaced
6603 // by this instruction.
6604 // ins_cost -- The estimated cost of this instruction is used by instruction
6605 // selection to identify a minimum cost tree of machine
6606 // instructions that matches a tree of machine-independent
6607 // instructions.
6608 // format -- A string providing the disassembly for this instruction.
6609 // The value of an instruction's operand may be inserted
6610 // by referring to it with a '$' prefix.
6611 // opcode -- Three instruction opcodes may be provided. These are referred
6612 // to within an encode class as $primary, $secondary, and $tertiary
6613 // rrspectively. The primary opcode is commonly used to
6614 // indicate the type of machine instruction, while secondary
6615 // and tertiary are often used for prefix options or addressing
6616 // modes.
6617 // ins_encode -- A list of encode classes with parameters. The encode class
6618 // name must have been defined in an 'enc_class' specification
6619 // in the encode section of the architecture description.
6620
6621 // ============================================================================
6622 // Memory (Load/Store) Instructions
6623
6624 // Load Instructions
6625
6626 // Load Byte (8 bit signed)
6627 instruct loadB(iRegINoSp dst, memory1 mem)
6628 %{
6629 match(Set dst (LoadB mem));
6630 predicate(!needs_acquiring_load(n));
6631
6632 ins_cost(4 * INSN_COST);
6633 format %{ "ldrsbw $dst, $mem\t# byte" %}
6634
6635 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6636
6637 ins_pipe(iload_reg_mem);
6638 %}
6639
6640 // Load Byte (8 bit signed) into long
6641 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6642 %{
6643 match(Set dst (ConvI2L (LoadB mem)));
6644 predicate(!needs_acquiring_load(n->in(1)));
6645
6646 ins_cost(4 * INSN_COST);
6647 format %{ "ldrsb $dst, $mem\t# byte" %}
6648
6649 ins_encode(aarch64_enc_ldrsb(dst, mem));
6650
6651 ins_pipe(iload_reg_mem);
6652 %}
6653
6654 // Load Byte (8 bit unsigned)
6655 instruct loadUB(iRegINoSp dst, memory1 mem)
6656 %{
6657 match(Set dst (LoadUB mem));
6658 predicate(!needs_acquiring_load(n));
6659
6660 ins_cost(4 * INSN_COST);
6661 format %{ "ldrbw $dst, $mem\t# byte" %}
6662
6663 ins_encode(aarch64_enc_ldrb(dst, mem));
6664
6665 ins_pipe(iload_reg_mem);
6666 %}
6667
6668 // Load Byte (8 bit unsigned) into long
6669 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6670 %{
6671 match(Set dst (ConvI2L (LoadUB mem)));
6672 predicate(!needs_acquiring_load(n->in(1)));
6673
6674 ins_cost(4 * INSN_COST);
6675 format %{ "ldrb $dst, $mem\t# byte" %}
6676
6677 ins_encode(aarch64_enc_ldrb(dst, mem));
6678
6679 ins_pipe(iload_reg_mem);
6680 %}
6681
6682 // Load Short (16 bit signed)
6683 instruct loadS(iRegINoSp dst, memory2 mem)
6684 %{
6685 match(Set dst (LoadS mem));
6686 predicate(!needs_acquiring_load(n));
6687
6688 ins_cost(4 * INSN_COST);
6689 format %{ "ldrshw $dst, $mem\t# short" %}
6690
6691 ins_encode(aarch64_enc_ldrshw(dst, mem));
6692
6693 ins_pipe(iload_reg_mem);
6694 %}
6695
6696 // Load Short (16 bit signed) into long
6697 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6698 %{
6699 match(Set dst (ConvI2L (LoadS mem)));
6700 predicate(!needs_acquiring_load(n->in(1)));
6701
6702 ins_cost(4 * INSN_COST);
6703 format %{ "ldrsh $dst, $mem\t# short" %}
6704
6705 ins_encode(aarch64_enc_ldrsh(dst, mem));
6706
6707 ins_pipe(iload_reg_mem);
6708 %}
6709
6710 // Load Char (16 bit unsigned)
6711 instruct loadUS(iRegINoSp dst, memory2 mem)
6712 %{
6713 match(Set dst (LoadUS mem));
6714 predicate(!needs_acquiring_load(n));
6715
6716 ins_cost(4 * INSN_COST);
6717 format %{ "ldrh $dst, $mem\t# short" %}
6718
6719 ins_encode(aarch64_enc_ldrh(dst, mem));
6720
6721 ins_pipe(iload_reg_mem);
6722 %}
6723
6724 // Load Short/Char (16 bit unsigned) into long
6725 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6726 %{
6727 match(Set dst (ConvI2L (LoadUS mem)));
6728 predicate(!needs_acquiring_load(n->in(1)));
6729
6730 ins_cost(4 * INSN_COST);
6731 format %{ "ldrh $dst, $mem\t# short" %}
6732
6733 ins_encode(aarch64_enc_ldrh(dst, mem));
6734
6735 ins_pipe(iload_reg_mem);
6736 %}
6737
6738 // Load Integer (32 bit signed)
6739 instruct loadI(iRegINoSp dst, memory4 mem)
6740 %{
6741 match(Set dst (LoadI mem));
6742 predicate(!needs_acquiring_load(n));
6743
6744 ins_cost(4 * INSN_COST);
6745 format %{ "ldrw $dst, $mem\t# int" %}
6746
6747 ins_encode(aarch64_enc_ldrw(dst, mem));
6748
6749 ins_pipe(iload_reg_mem);
6750 %}
6751
6752 // Load Integer (32 bit signed) into long
6753 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6754 %{
6755 match(Set dst (ConvI2L (LoadI mem)));
6756 predicate(!needs_acquiring_load(n->in(1)));
6757
6758 ins_cost(4 * INSN_COST);
6759 format %{ "ldrsw $dst, $mem\t# int" %}
6760
6761 ins_encode(aarch64_enc_ldrsw(dst, mem));
6762
6763 ins_pipe(iload_reg_mem);
6764 %}
6765
6766 // Load Integer (32 bit unsigned) into long
6767 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6768 %{
6769 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6770 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6771
6772 ins_cost(4 * INSN_COST);
6773 format %{ "ldrw $dst, $mem\t# int" %}
6774
6775 ins_encode(aarch64_enc_ldrw(dst, mem));
6776
6777 ins_pipe(iload_reg_mem);
6778 %}
6779
6780 // Load Long (64 bit signed)
6781 instruct loadL(iRegLNoSp dst, memory8 mem)
6782 %{
6783 match(Set dst (LoadL mem));
6784 predicate(!needs_acquiring_load(n));
6785
6786 ins_cost(4 * INSN_COST);
6787 format %{ "ldr $dst, $mem\t# int" %}
6788
6789 ins_encode(aarch64_enc_ldr(dst, mem));
6790
6791 ins_pipe(iload_reg_mem);
6792 %}
6793
6794 // Load Range
6795 instruct loadRange(iRegINoSp dst, memory4 mem)
6796 %{
6797 match(Set dst (LoadRange mem));
6798
6799 ins_cost(4 * INSN_COST);
6800 format %{ "ldrw $dst, $mem\t# range" %}
6801
6802 ins_encode(aarch64_enc_ldrw(dst, mem));
6803
6804 ins_pipe(iload_reg_mem);
6805 %}
6806
6807 // Load Pointer
6808 instruct loadP(iRegPNoSp dst, memory8 mem)
6809 %{
6810 match(Set dst (LoadP mem));
6811 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6812
6813 ins_cost(4 * INSN_COST);
6814 format %{ "ldr $dst, $mem\t# ptr" %}
6815
6816 ins_encode(aarch64_enc_ldr(dst, mem));
6817
6818 ins_pipe(iload_reg_mem);
6819 %}
6820
6821 // Load Compressed Pointer
6822 instruct loadN(iRegNNoSp dst, memory4 mem)
6823 %{
6824 match(Set dst (LoadN mem));
6825 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6826
6827 ins_cost(4 * INSN_COST);
6828 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6829
6830 ins_encode(aarch64_enc_ldrw(dst, mem));
6831
6832 ins_pipe(iload_reg_mem);
6833 %}
6834
6835 // Load Klass Pointer
6836 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6837 %{
6838 match(Set dst (LoadKlass mem));
6839 predicate(!needs_acquiring_load(n));
6840
6841 ins_cost(4 * INSN_COST);
6842 format %{ "ldr $dst, $mem\t# class" %}
6843
6844 ins_encode(aarch64_enc_ldr(dst, mem));
6845
6846 ins_pipe(iload_reg_mem);
6847 %}
6848
6849 // Load Narrow Klass Pointer
6850 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6851 %{
6852 match(Set dst (LoadNKlass mem));
6853 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6854
6855 ins_cost(4 * INSN_COST);
6856 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6857
6858 ins_encode(aarch64_enc_ldrw(dst, mem));
6859
6860 ins_pipe(iload_reg_mem);
6861 %}
6862
6863 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6864 %{
6865 match(Set dst (LoadNKlass mem));
6866 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6867
6868 ins_cost(4 * INSN_COST);
6869 format %{
6870 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6871 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6872 %}
6873 ins_encode %{
6874 // inlined aarch64_enc_ldrw
6875 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6876 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6877 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6878 %}
6879 ins_pipe(iload_reg_mem);
6880 %}
6881
6882 // Load Float
6883 instruct loadF(vRegF dst, memory4 mem)
6884 %{
6885 match(Set dst (LoadF mem));
6886 predicate(!needs_acquiring_load(n));
6887
6888 ins_cost(4 * INSN_COST);
6889 format %{ "ldrs $dst, $mem\t# float" %}
6890
6891 ins_encode( aarch64_enc_ldrs(dst, mem) );
6892
6893 ins_pipe(pipe_class_memory);
6894 %}
6895
6896 // Load Double
6897 instruct loadD(vRegD dst, memory8 mem)
6898 %{
6899 match(Set dst (LoadD mem));
6900 predicate(!needs_acquiring_load(n));
6901
6902 ins_cost(4 * INSN_COST);
6903 format %{ "ldrd $dst, $mem\t# double" %}
6904
6905 ins_encode( aarch64_enc_ldrd(dst, mem) );
6906
6907 ins_pipe(pipe_class_memory);
6908 %}
6909
6910
6911 // Load Int Constant
6912 instruct loadConI(iRegINoSp dst, immI src)
6913 %{
6914 match(Set dst src);
6915
6916 ins_cost(INSN_COST);
6917 format %{ "mov $dst, $src\t# int" %}
6918
6919 ins_encode( aarch64_enc_movw_imm(dst, src) );
6920
6921 ins_pipe(ialu_imm);
6922 %}
6923
6924 // Load Long Constant
6925 instruct loadConL(iRegLNoSp dst, immL src)
6926 %{
6927 match(Set dst src);
6928
6929 ins_cost(INSN_COST);
6930 format %{ "mov $dst, $src\t# long" %}
6931
6932 ins_encode( aarch64_enc_mov_imm(dst, src) );
6933
6934 ins_pipe(ialu_imm);
6935 %}
6936
6937 // Load Pointer Constant
6938
6939 instruct loadConP(iRegPNoSp dst, immP con)
6940 %{
6941 match(Set dst con);
6942
6943 ins_cost(INSN_COST * 4);
6944 format %{
6945 "mov $dst, $con\t# ptr"
6946 %}
6947
6948 ins_encode(aarch64_enc_mov_p(dst, con));
6949
6950 ins_pipe(ialu_imm);
6951 %}
6952
6953 // Load Null Pointer Constant
6954
6955 instruct loadConP0(iRegPNoSp dst, immP0 con)
6956 %{
6957 match(Set dst con);
6958
6959 ins_cost(INSN_COST);
6960 format %{ "mov $dst, $con\t# nullptr ptr" %}
6961
6962 ins_encode(aarch64_enc_mov_p0(dst, con));
6963
6964 ins_pipe(ialu_imm);
6965 %}
6966
6967 // Load Pointer Constant One
6968
6969 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6970 %{
6971 match(Set dst con);
6972
6973 ins_cost(INSN_COST);
6974 format %{ "mov $dst, $con\t# nullptr ptr" %}
6975
6976 ins_encode(aarch64_enc_mov_p1(dst, con));
6977
6978 ins_pipe(ialu_imm);
6979 %}
6980
6981 // Load Narrow Pointer Constant
6982
6983 instruct loadConN(iRegNNoSp dst, immN con)
6984 %{
6985 match(Set dst con);
6986
6987 ins_cost(INSN_COST * 4);
6988 format %{ "mov $dst, $con\t# compressed ptr" %}
6989
6990 ins_encode(aarch64_enc_mov_n(dst, con));
6991
6992 ins_pipe(ialu_imm);
6993 %}
6994
6995 // Load Narrow Null Pointer Constant
6996
6997 instruct loadConN0(iRegNNoSp dst, immN0 con)
6998 %{
6999 match(Set dst con);
7000
7001 ins_cost(INSN_COST);
7002 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
7003
7004 ins_encode(aarch64_enc_mov_n0(dst, con));
7005
7006 ins_pipe(ialu_imm);
7007 %}
7008
7009 // Load Narrow Klass Constant
7010
7011 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7012 %{
7013 match(Set dst con);
7014
7015 ins_cost(INSN_COST);
7016 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7017
7018 ins_encode(aarch64_enc_mov_nk(dst, con));
7019
7020 ins_pipe(ialu_imm);
7021 %}
7022
7023 // Load Packed Float Constant
7024
7025 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7026 match(Set dst con);
7027 ins_cost(INSN_COST * 4);
7028 format %{ "fmovs $dst, $con"%}
7029 ins_encode %{
7030 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7031 %}
7032
7033 ins_pipe(fp_imm_s);
7034 %}
7035
7036 // Load Float Constant
7037
7038 instruct loadConF(vRegF dst, immF con) %{
7039 match(Set dst con);
7040
7041 ins_cost(INSN_COST * 4);
7042
7043 format %{
7044 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7045 %}
7046
7047 ins_encode %{
7048 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7049 %}
7050
7051 ins_pipe(fp_load_constant_s);
7052 %}
7053
7054 // Load Packed Double Constant
7055
7056 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7057 match(Set dst con);
7058 ins_cost(INSN_COST);
7059 format %{ "fmovd $dst, $con"%}
7060 ins_encode %{
7061 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7062 %}
7063
7064 ins_pipe(fp_imm_d);
7065 %}
7066
7067 // Load Double Constant
7068
7069 instruct loadConD(vRegD dst, immD con) %{
7070 match(Set dst con);
7071
7072 ins_cost(INSN_COST * 5);
7073 format %{
7074 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7075 %}
7076
7077 ins_encode %{
7078 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7079 %}
7080
7081 ins_pipe(fp_load_constant_d);
7082 %}
7083
7084 // Load Half Float Constant
7085 instruct loadConH(vRegF dst, immH con) %{
7086 match(Set dst con);
7087 format %{ "mov rscratch1, $con\n\t"
7088 "fmov $dst, rscratch1"
7089 %}
7090 ins_encode %{
7091 __ movw(rscratch1, (uint32_t)$con$$constant);
7092 __ fmovs($dst$$FloatRegister, rscratch1);
7093 %}
7094 ins_pipe(pipe_class_default);
7095 %}
7096
7097 // Store Instructions
7098
7099 // Store Byte
7100 instruct storeB(iRegIorL2I src, memory1 mem)
7101 %{
7102 match(Set mem (StoreB mem src));
7103 predicate(!needs_releasing_store(n));
7104
7105 ins_cost(INSN_COST);
7106 format %{ "strb $src, $mem\t# byte" %}
7107
7108 ins_encode(aarch64_enc_strb(src, mem));
7109
7110 ins_pipe(istore_reg_mem);
7111 %}
7112
7113
7114 instruct storeimmB0(immI0 zero, memory1 mem)
7115 %{
7116 match(Set mem (StoreB mem zero));
7117 predicate(!needs_releasing_store(n));
7118
7119 ins_cost(INSN_COST);
7120 format %{ "strb rscractch2, $mem\t# byte" %}
7121
7122 ins_encode(aarch64_enc_strb0(mem));
7123
7124 ins_pipe(istore_mem);
7125 %}
7126
7127 // Store Char/Short
7128 instruct storeC(iRegIorL2I src, memory2 mem)
7129 %{
7130 match(Set mem (StoreC mem src));
7131 predicate(!needs_releasing_store(n));
7132
7133 ins_cost(INSN_COST);
7134 format %{ "strh $src, $mem\t# short" %}
7135
7136 ins_encode(aarch64_enc_strh(src, mem));
7137
7138 ins_pipe(istore_reg_mem);
7139 %}
7140
7141 instruct storeimmC0(immI0 zero, memory2 mem)
7142 %{
7143 match(Set mem (StoreC mem zero));
7144 predicate(!needs_releasing_store(n));
7145
7146 ins_cost(INSN_COST);
7147 format %{ "strh zr, $mem\t# short" %}
7148
7149 ins_encode(aarch64_enc_strh0(mem));
7150
7151 ins_pipe(istore_mem);
7152 %}
7153
7154 // Store Integer
7155
7156 instruct storeI(iRegIorL2I src, memory4 mem)
7157 %{
7158 match(Set mem(StoreI mem src));
7159 predicate(!needs_releasing_store(n));
7160
7161 ins_cost(INSN_COST);
7162 format %{ "strw $src, $mem\t# int" %}
7163
7164 ins_encode(aarch64_enc_strw(src, mem));
7165
7166 ins_pipe(istore_reg_mem);
7167 %}
7168
7169 instruct storeimmI0(immI0 zero, memory4 mem)
7170 %{
7171 match(Set mem(StoreI mem zero));
7172 predicate(!needs_releasing_store(n));
7173
7174 ins_cost(INSN_COST);
7175 format %{ "strw zr, $mem\t# int" %}
7176
7177 ins_encode(aarch64_enc_strw0(mem));
7178
7179 ins_pipe(istore_mem);
7180 %}
7181
7182 // Store Long (64 bit signed)
7183 instruct storeL(iRegL src, memory8 mem)
7184 %{
7185 match(Set mem (StoreL mem src));
7186 predicate(!needs_releasing_store(n));
7187
7188 ins_cost(INSN_COST);
7189 format %{ "str $src, $mem\t# int" %}
7190
7191 ins_encode(aarch64_enc_str(src, mem));
7192
7193 ins_pipe(istore_reg_mem);
7194 %}
7195
7196 // Store Long (64 bit signed)
7197 instruct storeimmL0(immL0 zero, memory8 mem)
7198 %{
7199 match(Set mem (StoreL mem zero));
7200 predicate(!needs_releasing_store(n));
7201
7202 ins_cost(INSN_COST);
7203 format %{ "str zr, $mem\t# int" %}
7204
7205 ins_encode(aarch64_enc_str0(mem));
7206
7207 ins_pipe(istore_mem);
7208 %}
7209
7210 // Store Pointer
7211 instruct storeP(iRegP src, memory8 mem)
7212 %{
7213 match(Set mem (StoreP mem src));
7214 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7215
7216 ins_cost(INSN_COST);
7217 format %{ "str $src, $mem\t# ptr" %}
7218
7219 ins_encode(aarch64_enc_str(src, mem));
7220
7221 ins_pipe(istore_reg_mem);
7222 %}
7223
7224 // Store Pointer
7225 instruct storeimmP0(immP0 zero, memory8 mem)
7226 %{
7227 match(Set mem (StoreP mem zero));
7228 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7229
7230 ins_cost(INSN_COST);
7231 format %{ "str zr, $mem\t# ptr" %}
7232
7233 ins_encode(aarch64_enc_str0(mem));
7234
7235 ins_pipe(istore_mem);
7236 %}
7237
7238 // Store Compressed Pointer
7239 instruct storeN(iRegN src, memory4 mem)
7240 %{
7241 match(Set mem (StoreN mem src));
7242 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7243
7244 ins_cost(INSN_COST);
7245 format %{ "strw $src, $mem\t# compressed ptr" %}
7246
7247 ins_encode(aarch64_enc_strw(src, mem));
7248
7249 ins_pipe(istore_reg_mem);
7250 %}
7251
7252 instruct storeImmN0(immN0 zero, memory4 mem)
7253 %{
7254 match(Set mem (StoreN mem zero));
7255 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7256
7257 ins_cost(INSN_COST);
7258 format %{ "strw zr, $mem\t# compressed ptr" %}
7259
7260 ins_encode(aarch64_enc_strw0(mem));
7261
7262 ins_pipe(istore_mem);
7263 %}
7264
7265 // Store Float
7266 instruct storeF(vRegF src, memory4 mem)
7267 %{
7268 match(Set mem (StoreF mem src));
7269 predicate(!needs_releasing_store(n));
7270
7271 ins_cost(INSN_COST);
7272 format %{ "strs $src, $mem\t# float" %}
7273
7274 ins_encode( aarch64_enc_strs(src, mem) );
7275
7276 ins_pipe(pipe_class_memory);
7277 %}
7278
7279 // TODO
7280 // implement storeImmF0 and storeFImmPacked
7281
7282 // Store Double
7283 instruct storeD(vRegD src, memory8 mem)
7284 %{
7285 match(Set mem (StoreD mem src));
7286 predicate(!needs_releasing_store(n));
7287
7288 ins_cost(INSN_COST);
7289 format %{ "strd $src, $mem\t# double" %}
7290
7291 ins_encode( aarch64_enc_strd(src, mem) );
7292
7293 ins_pipe(pipe_class_memory);
7294 %}
7295
7296 // Store Compressed Klass Pointer
7297 instruct storeNKlass(iRegN src, memory4 mem)
7298 %{
7299 predicate(!needs_releasing_store(n));
7300 match(Set mem (StoreNKlass mem src));
7301
7302 ins_cost(INSN_COST);
7303 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7304
7305 ins_encode(aarch64_enc_strw(src, mem));
7306
7307 ins_pipe(istore_reg_mem);
7308 %}
7309
7310 // TODO
7311 // implement storeImmD0 and storeDImmPacked
7312
7313 // prefetch instructions
7314 // Must be safe to execute with invalid address (cannot fault).
7315
7316 instruct prefetchalloc( memory8 mem ) %{
7317 match(PrefetchAllocation mem);
7318
7319 ins_cost(INSN_COST);
7320 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7321
7322 ins_encode( aarch64_enc_prefetchw(mem) );
7323
7324 ins_pipe(iload_prefetch);
7325 %}
7326
7327 // ---------------- volatile loads and stores ----------------
7328
7329 // Load Byte (8 bit signed)
7330 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7331 %{
7332 match(Set dst (LoadB mem));
7333
7334 ins_cost(VOLATILE_REF_COST);
7335 format %{ "ldarsb $dst, $mem\t# byte" %}
7336
7337 ins_encode(aarch64_enc_ldarsb(dst, mem));
7338
7339 ins_pipe(pipe_serial);
7340 %}
7341
7342 // Load Byte (8 bit signed) into long
7343 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7344 %{
7345 match(Set dst (ConvI2L (LoadB mem)));
7346
7347 ins_cost(VOLATILE_REF_COST);
7348 format %{ "ldarsb $dst, $mem\t# byte" %}
7349
7350 ins_encode(aarch64_enc_ldarsb(dst, mem));
7351
7352 ins_pipe(pipe_serial);
7353 %}
7354
7355 // Load Byte (8 bit unsigned)
7356 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7357 %{
7358 match(Set dst (LoadUB mem));
7359
7360 ins_cost(VOLATILE_REF_COST);
7361 format %{ "ldarb $dst, $mem\t# byte" %}
7362
7363 ins_encode(aarch64_enc_ldarb(dst, mem));
7364
7365 ins_pipe(pipe_serial);
7366 %}
7367
7368 // Load Byte (8 bit unsigned) into long
7369 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7370 %{
7371 match(Set dst (ConvI2L (LoadUB mem)));
7372
7373 ins_cost(VOLATILE_REF_COST);
7374 format %{ "ldarb $dst, $mem\t# byte" %}
7375
7376 ins_encode(aarch64_enc_ldarb(dst, mem));
7377
7378 ins_pipe(pipe_serial);
7379 %}
7380
7381 // Load Short (16 bit signed)
7382 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7383 %{
7384 match(Set dst (LoadS mem));
7385
7386 ins_cost(VOLATILE_REF_COST);
7387 format %{ "ldarshw $dst, $mem\t# short" %}
7388
7389 ins_encode(aarch64_enc_ldarshw(dst, mem));
7390
7391 ins_pipe(pipe_serial);
7392 %}
7393
7394 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7395 %{
7396 match(Set dst (LoadUS mem));
7397
7398 ins_cost(VOLATILE_REF_COST);
7399 format %{ "ldarhw $dst, $mem\t# short" %}
7400
7401 ins_encode(aarch64_enc_ldarhw(dst, mem));
7402
7403 ins_pipe(pipe_serial);
7404 %}
7405
7406 // Load Short/Char (16 bit unsigned) into long
7407 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7408 %{
7409 match(Set dst (ConvI2L (LoadUS mem)));
7410
7411 ins_cost(VOLATILE_REF_COST);
7412 format %{ "ldarh $dst, $mem\t# short" %}
7413
7414 ins_encode(aarch64_enc_ldarh(dst, mem));
7415
7416 ins_pipe(pipe_serial);
7417 %}
7418
7419 // Load Short/Char (16 bit signed) into long
7420 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7421 %{
7422 match(Set dst (ConvI2L (LoadS mem)));
7423
7424 ins_cost(VOLATILE_REF_COST);
7425 format %{ "ldarh $dst, $mem\t# short" %}
7426
7427 ins_encode(aarch64_enc_ldarsh(dst, mem));
7428
7429 ins_pipe(pipe_serial);
7430 %}
7431
7432 // Load Integer (32 bit signed)
7433 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7434 %{
7435 match(Set dst (LoadI mem));
7436
7437 ins_cost(VOLATILE_REF_COST);
7438 format %{ "ldarw $dst, $mem\t# int" %}
7439
7440 ins_encode(aarch64_enc_ldarw(dst, mem));
7441
7442 ins_pipe(pipe_serial);
7443 %}
7444
7445 // Load Integer (32 bit unsigned) into long
7446 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7447 %{
7448 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7449
7450 ins_cost(VOLATILE_REF_COST);
7451 format %{ "ldarw $dst, $mem\t# int" %}
7452
7453 ins_encode(aarch64_enc_ldarw(dst, mem));
7454
7455 ins_pipe(pipe_serial);
7456 %}
7457
7458 // Load Long (64 bit signed)
7459 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7460 %{
7461 match(Set dst (LoadL mem));
7462
7463 ins_cost(VOLATILE_REF_COST);
7464 format %{ "ldar $dst, $mem\t# int" %}
7465
7466 ins_encode(aarch64_enc_ldar(dst, mem));
7467
7468 ins_pipe(pipe_serial);
7469 %}
7470
7471 // Load Pointer
7472 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7473 %{
7474 match(Set dst (LoadP mem));
7475 predicate(n->as_Load()->barrier_data() == 0);
7476
7477 ins_cost(VOLATILE_REF_COST);
7478 format %{ "ldar $dst, $mem\t# ptr" %}
7479
7480 ins_encode(aarch64_enc_ldar(dst, mem));
7481
7482 ins_pipe(pipe_serial);
7483 %}
7484
7485 // Load Compressed Pointer
7486 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7487 %{
7488 match(Set dst (LoadN mem));
7489 predicate(n->as_Load()->barrier_data() == 0);
7490
7491 ins_cost(VOLATILE_REF_COST);
7492 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7493
7494 ins_encode(aarch64_enc_ldarw(dst, mem));
7495
7496 ins_pipe(pipe_serial);
7497 %}
7498
7499 // Load Float
7500 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7501 %{
7502 match(Set dst (LoadF mem));
7503
7504 ins_cost(VOLATILE_REF_COST);
7505 format %{ "ldars $dst, $mem\t# float" %}
7506
7507 ins_encode( aarch64_enc_fldars(dst, mem) );
7508
7509 ins_pipe(pipe_serial);
7510 %}
7511
7512 // Load Double
7513 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7514 %{
7515 match(Set dst (LoadD mem));
7516
7517 ins_cost(VOLATILE_REF_COST);
7518 format %{ "ldard $dst, $mem\t# double" %}
7519
7520 ins_encode( aarch64_enc_fldard(dst, mem) );
7521
7522 ins_pipe(pipe_serial);
7523 %}
7524
7525 // Store Byte
7526 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7527 %{
7528 match(Set mem (StoreB mem src));
7529
7530 ins_cost(VOLATILE_REF_COST);
7531 format %{ "stlrb $src, $mem\t# byte" %}
7532
7533 ins_encode(aarch64_enc_stlrb(src, mem));
7534
7535 ins_pipe(pipe_class_memory);
7536 %}
7537
7538 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7539 %{
7540 match(Set mem (StoreB mem zero));
7541
7542 ins_cost(VOLATILE_REF_COST);
7543 format %{ "stlrb zr, $mem\t# byte" %}
7544
7545 ins_encode(aarch64_enc_stlrb0(mem));
7546
7547 ins_pipe(pipe_class_memory);
7548 %}
7549
7550 // Store Char/Short
7551 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7552 %{
7553 match(Set mem (StoreC mem src));
7554
7555 ins_cost(VOLATILE_REF_COST);
7556 format %{ "stlrh $src, $mem\t# short" %}
7557
7558 ins_encode(aarch64_enc_stlrh(src, mem));
7559
7560 ins_pipe(pipe_class_memory);
7561 %}
7562
7563 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7564 %{
7565 match(Set mem (StoreC mem zero));
7566
7567 ins_cost(VOLATILE_REF_COST);
7568 format %{ "stlrh zr, $mem\t# short" %}
7569
7570 ins_encode(aarch64_enc_stlrh0(mem));
7571
7572 ins_pipe(pipe_class_memory);
7573 %}
7574
7575 // Store Integer
7576
7577 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7578 %{
7579 match(Set mem(StoreI mem src));
7580
7581 ins_cost(VOLATILE_REF_COST);
7582 format %{ "stlrw $src, $mem\t# int" %}
7583
7584 ins_encode(aarch64_enc_stlrw(src, mem));
7585
7586 ins_pipe(pipe_class_memory);
7587 %}
7588
7589 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7590 %{
7591 match(Set mem(StoreI mem zero));
7592
7593 ins_cost(VOLATILE_REF_COST);
7594 format %{ "stlrw zr, $mem\t# int" %}
7595
7596 ins_encode(aarch64_enc_stlrw0(mem));
7597
7598 ins_pipe(pipe_class_memory);
7599 %}
7600
7601 // Store Long (64 bit signed)
7602 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7603 %{
7604 match(Set mem (StoreL mem src));
7605
7606 ins_cost(VOLATILE_REF_COST);
7607 format %{ "stlr $src, $mem\t# int" %}
7608
7609 ins_encode(aarch64_enc_stlr(src, mem));
7610
7611 ins_pipe(pipe_class_memory);
7612 %}
7613
7614 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7615 %{
7616 match(Set mem (StoreL mem zero));
7617
7618 ins_cost(VOLATILE_REF_COST);
7619 format %{ "stlr zr, $mem\t# int" %}
7620
7621 ins_encode(aarch64_enc_stlr0(mem));
7622
7623 ins_pipe(pipe_class_memory);
7624 %}
7625
7626 // Store Pointer
7627 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7628 %{
7629 match(Set mem (StoreP mem src));
7630 predicate(n->as_Store()->barrier_data() == 0);
7631
7632 ins_cost(VOLATILE_REF_COST);
7633 format %{ "stlr $src, $mem\t# ptr" %}
7634
7635 ins_encode(aarch64_enc_stlr(src, mem));
7636
7637 ins_pipe(pipe_class_memory);
7638 %}
7639
7640 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7641 %{
7642 match(Set mem (StoreP mem zero));
7643 predicate(n->as_Store()->barrier_data() == 0);
7644
7645 ins_cost(VOLATILE_REF_COST);
7646 format %{ "stlr zr, $mem\t# ptr" %}
7647
7648 ins_encode(aarch64_enc_stlr0(mem));
7649
7650 ins_pipe(pipe_class_memory);
7651 %}
7652
7653 // Store Compressed Pointer
7654 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7655 %{
7656 match(Set mem (StoreN mem src));
7657 predicate(n->as_Store()->barrier_data() == 0);
7658
7659 ins_cost(VOLATILE_REF_COST);
7660 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7661
7662 ins_encode(aarch64_enc_stlrw(src, mem));
7663
7664 ins_pipe(pipe_class_memory);
7665 %}
7666
7667 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7668 %{
7669 match(Set mem (StoreN mem zero));
7670 predicate(n->as_Store()->barrier_data() == 0);
7671
7672 ins_cost(VOLATILE_REF_COST);
7673 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7674
7675 ins_encode(aarch64_enc_stlrw0(mem));
7676
7677 ins_pipe(pipe_class_memory);
7678 %}
7679
7680 // Store Float
7681 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7682 %{
7683 match(Set mem (StoreF mem src));
7684
7685 ins_cost(VOLATILE_REF_COST);
7686 format %{ "stlrs $src, $mem\t# float" %}
7687
7688 ins_encode( aarch64_enc_fstlrs(src, mem) );
7689
7690 ins_pipe(pipe_class_memory);
7691 %}
7692
7693 // TODO
7694 // implement storeImmF0 and storeFImmPacked
7695
7696 // Store Double
7697 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7698 %{
7699 match(Set mem (StoreD mem src));
7700
7701 ins_cost(VOLATILE_REF_COST);
7702 format %{ "stlrd $src, $mem\t# double" %}
7703
7704 ins_encode( aarch64_enc_fstlrd(src, mem) );
7705
7706 ins_pipe(pipe_class_memory);
7707 %}
7708
7709 // ---------------- end of volatile loads and stores ----------------
7710
7711 instruct cacheWB(indirect addr)
7712 %{
7713 predicate(VM_Version::supports_data_cache_line_flush());
7714 match(CacheWB addr);
7715
7716 ins_cost(100);
7717 format %{"cache wb $addr" %}
7718 ins_encode %{
7719 assert($addr->index_position() < 0, "should be");
7720 assert($addr$$disp == 0, "should be");
7721 __ cache_wb(Address($addr$$base$$Register, 0));
7722 %}
7723 ins_pipe(pipe_slow); // XXX
7724 %}
7725
7726 instruct cacheWBPreSync()
7727 %{
7728 predicate(VM_Version::supports_data_cache_line_flush());
7729 match(CacheWBPreSync);
7730
7731 ins_cost(100);
7732 format %{"cache wb presync" %}
7733 ins_encode %{
7734 __ cache_wbsync(true);
7735 %}
7736 ins_pipe(pipe_slow); // XXX
7737 %}
7738
7739 instruct cacheWBPostSync()
7740 %{
7741 predicate(VM_Version::supports_data_cache_line_flush());
7742 match(CacheWBPostSync);
7743
7744 ins_cost(100);
7745 format %{"cache wb postsync" %}
7746 ins_encode %{
7747 __ cache_wbsync(false);
7748 %}
7749 ins_pipe(pipe_slow); // XXX
7750 %}
7751
7752 // ============================================================================
7753 // BSWAP Instructions
7754
7755 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7756 match(Set dst (ReverseBytesI src));
7757
7758 ins_cost(INSN_COST);
7759 format %{ "revw $dst, $src" %}
7760
7761 ins_encode %{
7762 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7763 %}
7764
7765 ins_pipe(ialu_reg);
7766 %}
7767
7768 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7769 match(Set dst (ReverseBytesL src));
7770
7771 ins_cost(INSN_COST);
7772 format %{ "rev $dst, $src" %}
7773
7774 ins_encode %{
7775 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7776 %}
7777
7778 ins_pipe(ialu_reg);
7779 %}
7780
7781 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7782 match(Set dst (ReverseBytesUS src));
7783
7784 ins_cost(INSN_COST);
7785 format %{ "rev16w $dst, $src" %}
7786
7787 ins_encode %{
7788 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7789 %}
7790
7791 ins_pipe(ialu_reg);
7792 %}
7793
7794 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7795 match(Set dst (ReverseBytesS src));
7796
7797 ins_cost(INSN_COST);
7798 format %{ "rev16w $dst, $src\n\t"
7799 "sbfmw $dst, $dst, #0, #15" %}
7800
7801 ins_encode %{
7802 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7803 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7804 %}
7805
7806 ins_pipe(ialu_reg);
7807 %}
7808
7809 // ============================================================================
7810 // Zero Count Instructions
7811
7812 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7813 match(Set dst (CountLeadingZerosI src));
7814
7815 ins_cost(INSN_COST);
7816 format %{ "clzw $dst, $src" %}
7817 ins_encode %{
7818 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7819 %}
7820
7821 ins_pipe(ialu_reg);
7822 %}
7823
7824 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7825 match(Set dst (CountLeadingZerosL src));
7826
7827 ins_cost(INSN_COST);
7828 format %{ "clz $dst, $src" %}
7829 ins_encode %{
7830 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7831 %}
7832
7833 ins_pipe(ialu_reg);
7834 %}
7835
7836 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7837 match(Set dst (CountTrailingZerosI src));
7838
7839 ins_cost(INSN_COST * 2);
7840 format %{ "rbitw $dst, $src\n\t"
7841 "clzw $dst, $dst" %}
7842 ins_encode %{
7843 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7844 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7845 %}
7846
7847 ins_pipe(ialu_reg);
7848 %}
7849
7850 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7851 match(Set dst (CountTrailingZerosL src));
7852
7853 ins_cost(INSN_COST * 2);
7854 format %{ "rbit $dst, $src\n\t"
7855 "clz $dst, $dst" %}
7856 ins_encode %{
7857 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7858 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7859 %}
7860
7861 ins_pipe(ialu_reg);
7862 %}
7863
7864 //---------- Population Count Instructions -------------------------------------
7865 //
7866
7867 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7868 match(Set dst (PopCountI src));
7869 effect(TEMP tmp);
7870 ins_cost(INSN_COST * 13);
7871
7872 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7873 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7874 "addv $tmp, $tmp\t# vector (8B)\n\t"
7875 "mov $dst, $tmp\t# vector (1D)" %}
7876 ins_encode %{
7877 __ fmovs($tmp$$FloatRegister, $src$$Register);
7878 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7879 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7880 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7881 %}
7882
7883 ins_pipe(pipe_class_default);
7884 %}
7885
7886 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7887 match(Set dst (PopCountI (LoadI mem)));
7888 effect(TEMP tmp);
7889 ins_cost(INSN_COST * 13);
7890
7891 format %{ "ldrs $tmp, $mem\n\t"
7892 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7893 "addv $tmp, $tmp\t# vector (8B)\n\t"
7894 "mov $dst, $tmp\t# vector (1D)" %}
7895 ins_encode %{
7896 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7897 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7898 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7899 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7900 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7901 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7902 %}
7903
7904 ins_pipe(pipe_class_default);
7905 %}
7906
7907 // Note: Long.bitCount(long) returns an int.
7908 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7909 match(Set dst (PopCountL src));
7910 effect(TEMP tmp);
7911 ins_cost(INSN_COST * 13);
7912
7913 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7914 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7915 "addv $tmp, $tmp\t# vector (8B)\n\t"
7916 "mov $dst, $tmp\t# vector (1D)" %}
7917 ins_encode %{
7918 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7919 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7920 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7921 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7922 %}
7923
7924 ins_pipe(pipe_class_default);
7925 %}
7926
7927 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7928 match(Set dst (PopCountL (LoadL mem)));
7929 effect(TEMP tmp);
7930 ins_cost(INSN_COST * 13);
7931
7932 format %{ "ldrd $tmp, $mem\n\t"
7933 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7934 "addv $tmp, $tmp\t# vector (8B)\n\t"
7935 "mov $dst, $tmp\t# vector (1D)" %}
7936 ins_encode %{
7937 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7938 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7939 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7940 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7941 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7942 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7943 %}
7944
7945 ins_pipe(pipe_class_default);
7946 %}
7947
7948 // ============================================================================
7949 // VerifyVectorAlignment Instruction
7950
7951 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7952 match(Set addr (VerifyVectorAlignment addr mask));
7953 effect(KILL cr);
7954 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7955 ins_encode %{
7956 Label Lskip;
7957 // check if masked bits of addr are zero
7958 __ tst($addr$$Register, $mask$$constant);
7959 __ br(Assembler::EQ, Lskip);
7960 __ stop("verify_vector_alignment found a misaligned vector memory access");
7961 __ bind(Lskip);
7962 %}
7963 ins_pipe(pipe_slow);
7964 %}
7965
7966 // ============================================================================
7967 // MemBar Instruction
7968
7969 instruct load_fence() %{
7970 match(LoadFence);
7971 ins_cost(VOLATILE_REF_COST);
7972
7973 format %{ "load_fence" %}
7974
7975 ins_encode %{
7976 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7977 %}
7978 ins_pipe(pipe_serial);
7979 %}
7980
7981 instruct unnecessary_membar_acquire() %{
7982 predicate(unnecessary_acquire(n));
7983 match(MemBarAcquire);
7984 ins_cost(0);
7985
7986 format %{ "membar_acquire (elided)" %}
7987
7988 ins_encode %{
7989 __ block_comment("membar_acquire (elided)");
7990 %}
7991
7992 ins_pipe(pipe_class_empty);
7993 %}
7994
7995 instruct membar_acquire() %{
7996 match(MemBarAcquire);
7997 ins_cost(VOLATILE_REF_COST);
7998
7999 format %{ "membar_acquire\n\t"
8000 "dmb ishld" %}
8001
8002 ins_encode %{
8003 __ block_comment("membar_acquire");
8004 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8005 %}
8006
8007 ins_pipe(pipe_serial);
8008 %}
8009
8010
8011 instruct membar_acquire_lock() %{
8012 match(MemBarAcquireLock);
8013 ins_cost(VOLATILE_REF_COST);
8014
8015 format %{ "membar_acquire_lock (elided)" %}
8016
8017 ins_encode %{
8018 __ block_comment("membar_acquire_lock (elided)");
8019 %}
8020
8021 ins_pipe(pipe_serial);
8022 %}
8023
8024 instruct store_fence() %{
8025 match(StoreFence);
8026 ins_cost(VOLATILE_REF_COST);
8027
8028 format %{ "store_fence" %}
8029
8030 ins_encode %{
8031 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8032 %}
8033 ins_pipe(pipe_serial);
8034 %}
8035
8036 instruct unnecessary_membar_release() %{
8037 predicate(unnecessary_release(n));
8038 match(MemBarRelease);
8039 ins_cost(0);
8040
8041 format %{ "membar_release (elided)" %}
8042
8043 ins_encode %{
8044 __ block_comment("membar_release (elided)");
8045 %}
8046 ins_pipe(pipe_serial);
8047 %}
8048
8049 instruct membar_release() %{
8050 match(MemBarRelease);
8051 ins_cost(VOLATILE_REF_COST);
8052
8053 format %{ "membar_release\n\t"
8054 "dmb ishst\n\tdmb ishld" %}
8055
8056 ins_encode %{
8057 __ block_comment("membar_release");
8058 // These will be merged if AlwaysMergeDMB is enabled.
8059 __ membar(Assembler::StoreStore);
8060 __ membar(Assembler::LoadStore);
8061 %}
8062 ins_pipe(pipe_serial);
8063 %}
8064
8065 instruct membar_storestore() %{
8066 match(MemBarStoreStore);
8067 match(StoreStoreFence);
8068 ins_cost(VOLATILE_REF_COST);
8069
8070 format %{ "MEMBAR-store-store" %}
8071
8072 ins_encode %{
8073 __ membar(Assembler::StoreStore);
8074 %}
8075 ins_pipe(pipe_serial);
8076 %}
8077
8078 instruct membar_release_lock() %{
8079 match(MemBarReleaseLock);
8080 ins_cost(VOLATILE_REF_COST);
8081
8082 format %{ "membar_release_lock (elided)" %}
8083
8084 ins_encode %{
8085 __ block_comment("membar_release_lock (elided)");
8086 %}
8087
8088 ins_pipe(pipe_serial);
8089 %}
8090
8091 instruct unnecessary_membar_volatile() %{
8092 predicate(unnecessary_volatile(n));
8093 match(MemBarVolatile);
8094 ins_cost(0);
8095
8096 format %{ "membar_volatile (elided)" %}
8097
8098 ins_encode %{
8099 __ block_comment("membar_volatile (elided)");
8100 %}
8101
8102 ins_pipe(pipe_serial);
8103 %}
8104
8105 instruct membar_volatile() %{
8106 match(MemBarVolatile);
8107 ins_cost(VOLATILE_REF_COST*100);
8108
8109 format %{ "membar_volatile\n\t"
8110 "dmb ish"%}
8111
8112 ins_encode %{
8113 __ block_comment("membar_volatile");
8114 __ membar(Assembler::StoreLoad);
8115 %}
8116
8117 ins_pipe(pipe_serial);
8118 %}
8119
8120 // ============================================================================
8121 // Cast/Convert Instructions
8122
8123 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8124 match(Set dst (CastX2P src));
8125
8126 ins_cost(INSN_COST);
8127 format %{ "mov $dst, $src\t# long -> ptr" %}
8128
8129 ins_encode %{
8130 if ($dst$$reg != $src$$reg) {
8131 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8132 }
8133 %}
8134
8135 ins_pipe(ialu_reg);
8136 %}
8137
8138 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8139 match(Set dst (CastI2N src));
8140
8141 ins_cost(INSN_COST);
8142 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8143
8144 ins_encode %{
8145 if ($dst$$reg != $src$$reg) {
8146 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8147 }
8148 %}
8149
8150 ins_pipe(ialu_reg);
8151 %}
8152
8153 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8154 match(Set dst (CastP2X src));
8155
8156 ins_cost(INSN_COST);
8157 format %{ "mov $dst, $src\t# ptr -> long" %}
8158
8159 ins_encode %{
8160 if ($dst$$reg != $src$$reg) {
8161 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8162 }
8163 %}
8164
8165 ins_pipe(ialu_reg);
8166 %}
8167
8168 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8169 match(Set dst (CastP2X src));
8170
8171 ins_cost(INSN_COST);
8172 format %{ "mov $dst, $src\t# ptr -> long" %}
8173
8174 ins_encode %{
8175 if ($dst$$reg != $src$$reg) {
8176 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8177 }
8178 %}
8179
8180 ins_pipe(ialu_reg);
8181 %}
8182
8183 // Convert oop into int for vectors alignment masking
8184 instruct convP2I(iRegINoSp dst, iRegP src) %{
8185 match(Set dst (ConvL2I (CastP2X src)));
8186
8187 ins_cost(INSN_COST);
8188 format %{ "movw $dst, $src\t# ptr -> int" %}
8189 ins_encode %{
8190 __ movw($dst$$Register, $src$$Register);
8191 %}
8192
8193 ins_pipe(ialu_reg);
8194 %}
8195
8196 // Convert compressed oop into int for vectors alignment masking
8197 // in case of 32bit oops (heap < 4Gb).
8198 instruct convN2I(iRegINoSp dst, iRegN src)
8199 %{
8200 predicate(CompressedOops::shift() == 0);
8201 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8202
8203 ins_cost(INSN_COST);
8204 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8205 ins_encode %{
8206 __ movw($dst$$Register, $src$$Register);
8207 %}
8208
8209 ins_pipe(ialu_reg);
8210 %}
8211
8212
8213 // Convert oop pointer into compressed form
8214 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8215 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8216 match(Set dst (EncodeP src));
8217 effect(KILL cr);
8218 ins_cost(INSN_COST * 3);
8219 format %{ "encode_heap_oop $dst, $src" %}
8220 ins_encode %{
8221 Register s = $src$$Register;
8222 Register d = $dst$$Register;
8223 __ encode_heap_oop(d, s);
8224 %}
8225 ins_pipe(ialu_reg);
8226 %}
8227
8228 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8229 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8230 match(Set dst (EncodeP src));
8231 ins_cost(INSN_COST * 3);
8232 format %{ "encode_heap_oop_not_null $dst, $src" %}
8233 ins_encode %{
8234 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8235 %}
8236 ins_pipe(ialu_reg);
8237 %}
8238
8239 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8240 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8241 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8242 match(Set dst (DecodeN src));
8243 ins_cost(INSN_COST * 3);
8244 format %{ "decode_heap_oop $dst, $src" %}
8245 ins_encode %{
8246 Register s = $src$$Register;
8247 Register d = $dst$$Register;
8248 __ decode_heap_oop(d, s);
8249 %}
8250 ins_pipe(ialu_reg);
8251 %}
8252
8253 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8254 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8255 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8256 match(Set dst (DecodeN src));
8257 ins_cost(INSN_COST * 3);
8258 format %{ "decode_heap_oop_not_null $dst, $src" %}
8259 ins_encode %{
8260 Register s = $src$$Register;
8261 Register d = $dst$$Register;
8262 __ decode_heap_oop_not_null(d, s);
8263 %}
8264 ins_pipe(ialu_reg);
8265 %}
8266
8267 // n.b. AArch64 implementations of encode_klass_not_null and
8268 // decode_klass_not_null do not modify the flags register so, unlike
8269 // Intel, we don't kill CR as a side effect here
8270
8271 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8272 match(Set dst (EncodePKlass src));
8273
8274 ins_cost(INSN_COST * 3);
8275 format %{ "encode_klass_not_null $dst,$src" %}
8276
8277 ins_encode %{
8278 Register src_reg = as_Register($src$$reg);
8279 Register dst_reg = as_Register($dst$$reg);
8280 __ encode_klass_not_null(dst_reg, src_reg);
8281 %}
8282
8283 ins_pipe(ialu_reg);
8284 %}
8285
8286 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8287 match(Set dst (DecodeNKlass src));
8288
8289 ins_cost(INSN_COST * 3);
8290 format %{ "decode_klass_not_null $dst,$src" %}
8291
8292 ins_encode %{
8293 Register src_reg = as_Register($src$$reg);
8294 Register dst_reg = as_Register($dst$$reg);
8295 if (dst_reg != src_reg) {
8296 __ decode_klass_not_null(dst_reg, src_reg);
8297 } else {
8298 __ decode_klass_not_null(dst_reg);
8299 }
8300 %}
8301
8302 ins_pipe(ialu_reg);
8303 %}
8304
8305 instruct checkCastPP(iRegPNoSp dst)
8306 %{
8307 match(Set dst (CheckCastPP dst));
8308
8309 size(0);
8310 format %{ "# checkcastPP of $dst" %}
8311 ins_encode(/* empty encoding */);
8312 ins_pipe(pipe_class_empty);
8313 %}
8314
8315 instruct castPP(iRegPNoSp dst)
8316 %{
8317 match(Set dst (CastPP dst));
8318
8319 size(0);
8320 format %{ "# castPP of $dst" %}
8321 ins_encode(/* empty encoding */);
8322 ins_pipe(pipe_class_empty);
8323 %}
8324
8325 instruct castII(iRegI dst)
8326 %{
8327 predicate(VerifyConstraintCasts == 0);
8328 match(Set dst (CastII dst));
8329
8330 size(0);
8331 format %{ "# castII of $dst" %}
8332 ins_encode(/* empty encoding */);
8333 ins_cost(0);
8334 ins_pipe(pipe_class_empty);
8335 %}
8336
8337 instruct castII_checked(iRegI dst, rFlagsReg cr)
8338 %{
8339 predicate(VerifyConstraintCasts > 0);
8340 match(Set dst (CastII dst));
8341 effect(KILL cr);
8342
8343 format %{ "# castII_checked of $dst" %}
8344 ins_encode %{
8345 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8346 %}
8347 ins_pipe(pipe_slow);
8348 %}
8349
8350 instruct castLL(iRegL dst)
8351 %{
8352 predicate(VerifyConstraintCasts == 0);
8353 match(Set dst (CastLL dst));
8354
8355 size(0);
8356 format %{ "# castLL of $dst" %}
8357 ins_encode(/* empty encoding */);
8358 ins_cost(0);
8359 ins_pipe(pipe_class_empty);
8360 %}
8361
8362 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8363 %{
8364 predicate(VerifyConstraintCasts > 0);
8365 match(Set dst (CastLL dst));
8366 effect(KILL cr);
8367
8368 format %{ "# castLL_checked of $dst" %}
8369 ins_encode %{
8370 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8371 %}
8372 ins_pipe(pipe_slow);
8373 %}
8374
8375 instruct castHH(vRegF dst)
8376 %{
8377 match(Set dst (CastHH dst));
8378 size(0);
8379 format %{ "# castHH of $dst" %}
8380 ins_encode(/* empty encoding */);
8381 ins_cost(0);
8382 ins_pipe(pipe_class_empty);
8383 %}
8384
8385 instruct castFF(vRegF dst)
8386 %{
8387 match(Set dst (CastFF dst));
8388
8389 size(0);
8390 format %{ "# castFF of $dst" %}
8391 ins_encode(/* empty encoding */);
8392 ins_cost(0);
8393 ins_pipe(pipe_class_empty);
8394 %}
8395
8396 instruct castDD(vRegD dst)
8397 %{
8398 match(Set dst (CastDD dst));
8399
8400 size(0);
8401 format %{ "# castDD of $dst" %}
8402 ins_encode(/* empty encoding */);
8403 ins_cost(0);
8404 ins_pipe(pipe_class_empty);
8405 %}
8406
8407 instruct castVV(vReg dst)
8408 %{
8409 match(Set dst (CastVV dst));
8410
8411 size(0);
8412 format %{ "# castVV of $dst" %}
8413 ins_encode(/* empty encoding */);
8414 ins_cost(0);
8415 ins_pipe(pipe_class_empty);
8416 %}
8417
8418 instruct castVVMask(pRegGov dst)
8419 %{
8420 match(Set dst (CastVV dst));
8421
8422 size(0);
8423 format %{ "# castVV of $dst" %}
8424 ins_encode(/* empty encoding */);
8425 ins_cost(0);
8426 ins_pipe(pipe_class_empty);
8427 %}
8428
8429 // ============================================================================
8430 // Atomic operation instructions
8431 //
8432
8433 // standard CompareAndSwapX when we are using barriers
8434 // these have higher priority than the rules selected by a predicate
8435
8436 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8437 // can't match them
8438
8439 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8440
8441 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8442 ins_cost(2 * VOLATILE_REF_COST);
8443
8444 effect(KILL cr);
8445
8446 format %{
8447 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8448 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8449 %}
8450
8451 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8452 aarch64_enc_cset_eq(res));
8453
8454 ins_pipe(pipe_slow);
8455 %}
8456
8457 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8458
8459 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8460 ins_cost(2 * VOLATILE_REF_COST);
8461
8462 effect(KILL cr);
8463
8464 format %{
8465 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8466 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8467 %}
8468
8469 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8470 aarch64_enc_cset_eq(res));
8471
8472 ins_pipe(pipe_slow);
8473 %}
8474
8475 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8476
8477 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8478 ins_cost(2 * VOLATILE_REF_COST);
8479
8480 effect(KILL cr);
8481
8482 format %{
8483 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8484 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8485 %}
8486
8487 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8488 aarch64_enc_cset_eq(res));
8489
8490 ins_pipe(pipe_slow);
8491 %}
8492
8493 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8494
8495 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8496 ins_cost(2 * VOLATILE_REF_COST);
8497
8498 effect(KILL cr);
8499
8500 format %{
8501 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8502 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8503 %}
8504
8505 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8506 aarch64_enc_cset_eq(res));
8507
8508 ins_pipe(pipe_slow);
8509 %}
8510
8511 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8512
8513 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8514 predicate(n->as_LoadStore()->barrier_data() == 0);
8515 ins_cost(2 * VOLATILE_REF_COST);
8516
8517 effect(KILL cr);
8518
8519 format %{
8520 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8521 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8522 %}
8523
8524 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8525 aarch64_enc_cset_eq(res));
8526
8527 ins_pipe(pipe_slow);
8528 %}
8529
8530 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8531
8532 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8533 predicate(n->as_LoadStore()->barrier_data() == 0);
8534 ins_cost(2 * VOLATILE_REF_COST);
8535
8536 effect(KILL cr);
8537
8538 format %{
8539 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8540 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8541 %}
8542
8543 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8544 aarch64_enc_cset_eq(res));
8545
8546 ins_pipe(pipe_slow);
8547 %}
8548
8549 // alternative CompareAndSwapX when we are eliding barriers
8550
8551 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8552
8553 predicate(needs_acquiring_load_exclusive(n));
8554 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8555 ins_cost(VOLATILE_REF_COST);
8556
8557 effect(KILL cr);
8558
8559 format %{
8560 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8561 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8562 %}
8563
8564 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8565 aarch64_enc_cset_eq(res));
8566
8567 ins_pipe(pipe_slow);
8568 %}
8569
8570 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8571
8572 predicate(needs_acquiring_load_exclusive(n));
8573 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8574 ins_cost(VOLATILE_REF_COST);
8575
8576 effect(KILL cr);
8577
8578 format %{
8579 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8580 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8581 %}
8582
8583 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8584 aarch64_enc_cset_eq(res));
8585
8586 ins_pipe(pipe_slow);
8587 %}
8588
8589 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8590
8591 predicate(needs_acquiring_load_exclusive(n));
8592 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8593 ins_cost(VOLATILE_REF_COST);
8594
8595 effect(KILL cr);
8596
8597 format %{
8598 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8599 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8600 %}
8601
8602 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8603 aarch64_enc_cset_eq(res));
8604
8605 ins_pipe(pipe_slow);
8606 %}
8607
8608 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8609
8610 predicate(needs_acquiring_load_exclusive(n));
8611 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8612 ins_cost(VOLATILE_REF_COST);
8613
8614 effect(KILL cr);
8615
8616 format %{
8617 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8618 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8619 %}
8620
8621 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8622 aarch64_enc_cset_eq(res));
8623
8624 ins_pipe(pipe_slow);
8625 %}
8626
8627 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8628
8629 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8630 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8631 ins_cost(VOLATILE_REF_COST);
8632
8633 effect(KILL cr);
8634
8635 format %{
8636 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8637 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8638 %}
8639
8640 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8641 aarch64_enc_cset_eq(res));
8642
8643 ins_pipe(pipe_slow);
8644 %}
8645
8646 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8647
8648 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8649 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8650 ins_cost(VOLATILE_REF_COST);
8651
8652 effect(KILL cr);
8653
8654 format %{
8655 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8656 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8657 %}
8658
8659 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8660 aarch64_enc_cset_eq(res));
8661
8662 ins_pipe(pipe_slow);
8663 %}
8664
8665
8666 // ---------------------------------------------------------------------
8667
8668 // BEGIN This section of the file is automatically generated. Do not edit --------------
8669
8670 // Sundry CAS operations. Note that release is always true,
8671 // regardless of the memory ordering of the CAS. This is because we
8672 // need the volatile case to be sequentially consistent but there is
8673 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8674 // can't check the type of memory ordering here, so we always emit a
8675 // STLXR.
8676
8677 // This section is generated from cas.m4
8678
8679
8680 // This pattern is generated automatically from cas.m4.
8681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8682 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8683 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8684 ins_cost(2 * VOLATILE_REF_COST);
8685 effect(TEMP_DEF res, KILL cr);
8686 format %{
8687 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8688 %}
8689 ins_encode %{
8690 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8691 Assembler::byte, /*acquire*/ false, /*release*/ true,
8692 /*weak*/ false, $res$$Register);
8693 __ sxtbw($res$$Register, $res$$Register);
8694 %}
8695 ins_pipe(pipe_slow);
8696 %}
8697
8698 // This pattern is generated automatically from cas.m4.
8699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8700 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8701 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8702 ins_cost(2 * VOLATILE_REF_COST);
8703 effect(TEMP_DEF res, KILL cr);
8704 format %{
8705 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8706 %}
8707 ins_encode %{
8708 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8709 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8710 /*weak*/ false, $res$$Register);
8711 __ sxthw($res$$Register, $res$$Register);
8712 %}
8713 ins_pipe(pipe_slow);
8714 %}
8715
8716 // This pattern is generated automatically from cas.m4.
8717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8718 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8719 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8720 ins_cost(2 * VOLATILE_REF_COST);
8721 effect(TEMP_DEF res, KILL cr);
8722 format %{
8723 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8724 %}
8725 ins_encode %{
8726 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8727 Assembler::word, /*acquire*/ false, /*release*/ true,
8728 /*weak*/ false, $res$$Register);
8729 %}
8730 ins_pipe(pipe_slow);
8731 %}
8732
8733 // This pattern is generated automatically from cas.m4.
8734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8735 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8736 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8737 ins_cost(2 * VOLATILE_REF_COST);
8738 effect(TEMP_DEF res, KILL cr);
8739 format %{
8740 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8741 %}
8742 ins_encode %{
8743 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8744 Assembler::xword, /*acquire*/ false, /*release*/ true,
8745 /*weak*/ false, $res$$Register);
8746 %}
8747 ins_pipe(pipe_slow);
8748 %}
8749
8750 // This pattern is generated automatically from cas.m4.
8751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8752 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8753 predicate(n->as_LoadStore()->barrier_data() == 0);
8754 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8755 ins_cost(2 * VOLATILE_REF_COST);
8756 effect(TEMP_DEF res, KILL cr);
8757 format %{
8758 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8759 %}
8760 ins_encode %{
8761 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8762 Assembler::word, /*acquire*/ false, /*release*/ true,
8763 /*weak*/ false, $res$$Register);
8764 %}
8765 ins_pipe(pipe_slow);
8766 %}
8767
8768 // This pattern is generated automatically from cas.m4.
8769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8770 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8771 predicate(n->as_LoadStore()->barrier_data() == 0);
8772 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8773 ins_cost(2 * VOLATILE_REF_COST);
8774 effect(TEMP_DEF res, KILL cr);
8775 format %{
8776 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8777 %}
8778 ins_encode %{
8779 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8780 Assembler::xword, /*acquire*/ false, /*release*/ true,
8781 /*weak*/ false, $res$$Register);
8782 %}
8783 ins_pipe(pipe_slow);
8784 %}
8785
8786 // This pattern is generated automatically from cas.m4.
8787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8788 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8789 predicate(needs_acquiring_load_exclusive(n));
8790 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8791 ins_cost(VOLATILE_REF_COST);
8792 effect(TEMP_DEF res, KILL cr);
8793 format %{
8794 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8795 %}
8796 ins_encode %{
8797 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8798 Assembler::byte, /*acquire*/ true, /*release*/ true,
8799 /*weak*/ false, $res$$Register);
8800 __ sxtbw($res$$Register, $res$$Register);
8801 %}
8802 ins_pipe(pipe_slow);
8803 %}
8804
8805 // This pattern is generated automatically from cas.m4.
8806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8807 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8808 predicate(needs_acquiring_load_exclusive(n));
8809 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8810 ins_cost(VOLATILE_REF_COST);
8811 effect(TEMP_DEF res, KILL cr);
8812 format %{
8813 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8814 %}
8815 ins_encode %{
8816 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8817 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8818 /*weak*/ false, $res$$Register);
8819 __ sxthw($res$$Register, $res$$Register);
8820 %}
8821 ins_pipe(pipe_slow);
8822 %}
8823
8824 // This pattern is generated automatically from cas.m4.
8825 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8826 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8827 predicate(needs_acquiring_load_exclusive(n));
8828 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8829 ins_cost(VOLATILE_REF_COST);
8830 effect(TEMP_DEF res, KILL cr);
8831 format %{
8832 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8833 %}
8834 ins_encode %{
8835 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8836 Assembler::word, /*acquire*/ true, /*release*/ true,
8837 /*weak*/ false, $res$$Register);
8838 %}
8839 ins_pipe(pipe_slow);
8840 %}
8841
8842 // This pattern is generated automatically from cas.m4.
8843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8844 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8845 predicate(needs_acquiring_load_exclusive(n));
8846 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8847 ins_cost(VOLATILE_REF_COST);
8848 effect(TEMP_DEF res, KILL cr);
8849 format %{
8850 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8851 %}
8852 ins_encode %{
8853 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8854 Assembler::xword, /*acquire*/ true, /*release*/ true,
8855 /*weak*/ false, $res$$Register);
8856 %}
8857 ins_pipe(pipe_slow);
8858 %}
8859
8860 // This pattern is generated automatically from cas.m4.
8861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8862 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8863 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8864 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8865 ins_cost(VOLATILE_REF_COST);
8866 effect(TEMP_DEF res, KILL cr);
8867 format %{
8868 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8869 %}
8870 ins_encode %{
8871 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8872 Assembler::word, /*acquire*/ true, /*release*/ true,
8873 /*weak*/ false, $res$$Register);
8874 %}
8875 ins_pipe(pipe_slow);
8876 %}
8877
8878 // This pattern is generated automatically from cas.m4.
8879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8880 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8881 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8882 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8883 ins_cost(VOLATILE_REF_COST);
8884 effect(TEMP_DEF res, KILL cr);
8885 format %{
8886 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8887 %}
8888 ins_encode %{
8889 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8890 Assembler::xword, /*acquire*/ true, /*release*/ true,
8891 /*weak*/ false, $res$$Register);
8892 %}
8893 ins_pipe(pipe_slow);
8894 %}
8895
8896 // This pattern is generated automatically from cas.m4.
8897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8898 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8899 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8900 ins_cost(2 * VOLATILE_REF_COST);
8901 effect(KILL cr);
8902 format %{
8903 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8904 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8905 %}
8906 ins_encode %{
8907 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8908 Assembler::byte, /*acquire*/ false, /*release*/ true,
8909 /*weak*/ true, noreg);
8910 __ csetw($res$$Register, Assembler::EQ);
8911 %}
8912 ins_pipe(pipe_slow);
8913 %}
8914
8915 // This pattern is generated automatically from cas.m4.
8916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8917 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8918 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8919 ins_cost(2 * VOLATILE_REF_COST);
8920 effect(KILL cr);
8921 format %{
8922 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8923 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8924 %}
8925 ins_encode %{
8926 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8927 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8928 /*weak*/ true, noreg);
8929 __ csetw($res$$Register, Assembler::EQ);
8930 %}
8931 ins_pipe(pipe_slow);
8932 %}
8933
8934 // This pattern is generated automatically from cas.m4.
8935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8936 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8937 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8938 ins_cost(2 * VOLATILE_REF_COST);
8939 effect(KILL cr);
8940 format %{
8941 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8942 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8943 %}
8944 ins_encode %{
8945 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8946 Assembler::word, /*acquire*/ false, /*release*/ true,
8947 /*weak*/ true, noreg);
8948 __ csetw($res$$Register, Assembler::EQ);
8949 %}
8950 ins_pipe(pipe_slow);
8951 %}
8952
8953 // This pattern is generated automatically from cas.m4.
8954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8955 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8956 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8957 ins_cost(2 * VOLATILE_REF_COST);
8958 effect(KILL cr);
8959 format %{
8960 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8961 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8962 %}
8963 ins_encode %{
8964 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8965 Assembler::xword, /*acquire*/ false, /*release*/ true,
8966 /*weak*/ true, noreg);
8967 __ csetw($res$$Register, Assembler::EQ);
8968 %}
8969 ins_pipe(pipe_slow);
8970 %}
8971
8972 // This pattern is generated automatically from cas.m4.
8973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8974 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8975 predicate(n->as_LoadStore()->barrier_data() == 0);
8976 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8977 ins_cost(2 * VOLATILE_REF_COST);
8978 effect(KILL cr);
8979 format %{
8980 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8981 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8982 %}
8983 ins_encode %{
8984 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8985 Assembler::word, /*acquire*/ false, /*release*/ true,
8986 /*weak*/ true, noreg);
8987 __ csetw($res$$Register, Assembler::EQ);
8988 %}
8989 ins_pipe(pipe_slow);
8990 %}
8991
8992 // This pattern is generated automatically from cas.m4.
8993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8994 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8995 predicate(n->as_LoadStore()->barrier_data() == 0);
8996 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8997 ins_cost(2 * VOLATILE_REF_COST);
8998 effect(KILL cr);
8999 format %{
9000 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9001 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9002 %}
9003 ins_encode %{
9004 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9005 Assembler::xword, /*acquire*/ false, /*release*/ true,
9006 /*weak*/ true, noreg);
9007 __ csetw($res$$Register, Assembler::EQ);
9008 %}
9009 ins_pipe(pipe_slow);
9010 %}
9011
9012 // This pattern is generated automatically from cas.m4.
9013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9014 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9015 predicate(needs_acquiring_load_exclusive(n));
9016 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9017 ins_cost(VOLATILE_REF_COST);
9018 effect(KILL cr);
9019 format %{
9020 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9021 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9022 %}
9023 ins_encode %{
9024 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9025 Assembler::byte, /*acquire*/ true, /*release*/ true,
9026 /*weak*/ true, noreg);
9027 __ csetw($res$$Register, Assembler::EQ);
9028 %}
9029 ins_pipe(pipe_slow);
9030 %}
9031
9032 // This pattern is generated automatically from cas.m4.
9033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9034 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9035 predicate(needs_acquiring_load_exclusive(n));
9036 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9037 ins_cost(VOLATILE_REF_COST);
9038 effect(KILL cr);
9039 format %{
9040 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9041 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9042 %}
9043 ins_encode %{
9044 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9045 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9046 /*weak*/ true, noreg);
9047 __ csetw($res$$Register, Assembler::EQ);
9048 %}
9049 ins_pipe(pipe_slow);
9050 %}
9051
9052 // This pattern is generated automatically from cas.m4.
9053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9054 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9055 predicate(needs_acquiring_load_exclusive(n));
9056 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9057 ins_cost(VOLATILE_REF_COST);
9058 effect(KILL cr);
9059 format %{
9060 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9061 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9062 %}
9063 ins_encode %{
9064 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9065 Assembler::word, /*acquire*/ true, /*release*/ true,
9066 /*weak*/ true, noreg);
9067 __ csetw($res$$Register, Assembler::EQ);
9068 %}
9069 ins_pipe(pipe_slow);
9070 %}
9071
9072 // This pattern is generated automatically from cas.m4.
9073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9074 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9075 predicate(needs_acquiring_load_exclusive(n));
9076 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9077 ins_cost(VOLATILE_REF_COST);
9078 effect(KILL cr);
9079 format %{
9080 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9081 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9082 %}
9083 ins_encode %{
9084 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9085 Assembler::xword, /*acquire*/ true, /*release*/ true,
9086 /*weak*/ true, noreg);
9087 __ csetw($res$$Register, Assembler::EQ);
9088 %}
9089 ins_pipe(pipe_slow);
9090 %}
9091
9092 // This pattern is generated automatically from cas.m4.
9093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9094 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9095 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9096 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9097 ins_cost(VOLATILE_REF_COST);
9098 effect(KILL cr);
9099 format %{
9100 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9101 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9102 %}
9103 ins_encode %{
9104 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9105 Assembler::word, /*acquire*/ true, /*release*/ true,
9106 /*weak*/ true, noreg);
9107 __ csetw($res$$Register, Assembler::EQ);
9108 %}
9109 ins_pipe(pipe_slow);
9110 %}
9111
9112 // This pattern is generated automatically from cas.m4.
9113 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9114 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9115 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9116 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9117 ins_cost(VOLATILE_REF_COST);
9118 effect(KILL cr);
9119 format %{
9120 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9121 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9122 %}
9123 ins_encode %{
9124 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9125 Assembler::xword, /*acquire*/ true, /*release*/ true,
9126 /*weak*/ true, noreg);
9127 __ csetw($res$$Register, Assembler::EQ);
9128 %}
9129 ins_pipe(pipe_slow);
9130 %}
9131
9132 // END This section of the file is automatically generated. Do not edit --------------
9133 // ---------------------------------------------------------------------
9134
9135 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9136 match(Set prev (GetAndSetI mem newv));
9137 ins_cost(2 * VOLATILE_REF_COST);
9138 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9139 ins_encode %{
9140 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9141 %}
9142 ins_pipe(pipe_serial);
9143 %}
9144
9145 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9146 match(Set prev (GetAndSetL mem newv));
9147 ins_cost(2 * VOLATILE_REF_COST);
9148 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9149 ins_encode %{
9150 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9151 %}
9152 ins_pipe(pipe_serial);
9153 %}
9154
9155 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9156 predicate(n->as_LoadStore()->barrier_data() == 0);
9157 match(Set prev (GetAndSetN mem newv));
9158 ins_cost(2 * VOLATILE_REF_COST);
9159 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9160 ins_encode %{
9161 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9162 %}
9163 ins_pipe(pipe_serial);
9164 %}
9165
9166 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9167 predicate(n->as_LoadStore()->barrier_data() == 0);
9168 match(Set prev (GetAndSetP mem newv));
9169 ins_cost(2 * VOLATILE_REF_COST);
9170 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9171 ins_encode %{
9172 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9173 %}
9174 ins_pipe(pipe_serial);
9175 %}
9176
9177 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9178 predicate(needs_acquiring_load_exclusive(n));
9179 match(Set prev (GetAndSetI mem newv));
9180 ins_cost(VOLATILE_REF_COST);
9181 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9182 ins_encode %{
9183 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9184 %}
9185 ins_pipe(pipe_serial);
9186 %}
9187
9188 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9189 predicate(needs_acquiring_load_exclusive(n));
9190 match(Set prev (GetAndSetL mem newv));
9191 ins_cost(VOLATILE_REF_COST);
9192 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9193 ins_encode %{
9194 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9195 %}
9196 ins_pipe(pipe_serial);
9197 %}
9198
9199 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9200 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9201 match(Set prev (GetAndSetN mem newv));
9202 ins_cost(VOLATILE_REF_COST);
9203 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9204 ins_encode %{
9205 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9206 %}
9207 ins_pipe(pipe_serial);
9208 %}
9209
9210 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9211 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9212 match(Set prev (GetAndSetP mem newv));
9213 ins_cost(VOLATILE_REF_COST);
9214 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9215 ins_encode %{
9216 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9217 %}
9218 ins_pipe(pipe_serial);
9219 %}
9220
9221
9222 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9223 match(Set newval (GetAndAddL mem incr));
9224 ins_cost(2 * VOLATILE_REF_COST + 1);
9225 format %{ "get_and_addL $newval, [$mem], $incr" %}
9226 ins_encode %{
9227 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9228 %}
9229 ins_pipe(pipe_serial);
9230 %}
9231
9232 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9233 predicate(n->as_LoadStore()->result_not_used());
9234 match(Set dummy (GetAndAddL mem incr));
9235 ins_cost(2 * VOLATILE_REF_COST);
9236 format %{ "get_and_addL [$mem], $incr" %}
9237 ins_encode %{
9238 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9239 %}
9240 ins_pipe(pipe_serial);
9241 %}
9242
9243 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9244 match(Set newval (GetAndAddL mem incr));
9245 ins_cost(2 * VOLATILE_REF_COST + 1);
9246 format %{ "get_and_addL $newval, [$mem], $incr" %}
9247 ins_encode %{
9248 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9249 %}
9250 ins_pipe(pipe_serial);
9251 %}
9252
9253 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9254 predicate(n->as_LoadStore()->result_not_used());
9255 match(Set dummy (GetAndAddL mem incr));
9256 ins_cost(2 * VOLATILE_REF_COST);
9257 format %{ "get_and_addL [$mem], $incr" %}
9258 ins_encode %{
9259 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9260 %}
9261 ins_pipe(pipe_serial);
9262 %}
9263
9264 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9265 match(Set newval (GetAndAddI mem incr));
9266 ins_cost(2 * VOLATILE_REF_COST + 1);
9267 format %{ "get_and_addI $newval, [$mem], $incr" %}
9268 ins_encode %{
9269 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9270 %}
9271 ins_pipe(pipe_serial);
9272 %}
9273
9274 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9275 predicate(n->as_LoadStore()->result_not_used());
9276 match(Set dummy (GetAndAddI mem incr));
9277 ins_cost(2 * VOLATILE_REF_COST);
9278 format %{ "get_and_addI [$mem], $incr" %}
9279 ins_encode %{
9280 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9281 %}
9282 ins_pipe(pipe_serial);
9283 %}
9284
9285 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9286 match(Set newval (GetAndAddI mem incr));
9287 ins_cost(2 * VOLATILE_REF_COST + 1);
9288 format %{ "get_and_addI $newval, [$mem], $incr" %}
9289 ins_encode %{
9290 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9291 %}
9292 ins_pipe(pipe_serial);
9293 %}
9294
9295 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9296 predicate(n->as_LoadStore()->result_not_used());
9297 match(Set dummy (GetAndAddI mem incr));
9298 ins_cost(2 * VOLATILE_REF_COST);
9299 format %{ "get_and_addI [$mem], $incr" %}
9300 ins_encode %{
9301 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9302 %}
9303 ins_pipe(pipe_serial);
9304 %}
9305
9306 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9307 predicate(needs_acquiring_load_exclusive(n));
9308 match(Set newval (GetAndAddL mem incr));
9309 ins_cost(VOLATILE_REF_COST + 1);
9310 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9311 ins_encode %{
9312 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9313 %}
9314 ins_pipe(pipe_serial);
9315 %}
9316
9317 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9318 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9319 match(Set dummy (GetAndAddL mem incr));
9320 ins_cost(VOLATILE_REF_COST);
9321 format %{ "get_and_addL_acq [$mem], $incr" %}
9322 ins_encode %{
9323 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9324 %}
9325 ins_pipe(pipe_serial);
9326 %}
9327
9328 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9329 predicate(needs_acquiring_load_exclusive(n));
9330 match(Set newval (GetAndAddL mem incr));
9331 ins_cost(VOLATILE_REF_COST + 1);
9332 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9333 ins_encode %{
9334 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9335 %}
9336 ins_pipe(pipe_serial);
9337 %}
9338
9339 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9340 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9341 match(Set dummy (GetAndAddL mem incr));
9342 ins_cost(VOLATILE_REF_COST);
9343 format %{ "get_and_addL_acq [$mem], $incr" %}
9344 ins_encode %{
9345 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9346 %}
9347 ins_pipe(pipe_serial);
9348 %}
9349
9350 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9351 predicate(needs_acquiring_load_exclusive(n));
9352 match(Set newval (GetAndAddI mem incr));
9353 ins_cost(VOLATILE_REF_COST + 1);
9354 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9355 ins_encode %{
9356 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9357 %}
9358 ins_pipe(pipe_serial);
9359 %}
9360
9361 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9362 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9363 match(Set dummy (GetAndAddI mem incr));
9364 ins_cost(VOLATILE_REF_COST);
9365 format %{ "get_and_addI_acq [$mem], $incr" %}
9366 ins_encode %{
9367 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9368 %}
9369 ins_pipe(pipe_serial);
9370 %}
9371
9372 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9373 predicate(needs_acquiring_load_exclusive(n));
9374 match(Set newval (GetAndAddI mem incr));
9375 ins_cost(VOLATILE_REF_COST + 1);
9376 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9377 ins_encode %{
9378 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9379 %}
9380 ins_pipe(pipe_serial);
9381 %}
9382
9383 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9384 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9385 match(Set dummy (GetAndAddI mem incr));
9386 ins_cost(VOLATILE_REF_COST);
9387 format %{ "get_and_addI_acq [$mem], $incr" %}
9388 ins_encode %{
9389 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9390 %}
9391 ins_pipe(pipe_serial);
9392 %}
9393
9394 // Manifest a CmpU result in an integer register.
9395 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9396 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9397 %{
9398 match(Set dst (CmpU3 src1 src2));
9399 effect(KILL flags);
9400
9401 ins_cost(INSN_COST * 3);
9402 format %{
9403 "cmpw $src1, $src2\n\t"
9404 "csetw $dst, ne\n\t"
9405 "cnegw $dst, lo\t# CmpU3(reg)"
9406 %}
9407 ins_encode %{
9408 __ cmpw($src1$$Register, $src2$$Register);
9409 __ csetw($dst$$Register, Assembler::NE);
9410 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9411 %}
9412
9413 ins_pipe(pipe_class_default);
9414 %}
9415
9416 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9417 %{
9418 match(Set dst (CmpU3 src1 src2));
9419 effect(KILL flags);
9420
9421 ins_cost(INSN_COST * 3);
9422 format %{
9423 "subsw zr, $src1, $src2\n\t"
9424 "csetw $dst, ne\n\t"
9425 "cnegw $dst, lo\t# CmpU3(imm)"
9426 %}
9427 ins_encode %{
9428 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9429 __ csetw($dst$$Register, Assembler::NE);
9430 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9431 %}
9432
9433 ins_pipe(pipe_class_default);
9434 %}
9435
9436 // Manifest a CmpUL result in an integer register.
9437 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9438 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9439 %{
9440 match(Set dst (CmpUL3 src1 src2));
9441 effect(KILL flags);
9442
9443 ins_cost(INSN_COST * 3);
9444 format %{
9445 "cmp $src1, $src2\n\t"
9446 "csetw $dst, ne\n\t"
9447 "cnegw $dst, lo\t# CmpUL3(reg)"
9448 %}
9449 ins_encode %{
9450 __ cmp($src1$$Register, $src2$$Register);
9451 __ csetw($dst$$Register, Assembler::NE);
9452 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9453 %}
9454
9455 ins_pipe(pipe_class_default);
9456 %}
9457
9458 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9459 %{
9460 match(Set dst (CmpUL3 src1 src2));
9461 effect(KILL flags);
9462
9463 ins_cost(INSN_COST * 3);
9464 format %{
9465 "subs zr, $src1, $src2\n\t"
9466 "csetw $dst, ne\n\t"
9467 "cnegw $dst, lo\t# CmpUL3(imm)"
9468 %}
9469 ins_encode %{
9470 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9471 __ csetw($dst$$Register, Assembler::NE);
9472 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9473 %}
9474
9475 ins_pipe(pipe_class_default);
9476 %}
9477
9478 // Manifest a CmpL result in an integer register.
9479 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9480 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9481 %{
9482 match(Set dst (CmpL3 src1 src2));
9483 effect(KILL flags);
9484
9485 ins_cost(INSN_COST * 3);
9486 format %{
9487 "cmp $src1, $src2\n\t"
9488 "csetw $dst, ne\n\t"
9489 "cnegw $dst, lt\t# CmpL3(reg)"
9490 %}
9491 ins_encode %{
9492 __ cmp($src1$$Register, $src2$$Register);
9493 __ csetw($dst$$Register, Assembler::NE);
9494 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9495 %}
9496
9497 ins_pipe(pipe_class_default);
9498 %}
9499
9500 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9501 %{
9502 match(Set dst (CmpL3 src1 src2));
9503 effect(KILL flags);
9504
9505 ins_cost(INSN_COST * 3);
9506 format %{
9507 "subs zr, $src1, $src2\n\t"
9508 "csetw $dst, ne\n\t"
9509 "cnegw $dst, lt\t# CmpL3(imm)"
9510 %}
9511 ins_encode %{
9512 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9513 __ csetw($dst$$Register, Assembler::NE);
9514 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9515 %}
9516
9517 ins_pipe(pipe_class_default);
9518 %}
9519
9520 // ============================================================================
9521 // Conditional Move Instructions
9522
9523 // n.b. we have identical rules for both a signed compare op (cmpOp)
9524 // and an unsigned compare op (cmpOpU). it would be nice if we could
9525 // define an op class which merged both inputs and use it to type the
9526 // argument to a single rule. unfortunatelyt his fails because the
9527 // opclass does not live up to the COND_INTER interface of its
9528 // component operands. When the generic code tries to negate the
9529 // operand it ends up running the generci Machoper::negate method
9530 // which throws a ShouldNotHappen. So, we have to provide two flavours
9531 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9532
9533 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9534 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9535
9536 ins_cost(INSN_COST * 2);
9537 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9538
9539 ins_encode %{
9540 __ cselw(as_Register($dst$$reg),
9541 as_Register($src2$$reg),
9542 as_Register($src1$$reg),
9543 (Assembler::Condition)$cmp$$cmpcode);
9544 %}
9545
9546 ins_pipe(icond_reg_reg);
9547 %}
9548
9549 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9550 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9551
9552 ins_cost(INSN_COST * 2);
9553 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9554
9555 ins_encode %{
9556 __ cselw(as_Register($dst$$reg),
9557 as_Register($src2$$reg),
9558 as_Register($src1$$reg),
9559 (Assembler::Condition)$cmp$$cmpcode);
9560 %}
9561
9562 ins_pipe(icond_reg_reg);
9563 %}
9564
9565 // special cases where one arg is zero
9566
9567 // n.b. this is selected in preference to the rule above because it
9568 // avoids loading constant 0 into a source register
9569
9570 // TODO
9571 // we ought only to be able to cull one of these variants as the ideal
9572 // transforms ought always to order the zero consistently (to left/right?)
9573
9574 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9575 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9576
9577 ins_cost(INSN_COST * 2);
9578 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9579
9580 ins_encode %{
9581 __ cselw(as_Register($dst$$reg),
9582 as_Register($src$$reg),
9583 zr,
9584 (Assembler::Condition)$cmp$$cmpcode);
9585 %}
9586
9587 ins_pipe(icond_reg);
9588 %}
9589
9590 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9591 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9592
9593 ins_cost(INSN_COST * 2);
9594 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9595
9596 ins_encode %{
9597 __ cselw(as_Register($dst$$reg),
9598 as_Register($src$$reg),
9599 zr,
9600 (Assembler::Condition)$cmp$$cmpcode);
9601 %}
9602
9603 ins_pipe(icond_reg);
9604 %}
9605
9606 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9607 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9608
9609 ins_cost(INSN_COST * 2);
9610 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9611
9612 ins_encode %{
9613 __ cselw(as_Register($dst$$reg),
9614 zr,
9615 as_Register($src$$reg),
9616 (Assembler::Condition)$cmp$$cmpcode);
9617 %}
9618
9619 ins_pipe(icond_reg);
9620 %}
9621
9622 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9623 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9624
9625 ins_cost(INSN_COST * 2);
9626 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9627
9628 ins_encode %{
9629 __ cselw(as_Register($dst$$reg),
9630 zr,
9631 as_Register($src$$reg),
9632 (Assembler::Condition)$cmp$$cmpcode);
9633 %}
9634
9635 ins_pipe(icond_reg);
9636 %}
9637
9638 // special case for creating a boolean 0 or 1
9639
9640 // n.b. this is selected in preference to the rule above because it
9641 // avoids loading constants 0 and 1 into a source register
9642
9643 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9644 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9645
9646 ins_cost(INSN_COST * 2);
9647 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9648
9649 ins_encode %{
9650 // equivalently
9651 // cset(as_Register($dst$$reg),
9652 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9653 __ csincw(as_Register($dst$$reg),
9654 zr,
9655 zr,
9656 (Assembler::Condition)$cmp$$cmpcode);
9657 %}
9658
9659 ins_pipe(icond_none);
9660 %}
9661
9662 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9663 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9664
9665 ins_cost(INSN_COST * 2);
9666 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9667
9668 ins_encode %{
9669 // equivalently
9670 // cset(as_Register($dst$$reg),
9671 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9672 __ csincw(as_Register($dst$$reg),
9673 zr,
9674 zr,
9675 (Assembler::Condition)$cmp$$cmpcode);
9676 %}
9677
9678 ins_pipe(icond_none);
9679 %}
9680
9681 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9682 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9683
9684 ins_cost(INSN_COST * 2);
9685 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9686
9687 ins_encode %{
9688 __ csel(as_Register($dst$$reg),
9689 as_Register($src2$$reg),
9690 as_Register($src1$$reg),
9691 (Assembler::Condition)$cmp$$cmpcode);
9692 %}
9693
9694 ins_pipe(icond_reg_reg);
9695 %}
9696
9697 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9698 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9699
9700 ins_cost(INSN_COST * 2);
9701 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9702
9703 ins_encode %{
9704 __ csel(as_Register($dst$$reg),
9705 as_Register($src2$$reg),
9706 as_Register($src1$$reg),
9707 (Assembler::Condition)$cmp$$cmpcode);
9708 %}
9709
9710 ins_pipe(icond_reg_reg);
9711 %}
9712
9713 // special cases where one arg is zero
9714
9715 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9716 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9717
9718 ins_cost(INSN_COST * 2);
9719 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9720
9721 ins_encode %{
9722 __ csel(as_Register($dst$$reg),
9723 zr,
9724 as_Register($src$$reg),
9725 (Assembler::Condition)$cmp$$cmpcode);
9726 %}
9727
9728 ins_pipe(icond_reg);
9729 %}
9730
9731 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9732 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9733
9734 ins_cost(INSN_COST * 2);
9735 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9736
9737 ins_encode %{
9738 __ csel(as_Register($dst$$reg),
9739 zr,
9740 as_Register($src$$reg),
9741 (Assembler::Condition)$cmp$$cmpcode);
9742 %}
9743
9744 ins_pipe(icond_reg);
9745 %}
9746
9747 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9748 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9749
9750 ins_cost(INSN_COST * 2);
9751 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9752
9753 ins_encode %{
9754 __ csel(as_Register($dst$$reg),
9755 as_Register($src$$reg),
9756 zr,
9757 (Assembler::Condition)$cmp$$cmpcode);
9758 %}
9759
9760 ins_pipe(icond_reg);
9761 %}
9762
9763 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9764 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9765
9766 ins_cost(INSN_COST * 2);
9767 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9768
9769 ins_encode %{
9770 __ csel(as_Register($dst$$reg),
9771 as_Register($src$$reg),
9772 zr,
9773 (Assembler::Condition)$cmp$$cmpcode);
9774 %}
9775
9776 ins_pipe(icond_reg);
9777 %}
9778
9779 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9780 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9781
9782 ins_cost(INSN_COST * 2);
9783 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9784
9785 ins_encode %{
9786 __ csel(as_Register($dst$$reg),
9787 as_Register($src2$$reg),
9788 as_Register($src1$$reg),
9789 (Assembler::Condition)$cmp$$cmpcode);
9790 %}
9791
9792 ins_pipe(icond_reg_reg);
9793 %}
9794
9795 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9796 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9797
9798 ins_cost(INSN_COST * 2);
9799 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9800
9801 ins_encode %{
9802 __ csel(as_Register($dst$$reg),
9803 as_Register($src2$$reg),
9804 as_Register($src1$$reg),
9805 (Assembler::Condition)$cmp$$cmpcode);
9806 %}
9807
9808 ins_pipe(icond_reg_reg);
9809 %}
9810
9811 // special cases where one arg is zero
9812
9813 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9814 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9815
9816 ins_cost(INSN_COST * 2);
9817 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9818
9819 ins_encode %{
9820 __ csel(as_Register($dst$$reg),
9821 zr,
9822 as_Register($src$$reg),
9823 (Assembler::Condition)$cmp$$cmpcode);
9824 %}
9825
9826 ins_pipe(icond_reg);
9827 %}
9828
9829 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9830 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9831
9832 ins_cost(INSN_COST * 2);
9833 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9834
9835 ins_encode %{
9836 __ csel(as_Register($dst$$reg),
9837 zr,
9838 as_Register($src$$reg),
9839 (Assembler::Condition)$cmp$$cmpcode);
9840 %}
9841
9842 ins_pipe(icond_reg);
9843 %}
9844
9845 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9846 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9847
9848 ins_cost(INSN_COST * 2);
9849 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9850
9851 ins_encode %{
9852 __ csel(as_Register($dst$$reg),
9853 as_Register($src$$reg),
9854 zr,
9855 (Assembler::Condition)$cmp$$cmpcode);
9856 %}
9857
9858 ins_pipe(icond_reg);
9859 %}
9860
9861 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9862 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9863
9864 ins_cost(INSN_COST * 2);
9865 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9866
9867 ins_encode %{
9868 __ csel(as_Register($dst$$reg),
9869 as_Register($src$$reg),
9870 zr,
9871 (Assembler::Condition)$cmp$$cmpcode);
9872 %}
9873
9874 ins_pipe(icond_reg);
9875 %}
9876
9877 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9878 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9879
9880 ins_cost(INSN_COST * 2);
9881 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9882
9883 ins_encode %{
9884 __ cselw(as_Register($dst$$reg),
9885 as_Register($src2$$reg),
9886 as_Register($src1$$reg),
9887 (Assembler::Condition)$cmp$$cmpcode);
9888 %}
9889
9890 ins_pipe(icond_reg_reg);
9891 %}
9892
9893 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9894 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9895
9896 ins_cost(INSN_COST * 2);
9897 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9898
9899 ins_encode %{
9900 __ cselw(as_Register($dst$$reg),
9901 as_Register($src2$$reg),
9902 as_Register($src1$$reg),
9903 (Assembler::Condition)$cmp$$cmpcode);
9904 %}
9905
9906 ins_pipe(icond_reg_reg);
9907 %}
9908
9909 // special cases where one arg is zero
9910
9911 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9912 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9913
9914 ins_cost(INSN_COST * 2);
9915 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9916
9917 ins_encode %{
9918 __ cselw(as_Register($dst$$reg),
9919 zr,
9920 as_Register($src$$reg),
9921 (Assembler::Condition)$cmp$$cmpcode);
9922 %}
9923
9924 ins_pipe(icond_reg);
9925 %}
9926
9927 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9928 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9929
9930 ins_cost(INSN_COST * 2);
9931 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9932
9933 ins_encode %{
9934 __ cselw(as_Register($dst$$reg),
9935 zr,
9936 as_Register($src$$reg),
9937 (Assembler::Condition)$cmp$$cmpcode);
9938 %}
9939
9940 ins_pipe(icond_reg);
9941 %}
9942
9943 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9944 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9945
9946 ins_cost(INSN_COST * 2);
9947 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9948
9949 ins_encode %{
9950 __ cselw(as_Register($dst$$reg),
9951 as_Register($src$$reg),
9952 zr,
9953 (Assembler::Condition)$cmp$$cmpcode);
9954 %}
9955
9956 ins_pipe(icond_reg);
9957 %}
9958
9959 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9960 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9961
9962 ins_cost(INSN_COST * 2);
9963 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9964
9965 ins_encode %{
9966 __ cselw(as_Register($dst$$reg),
9967 as_Register($src$$reg),
9968 zr,
9969 (Assembler::Condition)$cmp$$cmpcode);
9970 %}
9971
9972 ins_pipe(icond_reg);
9973 %}
9974
9975 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9976 %{
9977 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9978
9979 ins_cost(INSN_COST * 3);
9980
9981 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9982 ins_encode %{
9983 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9984 __ fcsels(as_FloatRegister($dst$$reg),
9985 as_FloatRegister($src2$$reg),
9986 as_FloatRegister($src1$$reg),
9987 cond);
9988 %}
9989
9990 ins_pipe(fp_cond_reg_reg_s);
9991 %}
9992
9993 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9994 %{
9995 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9996
9997 ins_cost(INSN_COST * 3);
9998
9999 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10000 ins_encode %{
10001 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10002 __ fcsels(as_FloatRegister($dst$$reg),
10003 as_FloatRegister($src2$$reg),
10004 as_FloatRegister($src1$$reg),
10005 cond);
10006 %}
10007
10008 ins_pipe(fp_cond_reg_reg_s);
10009 %}
10010
10011 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10012 %{
10013 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10014
10015 ins_cost(INSN_COST * 3);
10016
10017 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10018 ins_encode %{
10019 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10020 __ fcseld(as_FloatRegister($dst$$reg),
10021 as_FloatRegister($src2$$reg),
10022 as_FloatRegister($src1$$reg),
10023 cond);
10024 %}
10025
10026 ins_pipe(fp_cond_reg_reg_d);
10027 %}
10028
10029 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10030 %{
10031 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10032
10033 ins_cost(INSN_COST * 3);
10034
10035 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10036 ins_encode %{
10037 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10038 __ fcseld(as_FloatRegister($dst$$reg),
10039 as_FloatRegister($src2$$reg),
10040 as_FloatRegister($src1$$reg),
10041 cond);
10042 %}
10043
10044 ins_pipe(fp_cond_reg_reg_d);
10045 %}
10046
10047 // ============================================================================
10048 // Arithmetic Instructions
10049 //
10050
10051 // Integer Addition
10052
10053 // TODO
10054 // these currently employ operations which do not set CR and hence are
10055 // not flagged as killing CR but we would like to isolate the cases
10056 // where we want to set flags from those where we don't. need to work
10057 // out how to do that.
10058
10059 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10060 match(Set dst (AddI src1 src2));
10061
10062 ins_cost(INSN_COST);
10063 format %{ "addw $dst, $src1, $src2" %}
10064
10065 ins_encode %{
10066 __ addw(as_Register($dst$$reg),
10067 as_Register($src1$$reg),
10068 as_Register($src2$$reg));
10069 %}
10070
10071 ins_pipe(ialu_reg_reg);
10072 %}
10073
10074 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10075 match(Set dst (AddI src1 src2));
10076
10077 ins_cost(INSN_COST);
10078 format %{ "addw $dst, $src1, $src2" %}
10079
10080 // use opcode to indicate that this is an add not a sub
10081 opcode(0x0);
10082
10083 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10084
10085 ins_pipe(ialu_reg_imm);
10086 %}
10087
10088 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10089 match(Set dst (AddI (ConvL2I src1) src2));
10090
10091 ins_cost(INSN_COST);
10092 format %{ "addw $dst, $src1, $src2" %}
10093
10094 // use opcode to indicate that this is an add not a sub
10095 opcode(0x0);
10096
10097 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10098
10099 ins_pipe(ialu_reg_imm);
10100 %}
10101
10102 // Pointer Addition
10103 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10104 match(Set dst (AddP src1 src2));
10105
10106 ins_cost(INSN_COST);
10107 format %{ "add $dst, $src1, $src2\t# ptr" %}
10108
10109 ins_encode %{
10110 __ add(as_Register($dst$$reg),
10111 as_Register($src1$$reg),
10112 as_Register($src2$$reg));
10113 %}
10114
10115 ins_pipe(ialu_reg_reg);
10116 %}
10117
10118 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10119 match(Set dst (AddP src1 (ConvI2L src2)));
10120
10121 ins_cost(1.9 * INSN_COST);
10122 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10123
10124 ins_encode %{
10125 __ add(as_Register($dst$$reg),
10126 as_Register($src1$$reg),
10127 as_Register($src2$$reg), ext::sxtw);
10128 %}
10129
10130 ins_pipe(ialu_reg_reg);
10131 %}
10132
10133 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10134 match(Set dst (AddP src1 (LShiftL src2 scale)));
10135
10136 ins_cost(1.9 * INSN_COST);
10137 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10138
10139 ins_encode %{
10140 __ lea(as_Register($dst$$reg),
10141 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10142 Address::lsl($scale$$constant)));
10143 %}
10144
10145 ins_pipe(ialu_reg_reg_shift);
10146 %}
10147
10148 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10149 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10150
10151 ins_cost(1.9 * INSN_COST);
10152 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10153
10154 ins_encode %{
10155 __ lea(as_Register($dst$$reg),
10156 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10157 Address::sxtw($scale$$constant)));
10158 %}
10159
10160 ins_pipe(ialu_reg_reg_shift);
10161 %}
10162
10163 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10164 match(Set dst (LShiftL (ConvI2L src) scale));
10165
10166 ins_cost(INSN_COST);
10167 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10168
10169 ins_encode %{
10170 __ sbfiz(as_Register($dst$$reg),
10171 as_Register($src$$reg),
10172 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10173 %}
10174
10175 ins_pipe(ialu_reg_shift);
10176 %}
10177
10178 // Pointer Immediate Addition
10179 // n.b. this needs to be more expensive than using an indirect memory
10180 // operand
10181 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10182 match(Set dst (AddP src1 src2));
10183
10184 ins_cost(INSN_COST);
10185 format %{ "add $dst, $src1, $src2\t# ptr" %}
10186
10187 // use opcode to indicate that this is an add not a sub
10188 opcode(0x0);
10189
10190 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10191
10192 ins_pipe(ialu_reg_imm);
10193 %}
10194
10195 // Long Addition
10196 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10197
10198 match(Set dst (AddL src1 src2));
10199
10200 ins_cost(INSN_COST);
10201 format %{ "add $dst, $src1, $src2" %}
10202
10203 ins_encode %{
10204 __ add(as_Register($dst$$reg),
10205 as_Register($src1$$reg),
10206 as_Register($src2$$reg));
10207 %}
10208
10209 ins_pipe(ialu_reg_reg);
10210 %}
10211
10212 // No constant pool entries requiredLong Immediate Addition.
10213 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10214 match(Set dst (AddL src1 src2));
10215
10216 ins_cost(INSN_COST);
10217 format %{ "add $dst, $src1, $src2" %}
10218
10219 // use opcode to indicate that this is an add not a sub
10220 opcode(0x0);
10221
10222 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10223
10224 ins_pipe(ialu_reg_imm);
10225 %}
10226
10227 // Integer Subtraction
10228 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10229 match(Set dst (SubI src1 src2));
10230
10231 ins_cost(INSN_COST);
10232 format %{ "subw $dst, $src1, $src2" %}
10233
10234 ins_encode %{
10235 __ subw(as_Register($dst$$reg),
10236 as_Register($src1$$reg),
10237 as_Register($src2$$reg));
10238 %}
10239
10240 ins_pipe(ialu_reg_reg);
10241 %}
10242
10243 // Immediate Subtraction
10244 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10245 match(Set dst (SubI src1 src2));
10246
10247 ins_cost(INSN_COST);
10248 format %{ "subw $dst, $src1, $src2" %}
10249
10250 // use opcode to indicate that this is a sub not an add
10251 opcode(0x1);
10252
10253 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10254
10255 ins_pipe(ialu_reg_imm);
10256 %}
10257
10258 // Long Subtraction
10259 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10260
10261 match(Set dst (SubL src1 src2));
10262
10263 ins_cost(INSN_COST);
10264 format %{ "sub $dst, $src1, $src2" %}
10265
10266 ins_encode %{
10267 __ sub(as_Register($dst$$reg),
10268 as_Register($src1$$reg),
10269 as_Register($src2$$reg));
10270 %}
10271
10272 ins_pipe(ialu_reg_reg);
10273 %}
10274
10275 // No constant pool entries requiredLong Immediate Subtraction.
10276 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10277 match(Set dst (SubL src1 src2));
10278
10279 ins_cost(INSN_COST);
10280 format %{ "sub$dst, $src1, $src2" %}
10281
10282 // use opcode to indicate that this is a sub not an add
10283 opcode(0x1);
10284
10285 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10286
10287 ins_pipe(ialu_reg_imm);
10288 %}
10289
10290 // Integer Negation (special case for sub)
10291
10292 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10293 match(Set dst (SubI zero src));
10294
10295 ins_cost(INSN_COST);
10296 format %{ "negw $dst, $src\t# int" %}
10297
10298 ins_encode %{
10299 __ negw(as_Register($dst$$reg),
10300 as_Register($src$$reg));
10301 %}
10302
10303 ins_pipe(ialu_reg);
10304 %}
10305
10306 // Long Negation
10307
10308 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10309 match(Set dst (SubL zero src));
10310
10311 ins_cost(INSN_COST);
10312 format %{ "neg $dst, $src\t# long" %}
10313
10314 ins_encode %{
10315 __ neg(as_Register($dst$$reg),
10316 as_Register($src$$reg));
10317 %}
10318
10319 ins_pipe(ialu_reg);
10320 %}
10321
10322 // Integer Multiply
10323
10324 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10325 match(Set dst (MulI src1 src2));
10326
10327 ins_cost(INSN_COST * 3);
10328 format %{ "mulw $dst, $src1, $src2" %}
10329
10330 ins_encode %{
10331 __ mulw(as_Register($dst$$reg),
10332 as_Register($src1$$reg),
10333 as_Register($src2$$reg));
10334 %}
10335
10336 ins_pipe(imul_reg_reg);
10337 %}
10338
10339 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10340 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10341
10342 ins_cost(INSN_COST * 3);
10343 format %{ "smull $dst, $src1, $src2" %}
10344
10345 ins_encode %{
10346 __ smull(as_Register($dst$$reg),
10347 as_Register($src1$$reg),
10348 as_Register($src2$$reg));
10349 %}
10350
10351 ins_pipe(imul_reg_reg);
10352 %}
10353
10354 // Long Multiply
10355
10356 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10357 match(Set dst (MulL src1 src2));
10358
10359 ins_cost(INSN_COST * 5);
10360 format %{ "mul $dst, $src1, $src2" %}
10361
10362 ins_encode %{
10363 __ mul(as_Register($dst$$reg),
10364 as_Register($src1$$reg),
10365 as_Register($src2$$reg));
10366 %}
10367
10368 ins_pipe(lmul_reg_reg);
10369 %}
10370
10371 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10372 %{
10373 match(Set dst (MulHiL src1 src2));
10374
10375 ins_cost(INSN_COST * 7);
10376 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10377
10378 ins_encode %{
10379 __ smulh(as_Register($dst$$reg),
10380 as_Register($src1$$reg),
10381 as_Register($src2$$reg));
10382 %}
10383
10384 ins_pipe(lmul_reg_reg);
10385 %}
10386
10387 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10388 %{
10389 match(Set dst (UMulHiL src1 src2));
10390
10391 ins_cost(INSN_COST * 7);
10392 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10393
10394 ins_encode %{
10395 __ umulh(as_Register($dst$$reg),
10396 as_Register($src1$$reg),
10397 as_Register($src2$$reg));
10398 %}
10399
10400 ins_pipe(lmul_reg_reg);
10401 %}
10402
10403 // Combined Integer Multiply & Add/Sub
10404
10405 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10406 match(Set dst (AddI src3 (MulI src1 src2)));
10407
10408 ins_cost(INSN_COST * 3);
10409 format %{ "madd $dst, $src1, $src2, $src3" %}
10410
10411 ins_encode %{
10412 __ maddw(as_Register($dst$$reg),
10413 as_Register($src1$$reg),
10414 as_Register($src2$$reg),
10415 as_Register($src3$$reg));
10416 %}
10417
10418 ins_pipe(imac_reg_reg);
10419 %}
10420
10421 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10422 match(Set dst (SubI src3 (MulI src1 src2)));
10423
10424 ins_cost(INSN_COST * 3);
10425 format %{ "msub $dst, $src1, $src2, $src3" %}
10426
10427 ins_encode %{
10428 __ msubw(as_Register($dst$$reg),
10429 as_Register($src1$$reg),
10430 as_Register($src2$$reg),
10431 as_Register($src3$$reg));
10432 %}
10433
10434 ins_pipe(imac_reg_reg);
10435 %}
10436
10437 // Combined Integer Multiply & Neg
10438
10439 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10440 match(Set dst (MulI (SubI zero src1) src2));
10441
10442 ins_cost(INSN_COST * 3);
10443 format %{ "mneg $dst, $src1, $src2" %}
10444
10445 ins_encode %{
10446 __ mnegw(as_Register($dst$$reg),
10447 as_Register($src1$$reg),
10448 as_Register($src2$$reg));
10449 %}
10450
10451 ins_pipe(imac_reg_reg);
10452 %}
10453
10454 // Combined Long Multiply & Add/Sub
10455
10456 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10457 match(Set dst (AddL src3 (MulL src1 src2)));
10458
10459 ins_cost(INSN_COST * 5);
10460 format %{ "madd $dst, $src1, $src2, $src3" %}
10461
10462 ins_encode %{
10463 __ madd(as_Register($dst$$reg),
10464 as_Register($src1$$reg),
10465 as_Register($src2$$reg),
10466 as_Register($src3$$reg));
10467 %}
10468
10469 ins_pipe(lmac_reg_reg);
10470 %}
10471
10472 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10473 match(Set dst (SubL src3 (MulL src1 src2)));
10474
10475 ins_cost(INSN_COST * 5);
10476 format %{ "msub $dst, $src1, $src2, $src3" %}
10477
10478 ins_encode %{
10479 __ msub(as_Register($dst$$reg),
10480 as_Register($src1$$reg),
10481 as_Register($src2$$reg),
10482 as_Register($src3$$reg));
10483 %}
10484
10485 ins_pipe(lmac_reg_reg);
10486 %}
10487
10488 // Combined Long Multiply & Neg
10489
10490 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10491 match(Set dst (MulL (SubL zero src1) src2));
10492
10493 ins_cost(INSN_COST * 5);
10494 format %{ "mneg $dst, $src1, $src2" %}
10495
10496 ins_encode %{
10497 __ mneg(as_Register($dst$$reg),
10498 as_Register($src1$$reg),
10499 as_Register($src2$$reg));
10500 %}
10501
10502 ins_pipe(lmac_reg_reg);
10503 %}
10504
10505 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10506
10507 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10508 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10509
10510 ins_cost(INSN_COST * 3);
10511 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10512
10513 ins_encode %{
10514 __ smaddl(as_Register($dst$$reg),
10515 as_Register($src1$$reg),
10516 as_Register($src2$$reg),
10517 as_Register($src3$$reg));
10518 %}
10519
10520 ins_pipe(imac_reg_reg);
10521 %}
10522
10523 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10524 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10525
10526 ins_cost(INSN_COST * 3);
10527 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10528
10529 ins_encode %{
10530 __ smsubl(as_Register($dst$$reg),
10531 as_Register($src1$$reg),
10532 as_Register($src2$$reg),
10533 as_Register($src3$$reg));
10534 %}
10535
10536 ins_pipe(imac_reg_reg);
10537 %}
10538
10539 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10540 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10541
10542 ins_cost(INSN_COST * 3);
10543 format %{ "smnegl $dst, $src1, $src2" %}
10544
10545 ins_encode %{
10546 __ smnegl(as_Register($dst$$reg),
10547 as_Register($src1$$reg),
10548 as_Register($src2$$reg));
10549 %}
10550
10551 ins_pipe(imac_reg_reg);
10552 %}
10553
10554 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10555
10556 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10557 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10558
10559 ins_cost(INSN_COST * 5);
10560 format %{ "mulw rscratch1, $src1, $src2\n\t"
10561 "maddw $dst, $src3, $src4, rscratch1" %}
10562
10563 ins_encode %{
10564 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10565 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10566
10567 ins_pipe(imac_reg_reg);
10568 %}
10569
10570 // Integer Divide
10571
10572 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10573 match(Set dst (DivI src1 src2));
10574
10575 ins_cost(INSN_COST * 19);
10576 format %{ "sdivw $dst, $src1, $src2" %}
10577
10578 ins_encode(aarch64_enc_divw(dst, src1, src2));
10579 ins_pipe(idiv_reg_reg);
10580 %}
10581
10582 // Long Divide
10583
10584 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10585 match(Set dst (DivL src1 src2));
10586
10587 ins_cost(INSN_COST * 35);
10588 format %{ "sdiv $dst, $src1, $src2" %}
10589
10590 ins_encode(aarch64_enc_div(dst, src1, src2));
10591 ins_pipe(ldiv_reg_reg);
10592 %}
10593
10594 // Integer Remainder
10595
10596 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10597 match(Set dst (ModI src1 src2));
10598
10599 ins_cost(INSN_COST * 22);
10600 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10601 "msubw $dst, rscratch1, $src2, $src1" %}
10602
10603 ins_encode(aarch64_enc_modw(dst, src1, src2));
10604 ins_pipe(idiv_reg_reg);
10605 %}
10606
10607 // Long Remainder
10608
10609 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10610 match(Set dst (ModL src1 src2));
10611
10612 ins_cost(INSN_COST * 38);
10613 format %{ "sdiv rscratch1, $src1, $src2\n"
10614 "msub $dst, rscratch1, $src2, $src1" %}
10615
10616 ins_encode(aarch64_enc_mod(dst, src1, src2));
10617 ins_pipe(ldiv_reg_reg);
10618 %}
10619
10620 // Unsigned Integer Divide
10621
10622 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10623 match(Set dst (UDivI src1 src2));
10624
10625 ins_cost(INSN_COST * 19);
10626 format %{ "udivw $dst, $src1, $src2" %}
10627
10628 ins_encode %{
10629 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10630 %}
10631
10632 ins_pipe(idiv_reg_reg);
10633 %}
10634
10635 // Unsigned Long Divide
10636
10637 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10638 match(Set dst (UDivL src1 src2));
10639
10640 ins_cost(INSN_COST * 35);
10641 format %{ "udiv $dst, $src1, $src2" %}
10642
10643 ins_encode %{
10644 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10645 %}
10646
10647 ins_pipe(ldiv_reg_reg);
10648 %}
10649
10650 // Unsigned Integer Remainder
10651
10652 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10653 match(Set dst (UModI src1 src2));
10654
10655 ins_cost(INSN_COST * 22);
10656 format %{ "udivw rscratch1, $src1, $src2\n\t"
10657 "msubw $dst, rscratch1, $src2, $src1" %}
10658
10659 ins_encode %{
10660 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10661 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10662 %}
10663
10664 ins_pipe(idiv_reg_reg);
10665 %}
10666
10667 // Unsigned Long Remainder
10668
10669 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10670 match(Set dst (UModL src1 src2));
10671
10672 ins_cost(INSN_COST * 38);
10673 format %{ "udiv rscratch1, $src1, $src2\n"
10674 "msub $dst, rscratch1, $src2, $src1" %}
10675
10676 ins_encode %{
10677 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10678 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10679 %}
10680
10681 ins_pipe(ldiv_reg_reg);
10682 %}
10683
10684 // Integer Shifts
10685
10686 // Shift Left Register
10687 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10688 match(Set dst (LShiftI src1 src2));
10689
10690 ins_cost(INSN_COST * 2);
10691 format %{ "lslvw $dst, $src1, $src2" %}
10692
10693 ins_encode %{
10694 __ lslvw(as_Register($dst$$reg),
10695 as_Register($src1$$reg),
10696 as_Register($src2$$reg));
10697 %}
10698
10699 ins_pipe(ialu_reg_reg_vshift);
10700 %}
10701
10702 // Shift Left Immediate
10703 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10704 match(Set dst (LShiftI src1 src2));
10705
10706 ins_cost(INSN_COST);
10707 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10708
10709 ins_encode %{
10710 __ lslw(as_Register($dst$$reg),
10711 as_Register($src1$$reg),
10712 $src2$$constant & 0x1f);
10713 %}
10714
10715 ins_pipe(ialu_reg_shift);
10716 %}
10717
10718 // Shift Right Logical Register
10719 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10720 match(Set dst (URShiftI src1 src2));
10721
10722 ins_cost(INSN_COST * 2);
10723 format %{ "lsrvw $dst, $src1, $src2" %}
10724
10725 ins_encode %{
10726 __ lsrvw(as_Register($dst$$reg),
10727 as_Register($src1$$reg),
10728 as_Register($src2$$reg));
10729 %}
10730
10731 ins_pipe(ialu_reg_reg_vshift);
10732 %}
10733
10734 // Shift Right Logical Immediate
10735 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10736 match(Set dst (URShiftI src1 src2));
10737
10738 ins_cost(INSN_COST);
10739 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10740
10741 ins_encode %{
10742 __ lsrw(as_Register($dst$$reg),
10743 as_Register($src1$$reg),
10744 $src2$$constant & 0x1f);
10745 %}
10746
10747 ins_pipe(ialu_reg_shift);
10748 %}
10749
10750 // Shift Right Arithmetic Register
10751 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10752 match(Set dst (RShiftI src1 src2));
10753
10754 ins_cost(INSN_COST * 2);
10755 format %{ "asrvw $dst, $src1, $src2" %}
10756
10757 ins_encode %{
10758 __ asrvw(as_Register($dst$$reg),
10759 as_Register($src1$$reg),
10760 as_Register($src2$$reg));
10761 %}
10762
10763 ins_pipe(ialu_reg_reg_vshift);
10764 %}
10765
10766 // Shift Right Arithmetic Immediate
10767 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10768 match(Set dst (RShiftI src1 src2));
10769
10770 ins_cost(INSN_COST);
10771 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10772
10773 ins_encode %{
10774 __ asrw(as_Register($dst$$reg),
10775 as_Register($src1$$reg),
10776 $src2$$constant & 0x1f);
10777 %}
10778
10779 ins_pipe(ialu_reg_shift);
10780 %}
10781
10782 // Combined Int Mask and Right Shift (using UBFM)
10783 // TODO
10784
10785 // Long Shifts
10786
10787 // Shift Left Register
10788 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10789 match(Set dst (LShiftL src1 src2));
10790
10791 ins_cost(INSN_COST * 2);
10792 format %{ "lslv $dst, $src1, $src2" %}
10793
10794 ins_encode %{
10795 __ lslv(as_Register($dst$$reg),
10796 as_Register($src1$$reg),
10797 as_Register($src2$$reg));
10798 %}
10799
10800 ins_pipe(ialu_reg_reg_vshift);
10801 %}
10802
10803 // Shift Left Immediate
10804 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10805 match(Set dst (LShiftL src1 src2));
10806
10807 ins_cost(INSN_COST);
10808 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10809
10810 ins_encode %{
10811 __ lsl(as_Register($dst$$reg),
10812 as_Register($src1$$reg),
10813 $src2$$constant & 0x3f);
10814 %}
10815
10816 ins_pipe(ialu_reg_shift);
10817 %}
10818
10819 // Shift Right Logical Register
10820 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10821 match(Set dst (URShiftL src1 src2));
10822
10823 ins_cost(INSN_COST * 2);
10824 format %{ "lsrv $dst, $src1, $src2" %}
10825
10826 ins_encode %{
10827 __ lsrv(as_Register($dst$$reg),
10828 as_Register($src1$$reg),
10829 as_Register($src2$$reg));
10830 %}
10831
10832 ins_pipe(ialu_reg_reg_vshift);
10833 %}
10834
10835 // Shift Right Logical Immediate
10836 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10837 match(Set dst (URShiftL src1 src2));
10838
10839 ins_cost(INSN_COST);
10840 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10841
10842 ins_encode %{
10843 __ lsr(as_Register($dst$$reg),
10844 as_Register($src1$$reg),
10845 $src2$$constant & 0x3f);
10846 %}
10847
10848 ins_pipe(ialu_reg_shift);
10849 %}
10850
10851 // A special-case pattern for card table stores.
10852 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10853 match(Set dst (URShiftL (CastP2X src1) src2));
10854
10855 ins_cost(INSN_COST);
10856 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10857
10858 ins_encode %{
10859 __ lsr(as_Register($dst$$reg),
10860 as_Register($src1$$reg),
10861 $src2$$constant & 0x3f);
10862 %}
10863
10864 ins_pipe(ialu_reg_shift);
10865 %}
10866
10867 // Shift Right Arithmetic Register
10868 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10869 match(Set dst (RShiftL src1 src2));
10870
10871 ins_cost(INSN_COST * 2);
10872 format %{ "asrv $dst, $src1, $src2" %}
10873
10874 ins_encode %{
10875 __ asrv(as_Register($dst$$reg),
10876 as_Register($src1$$reg),
10877 as_Register($src2$$reg));
10878 %}
10879
10880 ins_pipe(ialu_reg_reg_vshift);
10881 %}
10882
10883 // Shift Right Arithmetic Immediate
10884 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10885 match(Set dst (RShiftL src1 src2));
10886
10887 ins_cost(INSN_COST);
10888 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10889
10890 ins_encode %{
10891 __ asr(as_Register($dst$$reg),
10892 as_Register($src1$$reg),
10893 $src2$$constant & 0x3f);
10894 %}
10895
10896 ins_pipe(ialu_reg_shift);
10897 %}
10898
10899 // BEGIN This section of the file is automatically generated. Do not edit --------------
10900 // This section is generated from aarch64_ad.m4
10901
10902 // This pattern is automatically generated from aarch64_ad.m4
10903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10904 instruct regL_not_reg(iRegLNoSp dst,
10905 iRegL src1, immL_M1 m1,
10906 rFlagsReg cr) %{
10907 match(Set dst (XorL src1 m1));
10908 ins_cost(INSN_COST);
10909 format %{ "eon $dst, $src1, zr" %}
10910
10911 ins_encode %{
10912 __ eon(as_Register($dst$$reg),
10913 as_Register($src1$$reg),
10914 zr,
10915 Assembler::LSL, 0);
10916 %}
10917
10918 ins_pipe(ialu_reg);
10919 %}
10920
10921 // This pattern is automatically generated from aarch64_ad.m4
10922 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10923 instruct regI_not_reg(iRegINoSp dst,
10924 iRegIorL2I src1, immI_M1 m1,
10925 rFlagsReg cr) %{
10926 match(Set dst (XorI src1 m1));
10927 ins_cost(INSN_COST);
10928 format %{ "eonw $dst, $src1, zr" %}
10929
10930 ins_encode %{
10931 __ eonw(as_Register($dst$$reg),
10932 as_Register($src1$$reg),
10933 zr,
10934 Assembler::LSL, 0);
10935 %}
10936
10937 ins_pipe(ialu_reg);
10938 %}
10939
10940 // This pattern is automatically generated from aarch64_ad.m4
10941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10942 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10943 immI0 zero, iRegIorL2I src1, immI src2) %{
10944 match(Set dst (SubI zero (URShiftI src1 src2)));
10945
10946 ins_cost(1.9 * INSN_COST);
10947 format %{ "negw $dst, $src1, LSR $src2" %}
10948
10949 ins_encode %{
10950 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10951 Assembler::LSR, $src2$$constant & 0x1f);
10952 %}
10953
10954 ins_pipe(ialu_reg_shift);
10955 %}
10956
10957 // This pattern is automatically generated from aarch64_ad.m4
10958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10959 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10960 immI0 zero, iRegIorL2I src1, immI src2) %{
10961 match(Set dst (SubI zero (RShiftI src1 src2)));
10962
10963 ins_cost(1.9 * INSN_COST);
10964 format %{ "negw $dst, $src1, ASR $src2" %}
10965
10966 ins_encode %{
10967 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10968 Assembler::ASR, $src2$$constant & 0x1f);
10969 %}
10970
10971 ins_pipe(ialu_reg_shift);
10972 %}
10973
10974 // This pattern is automatically generated from aarch64_ad.m4
10975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10976 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10977 immI0 zero, iRegIorL2I src1, immI src2) %{
10978 match(Set dst (SubI zero (LShiftI src1 src2)));
10979
10980 ins_cost(1.9 * INSN_COST);
10981 format %{ "negw $dst, $src1, LSL $src2" %}
10982
10983 ins_encode %{
10984 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10985 Assembler::LSL, $src2$$constant & 0x1f);
10986 %}
10987
10988 ins_pipe(ialu_reg_shift);
10989 %}
10990
10991 // This pattern is automatically generated from aarch64_ad.m4
10992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10993 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10994 immL0 zero, iRegL src1, immI src2) %{
10995 match(Set dst (SubL zero (URShiftL src1 src2)));
10996
10997 ins_cost(1.9 * INSN_COST);
10998 format %{ "neg $dst, $src1, LSR $src2" %}
10999
11000 ins_encode %{
11001 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11002 Assembler::LSR, $src2$$constant & 0x3f);
11003 %}
11004
11005 ins_pipe(ialu_reg_shift);
11006 %}
11007
11008 // This pattern is automatically generated from aarch64_ad.m4
11009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11010 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11011 immL0 zero, iRegL src1, immI src2) %{
11012 match(Set dst (SubL zero (RShiftL src1 src2)));
11013
11014 ins_cost(1.9 * INSN_COST);
11015 format %{ "neg $dst, $src1, ASR $src2" %}
11016
11017 ins_encode %{
11018 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11019 Assembler::ASR, $src2$$constant & 0x3f);
11020 %}
11021
11022 ins_pipe(ialu_reg_shift);
11023 %}
11024
11025 // This pattern is automatically generated from aarch64_ad.m4
11026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11027 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11028 immL0 zero, iRegL src1, immI src2) %{
11029 match(Set dst (SubL zero (LShiftL src1 src2)));
11030
11031 ins_cost(1.9 * INSN_COST);
11032 format %{ "neg $dst, $src1, LSL $src2" %}
11033
11034 ins_encode %{
11035 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11036 Assembler::LSL, $src2$$constant & 0x3f);
11037 %}
11038
11039 ins_pipe(ialu_reg_shift);
11040 %}
11041
11042 // This pattern is automatically generated from aarch64_ad.m4
11043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11044 instruct AndI_reg_not_reg(iRegINoSp dst,
11045 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11046 match(Set dst (AndI src1 (XorI src2 m1)));
11047 ins_cost(INSN_COST);
11048 format %{ "bicw $dst, $src1, $src2" %}
11049
11050 ins_encode %{
11051 __ bicw(as_Register($dst$$reg),
11052 as_Register($src1$$reg),
11053 as_Register($src2$$reg),
11054 Assembler::LSL, 0);
11055 %}
11056
11057 ins_pipe(ialu_reg_reg);
11058 %}
11059
11060 // This pattern is automatically generated from aarch64_ad.m4
11061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11062 instruct AndL_reg_not_reg(iRegLNoSp dst,
11063 iRegL src1, iRegL src2, immL_M1 m1) %{
11064 match(Set dst (AndL src1 (XorL src2 m1)));
11065 ins_cost(INSN_COST);
11066 format %{ "bic $dst, $src1, $src2" %}
11067
11068 ins_encode %{
11069 __ bic(as_Register($dst$$reg),
11070 as_Register($src1$$reg),
11071 as_Register($src2$$reg),
11072 Assembler::LSL, 0);
11073 %}
11074
11075 ins_pipe(ialu_reg_reg);
11076 %}
11077
11078 // This pattern is automatically generated from aarch64_ad.m4
11079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11080 instruct OrI_reg_not_reg(iRegINoSp dst,
11081 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11082 match(Set dst (OrI src1 (XorI src2 m1)));
11083 ins_cost(INSN_COST);
11084 format %{ "ornw $dst, $src1, $src2" %}
11085
11086 ins_encode %{
11087 __ ornw(as_Register($dst$$reg),
11088 as_Register($src1$$reg),
11089 as_Register($src2$$reg),
11090 Assembler::LSL, 0);
11091 %}
11092
11093 ins_pipe(ialu_reg_reg);
11094 %}
11095
11096 // This pattern is automatically generated from aarch64_ad.m4
11097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11098 instruct OrL_reg_not_reg(iRegLNoSp dst,
11099 iRegL src1, iRegL src2, immL_M1 m1) %{
11100 match(Set dst (OrL src1 (XorL src2 m1)));
11101 ins_cost(INSN_COST);
11102 format %{ "orn $dst, $src1, $src2" %}
11103
11104 ins_encode %{
11105 __ orn(as_Register($dst$$reg),
11106 as_Register($src1$$reg),
11107 as_Register($src2$$reg),
11108 Assembler::LSL, 0);
11109 %}
11110
11111 ins_pipe(ialu_reg_reg);
11112 %}
11113
11114 // This pattern is automatically generated from aarch64_ad.m4
11115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11116 instruct XorI_reg_not_reg(iRegINoSp dst,
11117 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11118 match(Set dst (XorI m1 (XorI src2 src1)));
11119 ins_cost(INSN_COST);
11120 format %{ "eonw $dst, $src1, $src2" %}
11121
11122 ins_encode %{
11123 __ eonw(as_Register($dst$$reg),
11124 as_Register($src1$$reg),
11125 as_Register($src2$$reg),
11126 Assembler::LSL, 0);
11127 %}
11128
11129 ins_pipe(ialu_reg_reg);
11130 %}
11131
11132 // This pattern is automatically generated from aarch64_ad.m4
11133 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11134 instruct XorL_reg_not_reg(iRegLNoSp dst,
11135 iRegL src1, iRegL src2, immL_M1 m1) %{
11136 match(Set dst (XorL m1 (XorL src2 src1)));
11137 ins_cost(INSN_COST);
11138 format %{ "eon $dst, $src1, $src2" %}
11139
11140 ins_encode %{
11141 __ eon(as_Register($dst$$reg),
11142 as_Register($src1$$reg),
11143 as_Register($src2$$reg),
11144 Assembler::LSL, 0);
11145 %}
11146
11147 ins_pipe(ialu_reg_reg);
11148 %}
11149
11150 // This pattern is automatically generated from aarch64_ad.m4
11151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11152 // val & (-1 ^ (val >>> shift)) ==> bicw
11153 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11154 iRegIorL2I src1, iRegIorL2I src2,
11155 immI src3, immI_M1 src4) %{
11156 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11157 ins_cost(1.9 * INSN_COST);
11158 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11159
11160 ins_encode %{
11161 __ bicw(as_Register($dst$$reg),
11162 as_Register($src1$$reg),
11163 as_Register($src2$$reg),
11164 Assembler::LSR,
11165 $src3$$constant & 0x1f);
11166 %}
11167
11168 ins_pipe(ialu_reg_reg_shift);
11169 %}
11170
11171 // This pattern is automatically generated from aarch64_ad.m4
11172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11173 // val & (-1 ^ (val >>> shift)) ==> bic
11174 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11175 iRegL src1, iRegL src2,
11176 immI src3, immL_M1 src4) %{
11177 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11178 ins_cost(1.9 * INSN_COST);
11179 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11180
11181 ins_encode %{
11182 __ bic(as_Register($dst$$reg),
11183 as_Register($src1$$reg),
11184 as_Register($src2$$reg),
11185 Assembler::LSR,
11186 $src3$$constant & 0x3f);
11187 %}
11188
11189 ins_pipe(ialu_reg_reg_shift);
11190 %}
11191
11192 // This pattern is automatically generated from aarch64_ad.m4
11193 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11194 // val & (-1 ^ (val >> shift)) ==> bicw
11195 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11196 iRegIorL2I src1, iRegIorL2I src2,
11197 immI src3, immI_M1 src4) %{
11198 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11199 ins_cost(1.9 * INSN_COST);
11200 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11201
11202 ins_encode %{
11203 __ bicw(as_Register($dst$$reg),
11204 as_Register($src1$$reg),
11205 as_Register($src2$$reg),
11206 Assembler::ASR,
11207 $src3$$constant & 0x1f);
11208 %}
11209
11210 ins_pipe(ialu_reg_reg_shift);
11211 %}
11212
11213 // This pattern is automatically generated from aarch64_ad.m4
11214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11215 // val & (-1 ^ (val >> shift)) ==> bic
11216 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11217 iRegL src1, iRegL src2,
11218 immI src3, immL_M1 src4) %{
11219 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11220 ins_cost(1.9 * INSN_COST);
11221 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11222
11223 ins_encode %{
11224 __ bic(as_Register($dst$$reg),
11225 as_Register($src1$$reg),
11226 as_Register($src2$$reg),
11227 Assembler::ASR,
11228 $src3$$constant & 0x3f);
11229 %}
11230
11231 ins_pipe(ialu_reg_reg_shift);
11232 %}
11233
11234 // This pattern is automatically generated from aarch64_ad.m4
11235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11236 // val & (-1 ^ (val ror shift)) ==> bicw
11237 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11238 iRegIorL2I src1, iRegIorL2I src2,
11239 immI src3, immI_M1 src4) %{
11240 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11241 ins_cost(1.9 * INSN_COST);
11242 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11243
11244 ins_encode %{
11245 __ bicw(as_Register($dst$$reg),
11246 as_Register($src1$$reg),
11247 as_Register($src2$$reg),
11248 Assembler::ROR,
11249 $src3$$constant & 0x1f);
11250 %}
11251
11252 ins_pipe(ialu_reg_reg_shift);
11253 %}
11254
11255 // This pattern is automatically generated from aarch64_ad.m4
11256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11257 // val & (-1 ^ (val ror shift)) ==> bic
11258 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11259 iRegL src1, iRegL src2,
11260 immI src3, immL_M1 src4) %{
11261 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11262 ins_cost(1.9 * INSN_COST);
11263 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11264
11265 ins_encode %{
11266 __ bic(as_Register($dst$$reg),
11267 as_Register($src1$$reg),
11268 as_Register($src2$$reg),
11269 Assembler::ROR,
11270 $src3$$constant & 0x3f);
11271 %}
11272
11273 ins_pipe(ialu_reg_reg_shift);
11274 %}
11275
11276 // This pattern is automatically generated from aarch64_ad.m4
11277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11278 // val & (-1 ^ (val << shift)) ==> bicw
11279 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11280 iRegIorL2I src1, iRegIorL2I src2,
11281 immI src3, immI_M1 src4) %{
11282 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11283 ins_cost(1.9 * INSN_COST);
11284 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11285
11286 ins_encode %{
11287 __ bicw(as_Register($dst$$reg),
11288 as_Register($src1$$reg),
11289 as_Register($src2$$reg),
11290 Assembler::LSL,
11291 $src3$$constant & 0x1f);
11292 %}
11293
11294 ins_pipe(ialu_reg_reg_shift);
11295 %}
11296
11297 // This pattern is automatically generated from aarch64_ad.m4
11298 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11299 // val & (-1 ^ (val << shift)) ==> bic
11300 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11301 iRegL src1, iRegL src2,
11302 immI src3, immL_M1 src4) %{
11303 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11304 ins_cost(1.9 * INSN_COST);
11305 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11306
11307 ins_encode %{
11308 __ bic(as_Register($dst$$reg),
11309 as_Register($src1$$reg),
11310 as_Register($src2$$reg),
11311 Assembler::LSL,
11312 $src3$$constant & 0x3f);
11313 %}
11314
11315 ins_pipe(ialu_reg_reg_shift);
11316 %}
11317
11318 // This pattern is automatically generated from aarch64_ad.m4
11319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11320 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11321 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11322 iRegIorL2I src1, iRegIorL2I src2,
11323 immI src3, immI_M1 src4) %{
11324 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11325 ins_cost(1.9 * INSN_COST);
11326 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11327
11328 ins_encode %{
11329 __ eonw(as_Register($dst$$reg),
11330 as_Register($src1$$reg),
11331 as_Register($src2$$reg),
11332 Assembler::LSR,
11333 $src3$$constant & 0x1f);
11334 %}
11335
11336 ins_pipe(ialu_reg_reg_shift);
11337 %}
11338
11339 // This pattern is automatically generated from aarch64_ad.m4
11340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11341 // val ^ (-1 ^ (val >>> shift)) ==> eon
11342 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11343 iRegL src1, iRegL src2,
11344 immI src3, immL_M1 src4) %{
11345 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11346 ins_cost(1.9 * INSN_COST);
11347 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11348
11349 ins_encode %{
11350 __ eon(as_Register($dst$$reg),
11351 as_Register($src1$$reg),
11352 as_Register($src2$$reg),
11353 Assembler::LSR,
11354 $src3$$constant & 0x3f);
11355 %}
11356
11357 ins_pipe(ialu_reg_reg_shift);
11358 %}
11359
11360 // This pattern is automatically generated from aarch64_ad.m4
11361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11362 // val ^ (-1 ^ (val >> shift)) ==> eonw
11363 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11364 iRegIorL2I src1, iRegIorL2I src2,
11365 immI src3, immI_M1 src4) %{
11366 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11367 ins_cost(1.9 * INSN_COST);
11368 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11369
11370 ins_encode %{
11371 __ eonw(as_Register($dst$$reg),
11372 as_Register($src1$$reg),
11373 as_Register($src2$$reg),
11374 Assembler::ASR,
11375 $src3$$constant & 0x1f);
11376 %}
11377
11378 ins_pipe(ialu_reg_reg_shift);
11379 %}
11380
11381 // This pattern is automatically generated from aarch64_ad.m4
11382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11383 // val ^ (-1 ^ (val >> shift)) ==> eon
11384 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11385 iRegL src1, iRegL src2,
11386 immI src3, immL_M1 src4) %{
11387 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11388 ins_cost(1.9 * INSN_COST);
11389 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11390
11391 ins_encode %{
11392 __ eon(as_Register($dst$$reg),
11393 as_Register($src1$$reg),
11394 as_Register($src2$$reg),
11395 Assembler::ASR,
11396 $src3$$constant & 0x3f);
11397 %}
11398
11399 ins_pipe(ialu_reg_reg_shift);
11400 %}
11401
11402 // This pattern is automatically generated from aarch64_ad.m4
11403 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11404 // val ^ (-1 ^ (val ror shift)) ==> eonw
11405 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11406 iRegIorL2I src1, iRegIorL2I src2,
11407 immI src3, immI_M1 src4) %{
11408 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11409 ins_cost(1.9 * INSN_COST);
11410 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11411
11412 ins_encode %{
11413 __ eonw(as_Register($dst$$reg),
11414 as_Register($src1$$reg),
11415 as_Register($src2$$reg),
11416 Assembler::ROR,
11417 $src3$$constant & 0x1f);
11418 %}
11419
11420 ins_pipe(ialu_reg_reg_shift);
11421 %}
11422
11423 // This pattern is automatically generated from aarch64_ad.m4
11424 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11425 // val ^ (-1 ^ (val ror shift)) ==> eon
11426 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11427 iRegL src1, iRegL src2,
11428 immI src3, immL_M1 src4) %{
11429 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11430 ins_cost(1.9 * INSN_COST);
11431 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11432
11433 ins_encode %{
11434 __ eon(as_Register($dst$$reg),
11435 as_Register($src1$$reg),
11436 as_Register($src2$$reg),
11437 Assembler::ROR,
11438 $src3$$constant & 0x3f);
11439 %}
11440
11441 ins_pipe(ialu_reg_reg_shift);
11442 %}
11443
11444 // This pattern is automatically generated from aarch64_ad.m4
11445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11446 // val ^ (-1 ^ (val << shift)) ==> eonw
11447 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11448 iRegIorL2I src1, iRegIorL2I src2,
11449 immI src3, immI_M1 src4) %{
11450 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11451 ins_cost(1.9 * INSN_COST);
11452 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11453
11454 ins_encode %{
11455 __ eonw(as_Register($dst$$reg),
11456 as_Register($src1$$reg),
11457 as_Register($src2$$reg),
11458 Assembler::LSL,
11459 $src3$$constant & 0x1f);
11460 %}
11461
11462 ins_pipe(ialu_reg_reg_shift);
11463 %}
11464
11465 // This pattern is automatically generated from aarch64_ad.m4
11466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11467 // val ^ (-1 ^ (val << shift)) ==> eon
11468 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11469 iRegL src1, iRegL src2,
11470 immI src3, immL_M1 src4) %{
11471 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11472 ins_cost(1.9 * INSN_COST);
11473 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11474
11475 ins_encode %{
11476 __ eon(as_Register($dst$$reg),
11477 as_Register($src1$$reg),
11478 as_Register($src2$$reg),
11479 Assembler::LSL,
11480 $src3$$constant & 0x3f);
11481 %}
11482
11483 ins_pipe(ialu_reg_reg_shift);
11484 %}
11485
11486 // This pattern is automatically generated from aarch64_ad.m4
11487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11488 // val | (-1 ^ (val >>> shift)) ==> ornw
11489 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11490 iRegIorL2I src1, iRegIorL2I src2,
11491 immI src3, immI_M1 src4) %{
11492 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11493 ins_cost(1.9 * INSN_COST);
11494 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11495
11496 ins_encode %{
11497 __ ornw(as_Register($dst$$reg),
11498 as_Register($src1$$reg),
11499 as_Register($src2$$reg),
11500 Assembler::LSR,
11501 $src3$$constant & 0x1f);
11502 %}
11503
11504 ins_pipe(ialu_reg_reg_shift);
11505 %}
11506
11507 // This pattern is automatically generated from aarch64_ad.m4
11508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11509 // val | (-1 ^ (val >>> shift)) ==> orn
11510 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11511 iRegL src1, iRegL src2,
11512 immI src3, immL_M1 src4) %{
11513 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11514 ins_cost(1.9 * INSN_COST);
11515 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11516
11517 ins_encode %{
11518 __ orn(as_Register($dst$$reg),
11519 as_Register($src1$$reg),
11520 as_Register($src2$$reg),
11521 Assembler::LSR,
11522 $src3$$constant & 0x3f);
11523 %}
11524
11525 ins_pipe(ialu_reg_reg_shift);
11526 %}
11527
11528 // This pattern is automatically generated from aarch64_ad.m4
11529 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11530 // val | (-1 ^ (val >> shift)) ==> ornw
11531 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11532 iRegIorL2I src1, iRegIorL2I src2,
11533 immI src3, immI_M1 src4) %{
11534 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11535 ins_cost(1.9 * INSN_COST);
11536 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11537
11538 ins_encode %{
11539 __ ornw(as_Register($dst$$reg),
11540 as_Register($src1$$reg),
11541 as_Register($src2$$reg),
11542 Assembler::ASR,
11543 $src3$$constant & 0x1f);
11544 %}
11545
11546 ins_pipe(ialu_reg_reg_shift);
11547 %}
11548
11549 // This pattern is automatically generated from aarch64_ad.m4
11550 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11551 // val | (-1 ^ (val >> shift)) ==> orn
11552 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11553 iRegL src1, iRegL src2,
11554 immI src3, immL_M1 src4) %{
11555 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11556 ins_cost(1.9 * INSN_COST);
11557 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11558
11559 ins_encode %{
11560 __ orn(as_Register($dst$$reg),
11561 as_Register($src1$$reg),
11562 as_Register($src2$$reg),
11563 Assembler::ASR,
11564 $src3$$constant & 0x3f);
11565 %}
11566
11567 ins_pipe(ialu_reg_reg_shift);
11568 %}
11569
11570 // This pattern is automatically generated from aarch64_ad.m4
11571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11572 // val | (-1 ^ (val ror shift)) ==> ornw
11573 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11574 iRegIorL2I src1, iRegIorL2I src2,
11575 immI src3, immI_M1 src4) %{
11576 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11577 ins_cost(1.9 * INSN_COST);
11578 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11579
11580 ins_encode %{
11581 __ ornw(as_Register($dst$$reg),
11582 as_Register($src1$$reg),
11583 as_Register($src2$$reg),
11584 Assembler::ROR,
11585 $src3$$constant & 0x1f);
11586 %}
11587
11588 ins_pipe(ialu_reg_reg_shift);
11589 %}
11590
11591 // This pattern is automatically generated from aarch64_ad.m4
11592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11593 // val | (-1 ^ (val ror shift)) ==> orn
11594 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11595 iRegL src1, iRegL src2,
11596 immI src3, immL_M1 src4) %{
11597 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11598 ins_cost(1.9 * INSN_COST);
11599 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11600
11601 ins_encode %{
11602 __ orn(as_Register($dst$$reg),
11603 as_Register($src1$$reg),
11604 as_Register($src2$$reg),
11605 Assembler::ROR,
11606 $src3$$constant & 0x3f);
11607 %}
11608
11609 ins_pipe(ialu_reg_reg_shift);
11610 %}
11611
11612 // This pattern is automatically generated from aarch64_ad.m4
11613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11614 // val | (-1 ^ (val << shift)) ==> ornw
11615 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11616 iRegIorL2I src1, iRegIorL2I src2,
11617 immI src3, immI_M1 src4) %{
11618 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11619 ins_cost(1.9 * INSN_COST);
11620 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11621
11622 ins_encode %{
11623 __ ornw(as_Register($dst$$reg),
11624 as_Register($src1$$reg),
11625 as_Register($src2$$reg),
11626 Assembler::LSL,
11627 $src3$$constant & 0x1f);
11628 %}
11629
11630 ins_pipe(ialu_reg_reg_shift);
11631 %}
11632
11633 // This pattern is automatically generated from aarch64_ad.m4
11634 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11635 // val | (-1 ^ (val << shift)) ==> orn
11636 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11637 iRegL src1, iRegL src2,
11638 immI src3, immL_M1 src4) %{
11639 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11640 ins_cost(1.9 * INSN_COST);
11641 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11642
11643 ins_encode %{
11644 __ orn(as_Register($dst$$reg),
11645 as_Register($src1$$reg),
11646 as_Register($src2$$reg),
11647 Assembler::LSL,
11648 $src3$$constant & 0x3f);
11649 %}
11650
11651 ins_pipe(ialu_reg_reg_shift);
11652 %}
11653
11654 // This pattern is automatically generated from aarch64_ad.m4
11655 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11656 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11657 iRegIorL2I src1, iRegIorL2I src2,
11658 immI src3) %{
11659 match(Set dst (AndI src1 (URShiftI src2 src3)));
11660
11661 ins_cost(1.9 * INSN_COST);
11662 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11663
11664 ins_encode %{
11665 __ andw(as_Register($dst$$reg),
11666 as_Register($src1$$reg),
11667 as_Register($src2$$reg),
11668 Assembler::LSR,
11669 $src3$$constant & 0x1f);
11670 %}
11671
11672 ins_pipe(ialu_reg_reg_shift);
11673 %}
11674
11675 // This pattern is automatically generated from aarch64_ad.m4
11676 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11677 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11678 iRegL src1, iRegL src2,
11679 immI src3) %{
11680 match(Set dst (AndL src1 (URShiftL src2 src3)));
11681
11682 ins_cost(1.9 * INSN_COST);
11683 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11684
11685 ins_encode %{
11686 __ andr(as_Register($dst$$reg),
11687 as_Register($src1$$reg),
11688 as_Register($src2$$reg),
11689 Assembler::LSR,
11690 $src3$$constant & 0x3f);
11691 %}
11692
11693 ins_pipe(ialu_reg_reg_shift);
11694 %}
11695
11696 // This pattern is automatically generated from aarch64_ad.m4
11697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11698 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11699 iRegIorL2I src1, iRegIorL2I src2,
11700 immI src3) %{
11701 match(Set dst (AndI src1 (RShiftI src2 src3)));
11702
11703 ins_cost(1.9 * INSN_COST);
11704 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11705
11706 ins_encode %{
11707 __ andw(as_Register($dst$$reg),
11708 as_Register($src1$$reg),
11709 as_Register($src2$$reg),
11710 Assembler::ASR,
11711 $src3$$constant & 0x1f);
11712 %}
11713
11714 ins_pipe(ialu_reg_reg_shift);
11715 %}
11716
11717 // This pattern is automatically generated from aarch64_ad.m4
11718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11719 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11720 iRegL src1, iRegL src2,
11721 immI src3) %{
11722 match(Set dst (AndL src1 (RShiftL src2 src3)));
11723
11724 ins_cost(1.9 * INSN_COST);
11725 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11726
11727 ins_encode %{
11728 __ andr(as_Register($dst$$reg),
11729 as_Register($src1$$reg),
11730 as_Register($src2$$reg),
11731 Assembler::ASR,
11732 $src3$$constant & 0x3f);
11733 %}
11734
11735 ins_pipe(ialu_reg_reg_shift);
11736 %}
11737
11738 // This pattern is automatically generated from aarch64_ad.m4
11739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11740 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11741 iRegIorL2I src1, iRegIorL2I src2,
11742 immI src3) %{
11743 match(Set dst (AndI src1 (LShiftI src2 src3)));
11744
11745 ins_cost(1.9 * INSN_COST);
11746 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11747
11748 ins_encode %{
11749 __ andw(as_Register($dst$$reg),
11750 as_Register($src1$$reg),
11751 as_Register($src2$$reg),
11752 Assembler::LSL,
11753 $src3$$constant & 0x1f);
11754 %}
11755
11756 ins_pipe(ialu_reg_reg_shift);
11757 %}
11758
11759 // This pattern is automatically generated from aarch64_ad.m4
11760 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11761 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11762 iRegL src1, iRegL src2,
11763 immI src3) %{
11764 match(Set dst (AndL src1 (LShiftL src2 src3)));
11765
11766 ins_cost(1.9 * INSN_COST);
11767 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11768
11769 ins_encode %{
11770 __ andr(as_Register($dst$$reg),
11771 as_Register($src1$$reg),
11772 as_Register($src2$$reg),
11773 Assembler::LSL,
11774 $src3$$constant & 0x3f);
11775 %}
11776
11777 ins_pipe(ialu_reg_reg_shift);
11778 %}
11779
11780 // This pattern is automatically generated from aarch64_ad.m4
11781 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11782 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11783 iRegIorL2I src1, iRegIorL2I src2,
11784 immI src3) %{
11785 match(Set dst (AndI src1 (RotateRight src2 src3)));
11786
11787 ins_cost(1.9 * INSN_COST);
11788 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11789
11790 ins_encode %{
11791 __ andw(as_Register($dst$$reg),
11792 as_Register($src1$$reg),
11793 as_Register($src2$$reg),
11794 Assembler::ROR,
11795 $src3$$constant & 0x1f);
11796 %}
11797
11798 ins_pipe(ialu_reg_reg_shift);
11799 %}
11800
11801 // This pattern is automatically generated from aarch64_ad.m4
11802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11803 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11804 iRegL src1, iRegL src2,
11805 immI src3) %{
11806 match(Set dst (AndL src1 (RotateRight src2 src3)));
11807
11808 ins_cost(1.9 * INSN_COST);
11809 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11810
11811 ins_encode %{
11812 __ andr(as_Register($dst$$reg),
11813 as_Register($src1$$reg),
11814 as_Register($src2$$reg),
11815 Assembler::ROR,
11816 $src3$$constant & 0x3f);
11817 %}
11818
11819 ins_pipe(ialu_reg_reg_shift);
11820 %}
11821
11822 // This pattern is automatically generated from aarch64_ad.m4
11823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11824 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11825 iRegIorL2I src1, iRegIorL2I src2,
11826 immI src3) %{
11827 match(Set dst (XorI src1 (URShiftI src2 src3)));
11828
11829 ins_cost(1.9 * INSN_COST);
11830 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11831
11832 ins_encode %{
11833 __ eorw(as_Register($dst$$reg),
11834 as_Register($src1$$reg),
11835 as_Register($src2$$reg),
11836 Assembler::LSR,
11837 $src3$$constant & 0x1f);
11838 %}
11839
11840 ins_pipe(ialu_reg_reg_shift);
11841 %}
11842
11843 // This pattern is automatically generated from aarch64_ad.m4
11844 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11845 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11846 iRegL src1, iRegL src2,
11847 immI src3) %{
11848 match(Set dst (XorL src1 (URShiftL src2 src3)));
11849
11850 ins_cost(1.9 * INSN_COST);
11851 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11852
11853 ins_encode %{
11854 __ eor(as_Register($dst$$reg),
11855 as_Register($src1$$reg),
11856 as_Register($src2$$reg),
11857 Assembler::LSR,
11858 $src3$$constant & 0x3f);
11859 %}
11860
11861 ins_pipe(ialu_reg_reg_shift);
11862 %}
11863
11864 // This pattern is automatically generated from aarch64_ad.m4
11865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11866 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11867 iRegIorL2I src1, iRegIorL2I src2,
11868 immI src3) %{
11869 match(Set dst (XorI src1 (RShiftI src2 src3)));
11870
11871 ins_cost(1.9 * INSN_COST);
11872 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11873
11874 ins_encode %{
11875 __ eorw(as_Register($dst$$reg),
11876 as_Register($src1$$reg),
11877 as_Register($src2$$reg),
11878 Assembler::ASR,
11879 $src3$$constant & 0x1f);
11880 %}
11881
11882 ins_pipe(ialu_reg_reg_shift);
11883 %}
11884
11885 // This pattern is automatically generated from aarch64_ad.m4
11886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11887 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11888 iRegL src1, iRegL src2,
11889 immI src3) %{
11890 match(Set dst (XorL src1 (RShiftL src2 src3)));
11891
11892 ins_cost(1.9 * INSN_COST);
11893 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11894
11895 ins_encode %{
11896 __ eor(as_Register($dst$$reg),
11897 as_Register($src1$$reg),
11898 as_Register($src2$$reg),
11899 Assembler::ASR,
11900 $src3$$constant & 0x3f);
11901 %}
11902
11903 ins_pipe(ialu_reg_reg_shift);
11904 %}
11905
11906 // This pattern is automatically generated from aarch64_ad.m4
11907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11908 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11909 iRegIorL2I src1, iRegIorL2I src2,
11910 immI src3) %{
11911 match(Set dst (XorI src1 (LShiftI src2 src3)));
11912
11913 ins_cost(1.9 * INSN_COST);
11914 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11915
11916 ins_encode %{
11917 __ eorw(as_Register($dst$$reg),
11918 as_Register($src1$$reg),
11919 as_Register($src2$$reg),
11920 Assembler::LSL,
11921 $src3$$constant & 0x1f);
11922 %}
11923
11924 ins_pipe(ialu_reg_reg_shift);
11925 %}
11926
11927 // This pattern is automatically generated from aarch64_ad.m4
11928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11929 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11930 iRegL src1, iRegL src2,
11931 immI src3) %{
11932 match(Set dst (XorL src1 (LShiftL src2 src3)));
11933
11934 ins_cost(1.9 * INSN_COST);
11935 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11936
11937 ins_encode %{
11938 __ eor(as_Register($dst$$reg),
11939 as_Register($src1$$reg),
11940 as_Register($src2$$reg),
11941 Assembler::LSL,
11942 $src3$$constant & 0x3f);
11943 %}
11944
11945 ins_pipe(ialu_reg_reg_shift);
11946 %}
11947
11948 // This pattern is automatically generated from aarch64_ad.m4
11949 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11950 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11951 iRegIorL2I src1, iRegIorL2I src2,
11952 immI src3) %{
11953 match(Set dst (XorI src1 (RotateRight src2 src3)));
11954
11955 ins_cost(1.9 * INSN_COST);
11956 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11957
11958 ins_encode %{
11959 __ eorw(as_Register($dst$$reg),
11960 as_Register($src1$$reg),
11961 as_Register($src2$$reg),
11962 Assembler::ROR,
11963 $src3$$constant & 0x1f);
11964 %}
11965
11966 ins_pipe(ialu_reg_reg_shift);
11967 %}
11968
11969 // This pattern is automatically generated from aarch64_ad.m4
11970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11971 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11972 iRegL src1, iRegL src2,
11973 immI src3) %{
11974 match(Set dst (XorL src1 (RotateRight src2 src3)));
11975
11976 ins_cost(1.9 * INSN_COST);
11977 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11978
11979 ins_encode %{
11980 __ eor(as_Register($dst$$reg),
11981 as_Register($src1$$reg),
11982 as_Register($src2$$reg),
11983 Assembler::ROR,
11984 $src3$$constant & 0x3f);
11985 %}
11986
11987 ins_pipe(ialu_reg_reg_shift);
11988 %}
11989
11990 // This pattern is automatically generated from aarch64_ad.m4
11991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11992 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11993 iRegIorL2I src1, iRegIorL2I src2,
11994 immI src3) %{
11995 match(Set dst (OrI src1 (URShiftI src2 src3)));
11996
11997 ins_cost(1.9 * INSN_COST);
11998 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11999
12000 ins_encode %{
12001 __ orrw(as_Register($dst$$reg),
12002 as_Register($src1$$reg),
12003 as_Register($src2$$reg),
12004 Assembler::LSR,
12005 $src3$$constant & 0x1f);
12006 %}
12007
12008 ins_pipe(ialu_reg_reg_shift);
12009 %}
12010
12011 // This pattern is automatically generated from aarch64_ad.m4
12012 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12013 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12014 iRegL src1, iRegL src2,
12015 immI src3) %{
12016 match(Set dst (OrL src1 (URShiftL src2 src3)));
12017
12018 ins_cost(1.9 * INSN_COST);
12019 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12020
12021 ins_encode %{
12022 __ orr(as_Register($dst$$reg),
12023 as_Register($src1$$reg),
12024 as_Register($src2$$reg),
12025 Assembler::LSR,
12026 $src3$$constant & 0x3f);
12027 %}
12028
12029 ins_pipe(ialu_reg_reg_shift);
12030 %}
12031
12032 // This pattern is automatically generated from aarch64_ad.m4
12033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12034 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12035 iRegIorL2I src1, iRegIorL2I src2,
12036 immI src3) %{
12037 match(Set dst (OrI src1 (RShiftI src2 src3)));
12038
12039 ins_cost(1.9 * INSN_COST);
12040 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12041
12042 ins_encode %{
12043 __ orrw(as_Register($dst$$reg),
12044 as_Register($src1$$reg),
12045 as_Register($src2$$reg),
12046 Assembler::ASR,
12047 $src3$$constant & 0x1f);
12048 %}
12049
12050 ins_pipe(ialu_reg_reg_shift);
12051 %}
12052
12053 // This pattern is automatically generated from aarch64_ad.m4
12054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12055 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12056 iRegL src1, iRegL src2,
12057 immI src3) %{
12058 match(Set dst (OrL src1 (RShiftL src2 src3)));
12059
12060 ins_cost(1.9 * INSN_COST);
12061 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12062
12063 ins_encode %{
12064 __ orr(as_Register($dst$$reg),
12065 as_Register($src1$$reg),
12066 as_Register($src2$$reg),
12067 Assembler::ASR,
12068 $src3$$constant & 0x3f);
12069 %}
12070
12071 ins_pipe(ialu_reg_reg_shift);
12072 %}
12073
12074 // This pattern is automatically generated from aarch64_ad.m4
12075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12076 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12077 iRegIorL2I src1, iRegIorL2I src2,
12078 immI src3) %{
12079 match(Set dst (OrI src1 (LShiftI src2 src3)));
12080
12081 ins_cost(1.9 * INSN_COST);
12082 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12083
12084 ins_encode %{
12085 __ orrw(as_Register($dst$$reg),
12086 as_Register($src1$$reg),
12087 as_Register($src2$$reg),
12088 Assembler::LSL,
12089 $src3$$constant & 0x1f);
12090 %}
12091
12092 ins_pipe(ialu_reg_reg_shift);
12093 %}
12094
12095 // This pattern is automatically generated from aarch64_ad.m4
12096 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12097 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12098 iRegL src1, iRegL src2,
12099 immI src3) %{
12100 match(Set dst (OrL src1 (LShiftL src2 src3)));
12101
12102 ins_cost(1.9 * INSN_COST);
12103 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12104
12105 ins_encode %{
12106 __ orr(as_Register($dst$$reg),
12107 as_Register($src1$$reg),
12108 as_Register($src2$$reg),
12109 Assembler::LSL,
12110 $src3$$constant & 0x3f);
12111 %}
12112
12113 ins_pipe(ialu_reg_reg_shift);
12114 %}
12115
12116 // This pattern is automatically generated from aarch64_ad.m4
12117 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12118 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12119 iRegIorL2I src1, iRegIorL2I src2,
12120 immI src3) %{
12121 match(Set dst (OrI src1 (RotateRight src2 src3)));
12122
12123 ins_cost(1.9 * INSN_COST);
12124 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12125
12126 ins_encode %{
12127 __ orrw(as_Register($dst$$reg),
12128 as_Register($src1$$reg),
12129 as_Register($src2$$reg),
12130 Assembler::ROR,
12131 $src3$$constant & 0x1f);
12132 %}
12133
12134 ins_pipe(ialu_reg_reg_shift);
12135 %}
12136
12137 // This pattern is automatically generated from aarch64_ad.m4
12138 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12139 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12140 iRegL src1, iRegL src2,
12141 immI src3) %{
12142 match(Set dst (OrL src1 (RotateRight src2 src3)));
12143
12144 ins_cost(1.9 * INSN_COST);
12145 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12146
12147 ins_encode %{
12148 __ orr(as_Register($dst$$reg),
12149 as_Register($src1$$reg),
12150 as_Register($src2$$reg),
12151 Assembler::ROR,
12152 $src3$$constant & 0x3f);
12153 %}
12154
12155 ins_pipe(ialu_reg_reg_shift);
12156 %}
12157
12158 // This pattern is automatically generated from aarch64_ad.m4
12159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12160 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12161 iRegIorL2I src1, iRegIorL2I src2,
12162 immI src3) %{
12163 match(Set dst (AddI src1 (URShiftI src2 src3)));
12164
12165 ins_cost(1.9 * INSN_COST);
12166 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12167
12168 ins_encode %{
12169 __ addw(as_Register($dst$$reg),
12170 as_Register($src1$$reg),
12171 as_Register($src2$$reg),
12172 Assembler::LSR,
12173 $src3$$constant & 0x1f);
12174 %}
12175
12176 ins_pipe(ialu_reg_reg_shift);
12177 %}
12178
12179 // This pattern is automatically generated from aarch64_ad.m4
12180 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12181 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12182 iRegL src1, iRegL src2,
12183 immI src3) %{
12184 match(Set dst (AddL src1 (URShiftL src2 src3)));
12185
12186 ins_cost(1.9 * INSN_COST);
12187 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12188
12189 ins_encode %{
12190 __ add(as_Register($dst$$reg),
12191 as_Register($src1$$reg),
12192 as_Register($src2$$reg),
12193 Assembler::LSR,
12194 $src3$$constant & 0x3f);
12195 %}
12196
12197 ins_pipe(ialu_reg_reg_shift);
12198 %}
12199
12200 // This pattern is automatically generated from aarch64_ad.m4
12201 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12202 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12203 iRegIorL2I src1, iRegIorL2I src2,
12204 immI src3) %{
12205 match(Set dst (AddI src1 (RShiftI src2 src3)));
12206
12207 ins_cost(1.9 * INSN_COST);
12208 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12209
12210 ins_encode %{
12211 __ addw(as_Register($dst$$reg),
12212 as_Register($src1$$reg),
12213 as_Register($src2$$reg),
12214 Assembler::ASR,
12215 $src3$$constant & 0x1f);
12216 %}
12217
12218 ins_pipe(ialu_reg_reg_shift);
12219 %}
12220
12221 // This pattern is automatically generated from aarch64_ad.m4
12222 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12223 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12224 iRegL src1, iRegL src2,
12225 immI src3) %{
12226 match(Set dst (AddL src1 (RShiftL src2 src3)));
12227
12228 ins_cost(1.9 * INSN_COST);
12229 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12230
12231 ins_encode %{
12232 __ add(as_Register($dst$$reg),
12233 as_Register($src1$$reg),
12234 as_Register($src2$$reg),
12235 Assembler::ASR,
12236 $src3$$constant & 0x3f);
12237 %}
12238
12239 ins_pipe(ialu_reg_reg_shift);
12240 %}
12241
12242 // This pattern is automatically generated from aarch64_ad.m4
12243 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12244 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12245 iRegIorL2I src1, iRegIorL2I src2,
12246 immI src3) %{
12247 match(Set dst (AddI src1 (LShiftI src2 src3)));
12248
12249 ins_cost(1.9 * INSN_COST);
12250 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12251
12252 ins_encode %{
12253 __ addw(as_Register($dst$$reg),
12254 as_Register($src1$$reg),
12255 as_Register($src2$$reg),
12256 Assembler::LSL,
12257 $src3$$constant & 0x1f);
12258 %}
12259
12260 ins_pipe(ialu_reg_reg_shift);
12261 %}
12262
12263 // This pattern is automatically generated from aarch64_ad.m4
12264 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12265 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12266 iRegL src1, iRegL src2,
12267 immI src3) %{
12268 match(Set dst (AddL src1 (LShiftL src2 src3)));
12269
12270 ins_cost(1.9 * INSN_COST);
12271 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12272
12273 ins_encode %{
12274 __ add(as_Register($dst$$reg),
12275 as_Register($src1$$reg),
12276 as_Register($src2$$reg),
12277 Assembler::LSL,
12278 $src3$$constant & 0x3f);
12279 %}
12280
12281 ins_pipe(ialu_reg_reg_shift);
12282 %}
12283
12284 // This pattern is automatically generated from aarch64_ad.m4
12285 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12286 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12287 iRegIorL2I src1, iRegIorL2I src2,
12288 immI src3) %{
12289 match(Set dst (SubI src1 (URShiftI src2 src3)));
12290
12291 ins_cost(1.9 * INSN_COST);
12292 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12293
12294 ins_encode %{
12295 __ subw(as_Register($dst$$reg),
12296 as_Register($src1$$reg),
12297 as_Register($src2$$reg),
12298 Assembler::LSR,
12299 $src3$$constant & 0x1f);
12300 %}
12301
12302 ins_pipe(ialu_reg_reg_shift);
12303 %}
12304
12305 // This pattern is automatically generated from aarch64_ad.m4
12306 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12307 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12308 iRegL src1, iRegL src2,
12309 immI src3) %{
12310 match(Set dst (SubL src1 (URShiftL src2 src3)));
12311
12312 ins_cost(1.9 * INSN_COST);
12313 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12314
12315 ins_encode %{
12316 __ sub(as_Register($dst$$reg),
12317 as_Register($src1$$reg),
12318 as_Register($src2$$reg),
12319 Assembler::LSR,
12320 $src3$$constant & 0x3f);
12321 %}
12322
12323 ins_pipe(ialu_reg_reg_shift);
12324 %}
12325
12326 // This pattern is automatically generated from aarch64_ad.m4
12327 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12328 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12329 iRegIorL2I src1, iRegIorL2I src2,
12330 immI src3) %{
12331 match(Set dst (SubI src1 (RShiftI src2 src3)));
12332
12333 ins_cost(1.9 * INSN_COST);
12334 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12335
12336 ins_encode %{
12337 __ subw(as_Register($dst$$reg),
12338 as_Register($src1$$reg),
12339 as_Register($src2$$reg),
12340 Assembler::ASR,
12341 $src3$$constant & 0x1f);
12342 %}
12343
12344 ins_pipe(ialu_reg_reg_shift);
12345 %}
12346
12347 // This pattern is automatically generated from aarch64_ad.m4
12348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12349 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12350 iRegL src1, iRegL src2,
12351 immI src3) %{
12352 match(Set dst (SubL src1 (RShiftL src2 src3)));
12353
12354 ins_cost(1.9 * INSN_COST);
12355 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12356
12357 ins_encode %{
12358 __ sub(as_Register($dst$$reg),
12359 as_Register($src1$$reg),
12360 as_Register($src2$$reg),
12361 Assembler::ASR,
12362 $src3$$constant & 0x3f);
12363 %}
12364
12365 ins_pipe(ialu_reg_reg_shift);
12366 %}
12367
12368 // This pattern is automatically generated from aarch64_ad.m4
12369 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12370 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12371 iRegIorL2I src1, iRegIorL2I src2,
12372 immI src3) %{
12373 match(Set dst (SubI src1 (LShiftI src2 src3)));
12374
12375 ins_cost(1.9 * INSN_COST);
12376 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12377
12378 ins_encode %{
12379 __ subw(as_Register($dst$$reg),
12380 as_Register($src1$$reg),
12381 as_Register($src2$$reg),
12382 Assembler::LSL,
12383 $src3$$constant & 0x1f);
12384 %}
12385
12386 ins_pipe(ialu_reg_reg_shift);
12387 %}
12388
12389 // This pattern is automatically generated from aarch64_ad.m4
12390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12391 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12392 iRegL src1, iRegL src2,
12393 immI src3) %{
12394 match(Set dst (SubL src1 (LShiftL src2 src3)));
12395
12396 ins_cost(1.9 * INSN_COST);
12397 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12398
12399 ins_encode %{
12400 __ sub(as_Register($dst$$reg),
12401 as_Register($src1$$reg),
12402 as_Register($src2$$reg),
12403 Assembler::LSL,
12404 $src3$$constant & 0x3f);
12405 %}
12406
12407 ins_pipe(ialu_reg_reg_shift);
12408 %}
12409
12410 // This pattern is automatically generated from aarch64_ad.m4
12411 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12412
12413 // Shift Left followed by Shift Right.
12414 // This idiom is used by the compiler for the i2b bytecode etc.
12415 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12416 %{
12417 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12418 ins_cost(INSN_COST * 2);
12419 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12420 ins_encode %{
12421 int lshift = $lshift_count$$constant & 63;
12422 int rshift = $rshift_count$$constant & 63;
12423 int s = 63 - lshift;
12424 int r = (rshift - lshift) & 63;
12425 __ sbfm(as_Register($dst$$reg),
12426 as_Register($src$$reg),
12427 r, s);
12428 %}
12429
12430 ins_pipe(ialu_reg_shift);
12431 %}
12432
12433 // This pattern is automatically generated from aarch64_ad.m4
12434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12435
12436 // Shift Left followed by Shift Right.
12437 // This idiom is used by the compiler for the i2b bytecode etc.
12438 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12439 %{
12440 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12441 ins_cost(INSN_COST * 2);
12442 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12443 ins_encode %{
12444 int lshift = $lshift_count$$constant & 31;
12445 int rshift = $rshift_count$$constant & 31;
12446 int s = 31 - lshift;
12447 int r = (rshift - lshift) & 31;
12448 __ sbfmw(as_Register($dst$$reg),
12449 as_Register($src$$reg),
12450 r, s);
12451 %}
12452
12453 ins_pipe(ialu_reg_shift);
12454 %}
12455
12456 // This pattern is automatically generated from aarch64_ad.m4
12457 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12458
12459 // Shift Left followed by Shift Right.
12460 // This idiom is used by the compiler for the i2b bytecode etc.
12461 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12462 %{
12463 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12464 ins_cost(INSN_COST * 2);
12465 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12466 ins_encode %{
12467 int lshift = $lshift_count$$constant & 63;
12468 int rshift = $rshift_count$$constant & 63;
12469 int s = 63 - lshift;
12470 int r = (rshift - lshift) & 63;
12471 __ ubfm(as_Register($dst$$reg),
12472 as_Register($src$$reg),
12473 r, s);
12474 %}
12475
12476 ins_pipe(ialu_reg_shift);
12477 %}
12478
12479 // This pattern is automatically generated from aarch64_ad.m4
12480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12481
12482 // Shift Left followed by Shift Right.
12483 // This idiom is used by the compiler for the i2b bytecode etc.
12484 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12485 %{
12486 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12487 ins_cost(INSN_COST * 2);
12488 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12489 ins_encode %{
12490 int lshift = $lshift_count$$constant & 31;
12491 int rshift = $rshift_count$$constant & 31;
12492 int s = 31 - lshift;
12493 int r = (rshift - lshift) & 31;
12494 __ ubfmw(as_Register($dst$$reg),
12495 as_Register($src$$reg),
12496 r, s);
12497 %}
12498
12499 ins_pipe(ialu_reg_shift);
12500 %}
12501
12502 // Bitfield extract with shift & mask
12503
12504 // This pattern is automatically generated from aarch64_ad.m4
12505 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12506 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12507 %{
12508 match(Set dst (AndI (URShiftI src rshift) mask));
12509 // Make sure we are not going to exceed what ubfxw can do.
12510 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12511
12512 ins_cost(INSN_COST);
12513 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12514 ins_encode %{
12515 int rshift = $rshift$$constant & 31;
12516 intptr_t mask = $mask$$constant;
12517 int width = exact_log2(mask+1);
12518 __ ubfxw(as_Register($dst$$reg),
12519 as_Register($src$$reg), rshift, width);
12520 %}
12521 ins_pipe(ialu_reg_shift);
12522 %}
12523
12524 // This pattern is automatically generated from aarch64_ad.m4
12525 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12526 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12527 %{
12528 match(Set dst (AndL (URShiftL src rshift) mask));
12529 // Make sure we are not going to exceed what ubfx can do.
12530 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12531
12532 ins_cost(INSN_COST);
12533 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12534 ins_encode %{
12535 int rshift = $rshift$$constant & 63;
12536 intptr_t mask = $mask$$constant;
12537 int width = exact_log2_long(mask+1);
12538 __ ubfx(as_Register($dst$$reg),
12539 as_Register($src$$reg), rshift, width);
12540 %}
12541 ins_pipe(ialu_reg_shift);
12542 %}
12543
12544
12545 // This pattern is automatically generated from aarch64_ad.m4
12546 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12547
12548 // We can use ubfx when extending an And with a mask when we know mask
12549 // is positive. We know that because immI_bitmask guarantees it.
12550 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12551 %{
12552 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12553 // Make sure we are not going to exceed what ubfxw can do.
12554 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12555
12556 ins_cost(INSN_COST * 2);
12557 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12558 ins_encode %{
12559 int rshift = $rshift$$constant & 31;
12560 intptr_t mask = $mask$$constant;
12561 int width = exact_log2(mask+1);
12562 __ ubfx(as_Register($dst$$reg),
12563 as_Register($src$$reg), rshift, width);
12564 %}
12565 ins_pipe(ialu_reg_shift);
12566 %}
12567
12568
12569 // This pattern is automatically generated from aarch64_ad.m4
12570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12571
12572 // We can use ubfiz when masking by a positive number and then left shifting the result.
12573 // We know that the mask is positive because immI_bitmask guarantees it.
12574 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12575 %{
12576 match(Set dst (LShiftI (AndI src mask) lshift));
12577 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12578
12579 ins_cost(INSN_COST);
12580 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12581 ins_encode %{
12582 int lshift = $lshift$$constant & 31;
12583 intptr_t mask = $mask$$constant;
12584 int width = exact_log2(mask+1);
12585 __ ubfizw(as_Register($dst$$reg),
12586 as_Register($src$$reg), lshift, width);
12587 %}
12588 ins_pipe(ialu_reg_shift);
12589 %}
12590
12591 // This pattern is automatically generated from aarch64_ad.m4
12592 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12593
12594 // We can use ubfiz when masking by a positive number and then left shifting the result.
12595 // We know that the mask is positive because immL_bitmask guarantees it.
12596 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12597 %{
12598 match(Set dst (LShiftL (AndL src mask) lshift));
12599 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12600
12601 ins_cost(INSN_COST);
12602 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12603 ins_encode %{
12604 int lshift = $lshift$$constant & 63;
12605 intptr_t mask = $mask$$constant;
12606 int width = exact_log2_long(mask+1);
12607 __ ubfiz(as_Register($dst$$reg),
12608 as_Register($src$$reg), lshift, width);
12609 %}
12610 ins_pipe(ialu_reg_shift);
12611 %}
12612
12613 // This pattern is automatically generated from aarch64_ad.m4
12614 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12615
12616 // We can use ubfiz when masking by a positive number and then left shifting the result.
12617 // We know that the mask is positive because immI_bitmask guarantees it.
12618 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12619 %{
12620 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12621 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12622
12623 ins_cost(INSN_COST);
12624 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12625 ins_encode %{
12626 int lshift = $lshift$$constant & 31;
12627 intptr_t mask = $mask$$constant;
12628 int width = exact_log2(mask+1);
12629 __ ubfizw(as_Register($dst$$reg),
12630 as_Register($src$$reg), lshift, width);
12631 %}
12632 ins_pipe(ialu_reg_shift);
12633 %}
12634
12635 // This pattern is automatically generated from aarch64_ad.m4
12636 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12637
12638 // We can use ubfiz when masking by a positive number and then left shifting the result.
12639 // We know that the mask is positive because immL_bitmask guarantees it.
12640 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12641 %{
12642 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12643 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12644
12645 ins_cost(INSN_COST);
12646 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12647 ins_encode %{
12648 int lshift = $lshift$$constant & 63;
12649 intptr_t mask = $mask$$constant;
12650 int width = exact_log2_long(mask+1);
12651 __ ubfiz(as_Register($dst$$reg),
12652 as_Register($src$$reg), lshift, width);
12653 %}
12654 ins_pipe(ialu_reg_shift);
12655 %}
12656
12657
12658 // This pattern is automatically generated from aarch64_ad.m4
12659 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12660
12661 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12662 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12663 %{
12664 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12665 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12666
12667 ins_cost(INSN_COST);
12668 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12669 ins_encode %{
12670 int lshift = $lshift$$constant & 63;
12671 intptr_t mask = $mask$$constant;
12672 int width = exact_log2(mask+1);
12673 __ ubfiz(as_Register($dst$$reg),
12674 as_Register($src$$reg), lshift, width);
12675 %}
12676 ins_pipe(ialu_reg_shift);
12677 %}
12678
12679 // This pattern is automatically generated from aarch64_ad.m4
12680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12681
12682 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12683 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12684 %{
12685 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12686 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12687
12688 ins_cost(INSN_COST);
12689 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12690 ins_encode %{
12691 int lshift = $lshift$$constant & 31;
12692 intptr_t mask = $mask$$constant;
12693 int width = exact_log2(mask+1);
12694 __ ubfiz(as_Register($dst$$reg),
12695 as_Register($src$$reg), lshift, width);
12696 %}
12697 ins_pipe(ialu_reg_shift);
12698 %}
12699
12700 // This pattern is automatically generated from aarch64_ad.m4
12701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12702
12703 // Can skip int2long conversions after AND with small bitmask
12704 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12705 %{
12706 match(Set dst (ConvI2L (AndI src msk)));
12707 ins_cost(INSN_COST);
12708 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12709 ins_encode %{
12710 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12711 %}
12712 ins_pipe(ialu_reg_shift);
12713 %}
12714
12715
12716 // Rotations
12717
12718 // This pattern is automatically generated from aarch64_ad.m4
12719 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12720 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12721 %{
12722 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12723 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12724
12725 ins_cost(INSN_COST);
12726 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12727
12728 ins_encode %{
12729 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12730 $rshift$$constant & 63);
12731 %}
12732 ins_pipe(ialu_reg_reg_extr);
12733 %}
12734
12735
12736 // This pattern is automatically generated from aarch64_ad.m4
12737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12738 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12739 %{
12740 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12741 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12742
12743 ins_cost(INSN_COST);
12744 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12745
12746 ins_encode %{
12747 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12748 $rshift$$constant & 31);
12749 %}
12750 ins_pipe(ialu_reg_reg_extr);
12751 %}
12752
12753
12754 // This pattern is automatically generated from aarch64_ad.m4
12755 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12756 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12757 %{
12758 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12759 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12760
12761 ins_cost(INSN_COST);
12762 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12763
12764 ins_encode %{
12765 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12766 $rshift$$constant & 63);
12767 %}
12768 ins_pipe(ialu_reg_reg_extr);
12769 %}
12770
12771
12772 // This pattern is automatically generated from aarch64_ad.m4
12773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12774 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12775 %{
12776 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12777 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12778
12779 ins_cost(INSN_COST);
12780 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12781
12782 ins_encode %{
12783 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12784 $rshift$$constant & 31);
12785 %}
12786 ins_pipe(ialu_reg_reg_extr);
12787 %}
12788
12789 // This pattern is automatically generated from aarch64_ad.m4
12790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12791 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12792 %{
12793 match(Set dst (RotateRight src shift));
12794
12795 ins_cost(INSN_COST);
12796 format %{ "ror $dst, $src, $shift" %}
12797
12798 ins_encode %{
12799 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12800 $shift$$constant & 0x1f);
12801 %}
12802 ins_pipe(ialu_reg_reg_vshift);
12803 %}
12804
12805 // This pattern is automatically generated from aarch64_ad.m4
12806 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12807 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12808 %{
12809 match(Set dst (RotateRight src shift));
12810
12811 ins_cost(INSN_COST);
12812 format %{ "ror $dst, $src, $shift" %}
12813
12814 ins_encode %{
12815 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12816 $shift$$constant & 0x3f);
12817 %}
12818 ins_pipe(ialu_reg_reg_vshift);
12819 %}
12820
12821 // This pattern is automatically generated from aarch64_ad.m4
12822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12823 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12824 %{
12825 match(Set dst (RotateRight src shift));
12826
12827 ins_cost(INSN_COST);
12828 format %{ "ror $dst, $src, $shift" %}
12829
12830 ins_encode %{
12831 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12832 %}
12833 ins_pipe(ialu_reg_reg_vshift);
12834 %}
12835
12836 // This pattern is automatically generated from aarch64_ad.m4
12837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12838 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12839 %{
12840 match(Set dst (RotateRight src shift));
12841
12842 ins_cost(INSN_COST);
12843 format %{ "ror $dst, $src, $shift" %}
12844
12845 ins_encode %{
12846 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12847 %}
12848 ins_pipe(ialu_reg_reg_vshift);
12849 %}
12850
12851 // This pattern is automatically generated from aarch64_ad.m4
12852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12853 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12854 %{
12855 match(Set dst (RotateLeft src shift));
12856
12857 ins_cost(INSN_COST);
12858 format %{ "rol $dst, $src, $shift" %}
12859
12860 ins_encode %{
12861 __ subw(rscratch1, zr, as_Register($shift$$reg));
12862 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12863 %}
12864 ins_pipe(ialu_reg_reg_vshift);
12865 %}
12866
12867 // This pattern is automatically generated from aarch64_ad.m4
12868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12869 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12870 %{
12871 match(Set dst (RotateLeft src shift));
12872
12873 ins_cost(INSN_COST);
12874 format %{ "rol $dst, $src, $shift" %}
12875
12876 ins_encode %{
12877 __ subw(rscratch1, zr, as_Register($shift$$reg));
12878 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12879 %}
12880 ins_pipe(ialu_reg_reg_vshift);
12881 %}
12882
12883
12884 // Add/subtract (extended)
12885
12886 // This pattern is automatically generated from aarch64_ad.m4
12887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12888 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12889 %{
12890 match(Set dst (AddL src1 (ConvI2L src2)));
12891 ins_cost(INSN_COST);
12892 format %{ "add $dst, $src1, $src2, sxtw" %}
12893
12894 ins_encode %{
12895 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12896 as_Register($src2$$reg), ext::sxtw);
12897 %}
12898 ins_pipe(ialu_reg_reg);
12899 %}
12900
12901 // This pattern is automatically generated from aarch64_ad.m4
12902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12903 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12904 %{
12905 match(Set dst (SubL src1 (ConvI2L src2)));
12906 ins_cost(INSN_COST);
12907 format %{ "sub $dst, $src1, $src2, sxtw" %}
12908
12909 ins_encode %{
12910 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12911 as_Register($src2$$reg), ext::sxtw);
12912 %}
12913 ins_pipe(ialu_reg_reg);
12914 %}
12915
12916 // This pattern is automatically generated from aarch64_ad.m4
12917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12918 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12919 %{
12920 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12921 ins_cost(INSN_COST);
12922 format %{ "add $dst, $src1, $src2, sxth" %}
12923
12924 ins_encode %{
12925 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12926 as_Register($src2$$reg), ext::sxth);
12927 %}
12928 ins_pipe(ialu_reg_reg);
12929 %}
12930
12931 // This pattern is automatically generated from aarch64_ad.m4
12932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12933 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12934 %{
12935 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12936 ins_cost(INSN_COST);
12937 format %{ "add $dst, $src1, $src2, sxtb" %}
12938
12939 ins_encode %{
12940 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12941 as_Register($src2$$reg), ext::sxtb);
12942 %}
12943 ins_pipe(ialu_reg_reg);
12944 %}
12945
12946 // This pattern is automatically generated from aarch64_ad.m4
12947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12948 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12949 %{
12950 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12951 ins_cost(INSN_COST);
12952 format %{ "add $dst, $src1, $src2, uxtb" %}
12953
12954 ins_encode %{
12955 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12956 as_Register($src2$$reg), ext::uxtb);
12957 %}
12958 ins_pipe(ialu_reg_reg);
12959 %}
12960
12961 // This pattern is automatically generated from aarch64_ad.m4
12962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12963 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12964 %{
12965 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12966 ins_cost(INSN_COST);
12967 format %{ "add $dst, $src1, $src2, sxth" %}
12968
12969 ins_encode %{
12970 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12971 as_Register($src2$$reg), ext::sxth);
12972 %}
12973 ins_pipe(ialu_reg_reg);
12974 %}
12975
12976 // This pattern is automatically generated from aarch64_ad.m4
12977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12978 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12979 %{
12980 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12981 ins_cost(INSN_COST);
12982 format %{ "add $dst, $src1, $src2, sxtw" %}
12983
12984 ins_encode %{
12985 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12986 as_Register($src2$$reg), ext::sxtw);
12987 %}
12988 ins_pipe(ialu_reg_reg);
12989 %}
12990
12991 // This pattern is automatically generated from aarch64_ad.m4
12992 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12993 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12994 %{
12995 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12996 ins_cost(INSN_COST);
12997 format %{ "add $dst, $src1, $src2, sxtb" %}
12998
12999 ins_encode %{
13000 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13001 as_Register($src2$$reg), ext::sxtb);
13002 %}
13003 ins_pipe(ialu_reg_reg);
13004 %}
13005
13006 // This pattern is automatically generated from aarch64_ad.m4
13007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13008 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13009 %{
13010 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13011 ins_cost(INSN_COST);
13012 format %{ "add $dst, $src1, $src2, uxtb" %}
13013
13014 ins_encode %{
13015 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13016 as_Register($src2$$reg), ext::uxtb);
13017 %}
13018 ins_pipe(ialu_reg_reg);
13019 %}
13020
13021 // This pattern is automatically generated from aarch64_ad.m4
13022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13023 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13024 %{
13025 match(Set dst (AddI src1 (AndI src2 mask)));
13026 ins_cost(INSN_COST);
13027 format %{ "addw $dst, $src1, $src2, uxtb" %}
13028
13029 ins_encode %{
13030 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13031 as_Register($src2$$reg), ext::uxtb);
13032 %}
13033 ins_pipe(ialu_reg_reg);
13034 %}
13035
13036 // This pattern is automatically generated from aarch64_ad.m4
13037 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13038 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13039 %{
13040 match(Set dst (AddI src1 (AndI src2 mask)));
13041 ins_cost(INSN_COST);
13042 format %{ "addw $dst, $src1, $src2, uxth" %}
13043
13044 ins_encode %{
13045 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13046 as_Register($src2$$reg), ext::uxth);
13047 %}
13048 ins_pipe(ialu_reg_reg);
13049 %}
13050
13051 // This pattern is automatically generated from aarch64_ad.m4
13052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13053 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13054 %{
13055 match(Set dst (AddL src1 (AndL src2 mask)));
13056 ins_cost(INSN_COST);
13057 format %{ "add $dst, $src1, $src2, uxtb" %}
13058
13059 ins_encode %{
13060 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13061 as_Register($src2$$reg), ext::uxtb);
13062 %}
13063 ins_pipe(ialu_reg_reg);
13064 %}
13065
13066 // This pattern is automatically generated from aarch64_ad.m4
13067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13068 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13069 %{
13070 match(Set dst (AddL src1 (AndL src2 mask)));
13071 ins_cost(INSN_COST);
13072 format %{ "add $dst, $src1, $src2, uxth" %}
13073
13074 ins_encode %{
13075 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13076 as_Register($src2$$reg), ext::uxth);
13077 %}
13078 ins_pipe(ialu_reg_reg);
13079 %}
13080
13081 // This pattern is automatically generated from aarch64_ad.m4
13082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13083 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13084 %{
13085 match(Set dst (AddL src1 (AndL src2 mask)));
13086 ins_cost(INSN_COST);
13087 format %{ "add $dst, $src1, $src2, uxtw" %}
13088
13089 ins_encode %{
13090 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13091 as_Register($src2$$reg), ext::uxtw);
13092 %}
13093 ins_pipe(ialu_reg_reg);
13094 %}
13095
13096 // This pattern is automatically generated from aarch64_ad.m4
13097 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13098 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13099 %{
13100 match(Set dst (SubI src1 (AndI src2 mask)));
13101 ins_cost(INSN_COST);
13102 format %{ "subw $dst, $src1, $src2, uxtb" %}
13103
13104 ins_encode %{
13105 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13106 as_Register($src2$$reg), ext::uxtb);
13107 %}
13108 ins_pipe(ialu_reg_reg);
13109 %}
13110
13111 // This pattern is automatically generated from aarch64_ad.m4
13112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13113 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13114 %{
13115 match(Set dst (SubI src1 (AndI src2 mask)));
13116 ins_cost(INSN_COST);
13117 format %{ "subw $dst, $src1, $src2, uxth" %}
13118
13119 ins_encode %{
13120 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13121 as_Register($src2$$reg), ext::uxth);
13122 %}
13123 ins_pipe(ialu_reg_reg);
13124 %}
13125
13126 // This pattern is automatically generated from aarch64_ad.m4
13127 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13128 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13129 %{
13130 match(Set dst (SubL src1 (AndL src2 mask)));
13131 ins_cost(INSN_COST);
13132 format %{ "sub $dst, $src1, $src2, uxtb" %}
13133
13134 ins_encode %{
13135 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13136 as_Register($src2$$reg), ext::uxtb);
13137 %}
13138 ins_pipe(ialu_reg_reg);
13139 %}
13140
13141 // This pattern is automatically generated from aarch64_ad.m4
13142 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13143 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13144 %{
13145 match(Set dst (SubL src1 (AndL src2 mask)));
13146 ins_cost(INSN_COST);
13147 format %{ "sub $dst, $src1, $src2, uxth" %}
13148
13149 ins_encode %{
13150 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13151 as_Register($src2$$reg), ext::uxth);
13152 %}
13153 ins_pipe(ialu_reg_reg);
13154 %}
13155
13156 // This pattern is automatically generated from aarch64_ad.m4
13157 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13158 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13159 %{
13160 match(Set dst (SubL src1 (AndL src2 mask)));
13161 ins_cost(INSN_COST);
13162 format %{ "sub $dst, $src1, $src2, uxtw" %}
13163
13164 ins_encode %{
13165 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13166 as_Register($src2$$reg), ext::uxtw);
13167 %}
13168 ins_pipe(ialu_reg_reg);
13169 %}
13170
13171
13172 // This pattern is automatically generated from aarch64_ad.m4
13173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13174 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13175 %{
13176 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13177 ins_cost(1.9 * INSN_COST);
13178 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13179
13180 ins_encode %{
13181 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13182 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13183 %}
13184 ins_pipe(ialu_reg_reg_shift);
13185 %}
13186
13187 // This pattern is automatically generated from aarch64_ad.m4
13188 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13189 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13190 %{
13191 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13192 ins_cost(1.9 * INSN_COST);
13193 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13194
13195 ins_encode %{
13196 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13197 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13198 %}
13199 ins_pipe(ialu_reg_reg_shift);
13200 %}
13201
13202 // This pattern is automatically generated from aarch64_ad.m4
13203 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13204 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13205 %{
13206 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13207 ins_cost(1.9 * INSN_COST);
13208 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13209
13210 ins_encode %{
13211 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13212 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13213 %}
13214 ins_pipe(ialu_reg_reg_shift);
13215 %}
13216
13217 // This pattern is automatically generated from aarch64_ad.m4
13218 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13219 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13220 %{
13221 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13222 ins_cost(1.9 * INSN_COST);
13223 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13224
13225 ins_encode %{
13226 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13227 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13228 %}
13229 ins_pipe(ialu_reg_reg_shift);
13230 %}
13231
13232 // This pattern is automatically generated from aarch64_ad.m4
13233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13234 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13235 %{
13236 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13237 ins_cost(1.9 * INSN_COST);
13238 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13239
13240 ins_encode %{
13241 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13242 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13243 %}
13244 ins_pipe(ialu_reg_reg_shift);
13245 %}
13246
13247 // This pattern is automatically generated from aarch64_ad.m4
13248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13249 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13250 %{
13251 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13252 ins_cost(1.9 * INSN_COST);
13253 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13254
13255 ins_encode %{
13256 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13257 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13258 %}
13259 ins_pipe(ialu_reg_reg_shift);
13260 %}
13261
13262 // This pattern is automatically generated from aarch64_ad.m4
13263 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13264 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13265 %{
13266 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13267 ins_cost(1.9 * INSN_COST);
13268 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13269
13270 ins_encode %{
13271 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13272 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13273 %}
13274 ins_pipe(ialu_reg_reg_shift);
13275 %}
13276
13277 // This pattern is automatically generated from aarch64_ad.m4
13278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13279 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13280 %{
13281 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13282 ins_cost(1.9 * INSN_COST);
13283 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13284
13285 ins_encode %{
13286 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13287 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13288 %}
13289 ins_pipe(ialu_reg_reg_shift);
13290 %}
13291
13292 // This pattern is automatically generated from aarch64_ad.m4
13293 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13294 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13295 %{
13296 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13297 ins_cost(1.9 * INSN_COST);
13298 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13299
13300 ins_encode %{
13301 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13302 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13303 %}
13304 ins_pipe(ialu_reg_reg_shift);
13305 %}
13306
13307 // This pattern is automatically generated from aarch64_ad.m4
13308 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13309 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13310 %{
13311 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13312 ins_cost(1.9 * INSN_COST);
13313 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13314
13315 ins_encode %{
13316 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13317 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13318 %}
13319 ins_pipe(ialu_reg_reg_shift);
13320 %}
13321
13322 // This pattern is automatically generated from aarch64_ad.m4
13323 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13324 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13325 %{
13326 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13327 ins_cost(1.9 * INSN_COST);
13328 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13329
13330 ins_encode %{
13331 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13332 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13333 %}
13334 ins_pipe(ialu_reg_reg_shift);
13335 %}
13336
13337 // This pattern is automatically generated from aarch64_ad.m4
13338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13339 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13340 %{
13341 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13342 ins_cost(1.9 * INSN_COST);
13343 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13344
13345 ins_encode %{
13346 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13347 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13348 %}
13349 ins_pipe(ialu_reg_reg_shift);
13350 %}
13351
13352 // This pattern is automatically generated from aarch64_ad.m4
13353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13354 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13355 %{
13356 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13357 ins_cost(1.9 * INSN_COST);
13358 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13359
13360 ins_encode %{
13361 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13362 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13363 %}
13364 ins_pipe(ialu_reg_reg_shift);
13365 %}
13366
13367 // This pattern is automatically generated from aarch64_ad.m4
13368 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13369 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13370 %{
13371 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13372 ins_cost(1.9 * INSN_COST);
13373 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13374
13375 ins_encode %{
13376 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13377 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13378 %}
13379 ins_pipe(ialu_reg_reg_shift);
13380 %}
13381
13382 // This pattern is automatically generated from aarch64_ad.m4
13383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13384 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13385 %{
13386 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13387 ins_cost(1.9 * INSN_COST);
13388 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13389
13390 ins_encode %{
13391 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13392 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13393 %}
13394 ins_pipe(ialu_reg_reg_shift);
13395 %}
13396
13397 // This pattern is automatically generated from aarch64_ad.m4
13398 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13399 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13400 %{
13401 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13402 ins_cost(1.9 * INSN_COST);
13403 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13404
13405 ins_encode %{
13406 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13407 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13408 %}
13409 ins_pipe(ialu_reg_reg_shift);
13410 %}
13411
13412 // This pattern is automatically generated from aarch64_ad.m4
13413 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13414 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13415 %{
13416 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13417 ins_cost(1.9 * INSN_COST);
13418 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13419
13420 ins_encode %{
13421 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13422 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13423 %}
13424 ins_pipe(ialu_reg_reg_shift);
13425 %}
13426
13427 // This pattern is automatically generated from aarch64_ad.m4
13428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13429 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13430 %{
13431 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13432 ins_cost(1.9 * INSN_COST);
13433 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13434
13435 ins_encode %{
13436 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13437 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13438 %}
13439 ins_pipe(ialu_reg_reg_shift);
13440 %}
13441
13442 // This pattern is automatically generated from aarch64_ad.m4
13443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13444 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13445 %{
13446 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13447 ins_cost(1.9 * INSN_COST);
13448 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13449
13450 ins_encode %{
13451 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13452 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13453 %}
13454 ins_pipe(ialu_reg_reg_shift);
13455 %}
13456
13457 // This pattern is automatically generated from aarch64_ad.m4
13458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13459 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13460 %{
13461 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13462 ins_cost(1.9 * INSN_COST);
13463 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13464
13465 ins_encode %{
13466 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13467 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13468 %}
13469 ins_pipe(ialu_reg_reg_shift);
13470 %}
13471
13472 // This pattern is automatically generated from aarch64_ad.m4
13473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13474 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13475 %{
13476 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13477 ins_cost(1.9 * INSN_COST);
13478 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13479
13480 ins_encode %{
13481 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13482 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13483 %}
13484 ins_pipe(ialu_reg_reg_shift);
13485 %}
13486
13487 // This pattern is automatically generated from aarch64_ad.m4
13488 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13489 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13490 %{
13491 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13492 ins_cost(1.9 * INSN_COST);
13493 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13494
13495 ins_encode %{
13496 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13497 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13498 %}
13499 ins_pipe(ialu_reg_reg_shift);
13500 %}
13501
13502 // This pattern is automatically generated from aarch64_ad.m4
13503 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13504 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13505 %{
13506 effect(DEF dst, USE src1, USE src2, USE cr);
13507 ins_cost(INSN_COST * 2);
13508 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13509
13510 ins_encode %{
13511 __ cselw($dst$$Register,
13512 $src1$$Register,
13513 $src2$$Register,
13514 Assembler::LT);
13515 %}
13516 ins_pipe(icond_reg_reg);
13517 %}
13518
13519 // This pattern is automatically generated from aarch64_ad.m4
13520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13521 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13522 %{
13523 effect(DEF dst, USE src1, USE src2, USE cr);
13524 ins_cost(INSN_COST * 2);
13525 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13526
13527 ins_encode %{
13528 __ cselw($dst$$Register,
13529 $src1$$Register,
13530 $src2$$Register,
13531 Assembler::GT);
13532 %}
13533 ins_pipe(icond_reg_reg);
13534 %}
13535
13536 // This pattern is automatically generated from aarch64_ad.m4
13537 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13538 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13539 %{
13540 effect(DEF dst, USE src1, USE cr);
13541 ins_cost(INSN_COST * 2);
13542 format %{ "cselw $dst, $src1, zr lt\t" %}
13543
13544 ins_encode %{
13545 __ cselw($dst$$Register,
13546 $src1$$Register,
13547 zr,
13548 Assembler::LT);
13549 %}
13550 ins_pipe(icond_reg);
13551 %}
13552
13553 // This pattern is automatically generated from aarch64_ad.m4
13554 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13555 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13556 %{
13557 effect(DEF dst, USE src1, USE cr);
13558 ins_cost(INSN_COST * 2);
13559 format %{ "cselw $dst, $src1, zr gt\t" %}
13560
13561 ins_encode %{
13562 __ cselw($dst$$Register,
13563 $src1$$Register,
13564 zr,
13565 Assembler::GT);
13566 %}
13567 ins_pipe(icond_reg);
13568 %}
13569
13570 // This pattern is automatically generated from aarch64_ad.m4
13571 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13572 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13573 %{
13574 effect(DEF dst, USE src1, USE cr);
13575 ins_cost(INSN_COST * 2);
13576 format %{ "csincw $dst, $src1, zr le\t" %}
13577
13578 ins_encode %{
13579 __ csincw($dst$$Register,
13580 $src1$$Register,
13581 zr,
13582 Assembler::LE);
13583 %}
13584 ins_pipe(icond_reg);
13585 %}
13586
13587 // This pattern is automatically generated from aarch64_ad.m4
13588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13589 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13590 %{
13591 effect(DEF dst, USE src1, USE cr);
13592 ins_cost(INSN_COST * 2);
13593 format %{ "csincw $dst, $src1, zr gt\t" %}
13594
13595 ins_encode %{
13596 __ csincw($dst$$Register,
13597 $src1$$Register,
13598 zr,
13599 Assembler::GT);
13600 %}
13601 ins_pipe(icond_reg);
13602 %}
13603
13604 // This pattern is automatically generated from aarch64_ad.m4
13605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13606 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13607 %{
13608 effect(DEF dst, USE src1, USE cr);
13609 ins_cost(INSN_COST * 2);
13610 format %{ "csinvw $dst, $src1, zr lt\t" %}
13611
13612 ins_encode %{
13613 __ csinvw($dst$$Register,
13614 $src1$$Register,
13615 zr,
13616 Assembler::LT);
13617 %}
13618 ins_pipe(icond_reg);
13619 %}
13620
13621 // This pattern is automatically generated from aarch64_ad.m4
13622 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13623 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13624 %{
13625 effect(DEF dst, USE src1, USE cr);
13626 ins_cost(INSN_COST * 2);
13627 format %{ "csinvw $dst, $src1, zr ge\t" %}
13628
13629 ins_encode %{
13630 __ csinvw($dst$$Register,
13631 $src1$$Register,
13632 zr,
13633 Assembler::GE);
13634 %}
13635 ins_pipe(icond_reg);
13636 %}
13637
13638 // This pattern is automatically generated from aarch64_ad.m4
13639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13640 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13641 %{
13642 match(Set dst (MinI src imm));
13643 ins_cost(INSN_COST * 3);
13644 expand %{
13645 rFlagsReg cr;
13646 compI_reg_imm0(cr, src);
13647 cmovI_reg_imm0_lt(dst, src, cr);
13648 %}
13649 %}
13650
13651 // This pattern is automatically generated from aarch64_ad.m4
13652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13653 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13654 %{
13655 match(Set dst (MinI imm src));
13656 ins_cost(INSN_COST * 3);
13657 expand %{
13658 rFlagsReg cr;
13659 compI_reg_imm0(cr, src);
13660 cmovI_reg_imm0_lt(dst, src, cr);
13661 %}
13662 %}
13663
13664 // This pattern is automatically generated from aarch64_ad.m4
13665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13666 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13667 %{
13668 match(Set dst (MinI src imm));
13669 ins_cost(INSN_COST * 3);
13670 expand %{
13671 rFlagsReg cr;
13672 compI_reg_imm0(cr, src);
13673 cmovI_reg_imm1_le(dst, src, cr);
13674 %}
13675 %}
13676
13677 // This pattern is automatically generated from aarch64_ad.m4
13678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13679 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13680 %{
13681 match(Set dst (MinI imm src));
13682 ins_cost(INSN_COST * 3);
13683 expand %{
13684 rFlagsReg cr;
13685 compI_reg_imm0(cr, src);
13686 cmovI_reg_imm1_le(dst, src, cr);
13687 %}
13688 %}
13689
13690 // This pattern is automatically generated from aarch64_ad.m4
13691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13692 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13693 %{
13694 match(Set dst (MinI src imm));
13695 ins_cost(INSN_COST * 3);
13696 expand %{
13697 rFlagsReg cr;
13698 compI_reg_imm0(cr, src);
13699 cmovI_reg_immM1_lt(dst, src, cr);
13700 %}
13701 %}
13702
13703 // This pattern is automatically generated from aarch64_ad.m4
13704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13705 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13706 %{
13707 match(Set dst (MinI imm src));
13708 ins_cost(INSN_COST * 3);
13709 expand %{
13710 rFlagsReg cr;
13711 compI_reg_imm0(cr, src);
13712 cmovI_reg_immM1_lt(dst, src, cr);
13713 %}
13714 %}
13715
13716 // This pattern is automatically generated from aarch64_ad.m4
13717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13718 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13719 %{
13720 match(Set dst (MaxI src imm));
13721 ins_cost(INSN_COST * 3);
13722 expand %{
13723 rFlagsReg cr;
13724 compI_reg_imm0(cr, src);
13725 cmovI_reg_imm0_gt(dst, src, cr);
13726 %}
13727 %}
13728
13729 // This pattern is automatically generated from aarch64_ad.m4
13730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13731 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13732 %{
13733 match(Set dst (MaxI imm src));
13734 ins_cost(INSN_COST * 3);
13735 expand %{
13736 rFlagsReg cr;
13737 compI_reg_imm0(cr, src);
13738 cmovI_reg_imm0_gt(dst, src, cr);
13739 %}
13740 %}
13741
13742 // This pattern is automatically generated from aarch64_ad.m4
13743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13744 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13745 %{
13746 match(Set dst (MaxI src imm));
13747 ins_cost(INSN_COST * 3);
13748 expand %{
13749 rFlagsReg cr;
13750 compI_reg_imm0(cr, src);
13751 cmovI_reg_imm1_gt(dst, src, cr);
13752 %}
13753 %}
13754
13755 // This pattern is automatically generated from aarch64_ad.m4
13756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13757 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13758 %{
13759 match(Set dst (MaxI imm src));
13760 ins_cost(INSN_COST * 3);
13761 expand %{
13762 rFlagsReg cr;
13763 compI_reg_imm0(cr, src);
13764 cmovI_reg_imm1_gt(dst, src, cr);
13765 %}
13766 %}
13767
13768 // This pattern is automatically generated from aarch64_ad.m4
13769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13770 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13771 %{
13772 match(Set dst (MaxI src imm));
13773 ins_cost(INSN_COST * 3);
13774 expand %{
13775 rFlagsReg cr;
13776 compI_reg_imm0(cr, src);
13777 cmovI_reg_immM1_ge(dst, src, cr);
13778 %}
13779 %}
13780
13781 // This pattern is automatically generated from aarch64_ad.m4
13782 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13783 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13784 %{
13785 match(Set dst (MaxI imm src));
13786 ins_cost(INSN_COST * 3);
13787 expand %{
13788 rFlagsReg cr;
13789 compI_reg_imm0(cr, src);
13790 cmovI_reg_immM1_ge(dst, src, cr);
13791 %}
13792 %}
13793
13794 // This pattern is automatically generated from aarch64_ad.m4
13795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13796 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13797 %{
13798 match(Set dst (ReverseI src));
13799 ins_cost(INSN_COST);
13800 format %{ "rbitw $dst, $src" %}
13801 ins_encode %{
13802 __ rbitw($dst$$Register, $src$$Register);
13803 %}
13804 ins_pipe(ialu_reg);
13805 %}
13806
13807 // This pattern is automatically generated from aarch64_ad.m4
13808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13809 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13810 %{
13811 match(Set dst (ReverseL src));
13812 ins_cost(INSN_COST);
13813 format %{ "rbit $dst, $src" %}
13814 ins_encode %{
13815 __ rbit($dst$$Register, $src$$Register);
13816 %}
13817 ins_pipe(ialu_reg);
13818 %}
13819
13820
13821 // END This section of the file is automatically generated. Do not edit --------------
13822
13823
13824 // ============================================================================
13825 // Floating Point Arithmetic Instructions
13826
13827 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13828 match(Set dst (AddHF src1 src2));
13829 format %{ "faddh $dst, $src1, $src2" %}
13830 ins_encode %{
13831 __ faddh($dst$$FloatRegister,
13832 $src1$$FloatRegister,
13833 $src2$$FloatRegister);
13834 %}
13835 ins_pipe(fp_dop_reg_reg_s);
13836 %}
13837
13838 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13839 match(Set dst (AddF src1 src2));
13840
13841 ins_cost(INSN_COST * 5);
13842 format %{ "fadds $dst, $src1, $src2" %}
13843
13844 ins_encode %{
13845 __ fadds(as_FloatRegister($dst$$reg),
13846 as_FloatRegister($src1$$reg),
13847 as_FloatRegister($src2$$reg));
13848 %}
13849
13850 ins_pipe(fp_dop_reg_reg_s);
13851 %}
13852
13853 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13854 match(Set dst (AddD src1 src2));
13855
13856 ins_cost(INSN_COST * 5);
13857 format %{ "faddd $dst, $src1, $src2" %}
13858
13859 ins_encode %{
13860 __ faddd(as_FloatRegister($dst$$reg),
13861 as_FloatRegister($src1$$reg),
13862 as_FloatRegister($src2$$reg));
13863 %}
13864
13865 ins_pipe(fp_dop_reg_reg_d);
13866 %}
13867
13868 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13869 match(Set dst (SubHF src1 src2));
13870 format %{ "fsubh $dst, $src1, $src2" %}
13871 ins_encode %{
13872 __ fsubh($dst$$FloatRegister,
13873 $src1$$FloatRegister,
13874 $src2$$FloatRegister);
13875 %}
13876 ins_pipe(fp_dop_reg_reg_s);
13877 %}
13878
13879 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13880 match(Set dst (SubF src1 src2));
13881
13882 ins_cost(INSN_COST * 5);
13883 format %{ "fsubs $dst, $src1, $src2" %}
13884
13885 ins_encode %{
13886 __ fsubs(as_FloatRegister($dst$$reg),
13887 as_FloatRegister($src1$$reg),
13888 as_FloatRegister($src2$$reg));
13889 %}
13890
13891 ins_pipe(fp_dop_reg_reg_s);
13892 %}
13893
13894 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13895 match(Set dst (SubD src1 src2));
13896
13897 ins_cost(INSN_COST * 5);
13898 format %{ "fsubd $dst, $src1, $src2" %}
13899
13900 ins_encode %{
13901 __ fsubd(as_FloatRegister($dst$$reg),
13902 as_FloatRegister($src1$$reg),
13903 as_FloatRegister($src2$$reg));
13904 %}
13905
13906 ins_pipe(fp_dop_reg_reg_d);
13907 %}
13908
13909 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13910 match(Set dst (MulHF src1 src2));
13911 format %{ "fmulh $dst, $src1, $src2" %}
13912 ins_encode %{
13913 __ fmulh($dst$$FloatRegister,
13914 $src1$$FloatRegister,
13915 $src2$$FloatRegister);
13916 %}
13917 ins_pipe(fp_dop_reg_reg_s);
13918 %}
13919
13920 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13921 match(Set dst (MulF src1 src2));
13922
13923 ins_cost(INSN_COST * 6);
13924 format %{ "fmuls $dst, $src1, $src2" %}
13925
13926 ins_encode %{
13927 __ fmuls(as_FloatRegister($dst$$reg),
13928 as_FloatRegister($src1$$reg),
13929 as_FloatRegister($src2$$reg));
13930 %}
13931
13932 ins_pipe(fp_dop_reg_reg_s);
13933 %}
13934
13935 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13936 match(Set dst (MulD src1 src2));
13937
13938 ins_cost(INSN_COST * 6);
13939 format %{ "fmuld $dst, $src1, $src2" %}
13940
13941 ins_encode %{
13942 __ fmuld(as_FloatRegister($dst$$reg),
13943 as_FloatRegister($src1$$reg),
13944 as_FloatRegister($src2$$reg));
13945 %}
13946
13947 ins_pipe(fp_dop_reg_reg_d);
13948 %}
13949
13950 // src1 * src2 + src3 (half-precision float)
13951 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13952 match(Set dst (FmaHF src3 (Binary src1 src2)));
13953 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13954 ins_encode %{
13955 assert(UseFMA, "Needs FMA instructions support.");
13956 __ fmaddh($dst$$FloatRegister,
13957 $src1$$FloatRegister,
13958 $src2$$FloatRegister,
13959 $src3$$FloatRegister);
13960 %}
13961 ins_pipe(pipe_class_default);
13962 %}
13963
13964 // src1 * src2 + src3
13965 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13966 match(Set dst (FmaF src3 (Binary src1 src2)));
13967
13968 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13969
13970 ins_encode %{
13971 assert(UseFMA, "Needs FMA instructions support.");
13972 __ fmadds(as_FloatRegister($dst$$reg),
13973 as_FloatRegister($src1$$reg),
13974 as_FloatRegister($src2$$reg),
13975 as_FloatRegister($src3$$reg));
13976 %}
13977
13978 ins_pipe(pipe_class_default);
13979 %}
13980
13981 // src1 * src2 + src3
13982 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13983 match(Set dst (FmaD src3 (Binary src1 src2)));
13984
13985 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13986
13987 ins_encode %{
13988 assert(UseFMA, "Needs FMA instructions support.");
13989 __ fmaddd(as_FloatRegister($dst$$reg),
13990 as_FloatRegister($src1$$reg),
13991 as_FloatRegister($src2$$reg),
13992 as_FloatRegister($src3$$reg));
13993 %}
13994
13995 ins_pipe(pipe_class_default);
13996 %}
13997
13998 // src1 * (-src2) + src3
13999 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
14000 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14001 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
14002
14003 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
14004
14005 ins_encode %{
14006 assert(UseFMA, "Needs FMA instructions support.");
14007 __ fmsubs(as_FloatRegister($dst$$reg),
14008 as_FloatRegister($src1$$reg),
14009 as_FloatRegister($src2$$reg),
14010 as_FloatRegister($src3$$reg));
14011 %}
14012
14013 ins_pipe(pipe_class_default);
14014 %}
14015
14016 // src1 * (-src2) + src3
14017 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
14018 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14019 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14020
14021 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
14022
14023 ins_encode %{
14024 assert(UseFMA, "Needs FMA instructions support.");
14025 __ fmsubd(as_FloatRegister($dst$$reg),
14026 as_FloatRegister($src1$$reg),
14027 as_FloatRegister($src2$$reg),
14028 as_FloatRegister($src3$$reg));
14029 %}
14030
14031 ins_pipe(pipe_class_default);
14032 %}
14033
14034 // src1 * (-src2) - src3
14035 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14036 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14037 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14038
14039 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14040
14041 ins_encode %{
14042 assert(UseFMA, "Needs FMA instructions support.");
14043 __ fnmadds(as_FloatRegister($dst$$reg),
14044 as_FloatRegister($src1$$reg),
14045 as_FloatRegister($src2$$reg),
14046 as_FloatRegister($src3$$reg));
14047 %}
14048
14049 ins_pipe(pipe_class_default);
14050 %}
14051
14052 // src1 * (-src2) - src3
14053 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14054 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14055 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14056
14057 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14058
14059 ins_encode %{
14060 assert(UseFMA, "Needs FMA instructions support.");
14061 __ fnmaddd(as_FloatRegister($dst$$reg),
14062 as_FloatRegister($src1$$reg),
14063 as_FloatRegister($src2$$reg),
14064 as_FloatRegister($src3$$reg));
14065 %}
14066
14067 ins_pipe(pipe_class_default);
14068 %}
14069
14070 // src1 * src2 - src3
14071 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14072 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14073
14074 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14075
14076 ins_encode %{
14077 assert(UseFMA, "Needs FMA instructions support.");
14078 __ fnmsubs(as_FloatRegister($dst$$reg),
14079 as_FloatRegister($src1$$reg),
14080 as_FloatRegister($src2$$reg),
14081 as_FloatRegister($src3$$reg));
14082 %}
14083
14084 ins_pipe(pipe_class_default);
14085 %}
14086
14087 // src1 * src2 - src3
14088 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14089 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14090
14091 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14092
14093 ins_encode %{
14094 assert(UseFMA, "Needs FMA instructions support.");
14095 // n.b. insn name should be fnmsubd
14096 __ fnmsub(as_FloatRegister($dst$$reg),
14097 as_FloatRegister($src1$$reg),
14098 as_FloatRegister($src2$$reg),
14099 as_FloatRegister($src3$$reg));
14100 %}
14101
14102 ins_pipe(pipe_class_default);
14103 %}
14104
14105 // Math.max(HH)H (half-precision float)
14106 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14107 match(Set dst (MaxHF src1 src2));
14108 format %{ "fmaxh $dst, $src1, $src2" %}
14109 ins_encode %{
14110 __ fmaxh($dst$$FloatRegister,
14111 $src1$$FloatRegister,
14112 $src2$$FloatRegister);
14113 %}
14114 ins_pipe(fp_dop_reg_reg_s);
14115 %}
14116
14117 // Math.min(HH)H (half-precision float)
14118 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14119 match(Set dst (MinHF src1 src2));
14120 format %{ "fminh $dst, $src1, $src2" %}
14121 ins_encode %{
14122 __ fminh($dst$$FloatRegister,
14123 $src1$$FloatRegister,
14124 $src2$$FloatRegister);
14125 %}
14126 ins_pipe(fp_dop_reg_reg_s);
14127 %}
14128
14129 // Math.max(FF)F
14130 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14131 match(Set dst (MaxF src1 src2));
14132
14133 format %{ "fmaxs $dst, $src1, $src2" %}
14134 ins_encode %{
14135 __ fmaxs(as_FloatRegister($dst$$reg),
14136 as_FloatRegister($src1$$reg),
14137 as_FloatRegister($src2$$reg));
14138 %}
14139
14140 ins_pipe(fp_dop_reg_reg_s);
14141 %}
14142
14143 // Math.min(FF)F
14144 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14145 match(Set dst (MinF src1 src2));
14146
14147 format %{ "fmins $dst, $src1, $src2" %}
14148 ins_encode %{
14149 __ fmins(as_FloatRegister($dst$$reg),
14150 as_FloatRegister($src1$$reg),
14151 as_FloatRegister($src2$$reg));
14152 %}
14153
14154 ins_pipe(fp_dop_reg_reg_s);
14155 %}
14156
14157 // Math.max(DD)D
14158 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14159 match(Set dst (MaxD src1 src2));
14160
14161 format %{ "fmaxd $dst, $src1, $src2" %}
14162 ins_encode %{
14163 __ fmaxd(as_FloatRegister($dst$$reg),
14164 as_FloatRegister($src1$$reg),
14165 as_FloatRegister($src2$$reg));
14166 %}
14167
14168 ins_pipe(fp_dop_reg_reg_d);
14169 %}
14170
14171 // Math.min(DD)D
14172 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14173 match(Set dst (MinD src1 src2));
14174
14175 format %{ "fmind $dst, $src1, $src2" %}
14176 ins_encode %{
14177 __ fmind(as_FloatRegister($dst$$reg),
14178 as_FloatRegister($src1$$reg),
14179 as_FloatRegister($src2$$reg));
14180 %}
14181
14182 ins_pipe(fp_dop_reg_reg_d);
14183 %}
14184
14185 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14186 match(Set dst (DivHF src1 src2));
14187 format %{ "fdivh $dst, $src1, $src2" %}
14188 ins_encode %{
14189 __ fdivh($dst$$FloatRegister,
14190 $src1$$FloatRegister,
14191 $src2$$FloatRegister);
14192 %}
14193 ins_pipe(fp_div_s);
14194 %}
14195
14196 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14197 match(Set dst (DivF src1 src2));
14198
14199 ins_cost(INSN_COST * 18);
14200 format %{ "fdivs $dst, $src1, $src2" %}
14201
14202 ins_encode %{
14203 __ fdivs(as_FloatRegister($dst$$reg),
14204 as_FloatRegister($src1$$reg),
14205 as_FloatRegister($src2$$reg));
14206 %}
14207
14208 ins_pipe(fp_div_s);
14209 %}
14210
14211 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14212 match(Set dst (DivD src1 src2));
14213
14214 ins_cost(INSN_COST * 32);
14215 format %{ "fdivd $dst, $src1, $src2" %}
14216
14217 ins_encode %{
14218 __ fdivd(as_FloatRegister($dst$$reg),
14219 as_FloatRegister($src1$$reg),
14220 as_FloatRegister($src2$$reg));
14221 %}
14222
14223 ins_pipe(fp_div_d);
14224 %}
14225
14226 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14227 match(Set dst (NegF src));
14228
14229 ins_cost(INSN_COST * 3);
14230 format %{ "fneg $dst, $src" %}
14231
14232 ins_encode %{
14233 __ fnegs(as_FloatRegister($dst$$reg),
14234 as_FloatRegister($src$$reg));
14235 %}
14236
14237 ins_pipe(fp_uop_s);
14238 %}
14239
14240 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14241 match(Set dst (NegD src));
14242
14243 ins_cost(INSN_COST * 3);
14244 format %{ "fnegd $dst, $src" %}
14245
14246 ins_encode %{
14247 __ fnegd(as_FloatRegister($dst$$reg),
14248 as_FloatRegister($src$$reg));
14249 %}
14250
14251 ins_pipe(fp_uop_d);
14252 %}
14253
14254 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14255 %{
14256 match(Set dst (AbsI src));
14257
14258 effect(KILL cr);
14259 ins_cost(INSN_COST * 2);
14260 format %{ "cmpw $src, zr\n\t"
14261 "cnegw $dst, $src, Assembler::LT\t# int abs"
14262 %}
14263
14264 ins_encode %{
14265 __ cmpw(as_Register($src$$reg), zr);
14266 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14267 %}
14268 ins_pipe(pipe_class_default);
14269 %}
14270
14271 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14272 %{
14273 match(Set dst (AbsL src));
14274
14275 effect(KILL cr);
14276 ins_cost(INSN_COST * 2);
14277 format %{ "cmp $src, zr\n\t"
14278 "cneg $dst, $src, Assembler::LT\t# long abs"
14279 %}
14280
14281 ins_encode %{
14282 __ cmp(as_Register($src$$reg), zr);
14283 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14284 %}
14285 ins_pipe(pipe_class_default);
14286 %}
14287
14288 instruct absF_reg(vRegF dst, vRegF src) %{
14289 match(Set dst (AbsF src));
14290
14291 ins_cost(INSN_COST * 3);
14292 format %{ "fabss $dst, $src" %}
14293 ins_encode %{
14294 __ fabss(as_FloatRegister($dst$$reg),
14295 as_FloatRegister($src$$reg));
14296 %}
14297
14298 ins_pipe(fp_uop_s);
14299 %}
14300
14301 instruct absD_reg(vRegD dst, vRegD src) %{
14302 match(Set dst (AbsD src));
14303
14304 ins_cost(INSN_COST * 3);
14305 format %{ "fabsd $dst, $src" %}
14306 ins_encode %{
14307 __ fabsd(as_FloatRegister($dst$$reg),
14308 as_FloatRegister($src$$reg));
14309 %}
14310
14311 ins_pipe(fp_uop_d);
14312 %}
14313
14314 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14315 match(Set dst (AbsF (SubF src1 src2)));
14316
14317 ins_cost(INSN_COST * 3);
14318 format %{ "fabds $dst, $src1, $src2" %}
14319 ins_encode %{
14320 __ fabds(as_FloatRegister($dst$$reg),
14321 as_FloatRegister($src1$$reg),
14322 as_FloatRegister($src2$$reg));
14323 %}
14324
14325 ins_pipe(fp_uop_s);
14326 %}
14327
14328 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14329 match(Set dst (AbsD (SubD src1 src2)));
14330
14331 ins_cost(INSN_COST * 3);
14332 format %{ "fabdd $dst, $src1, $src2" %}
14333 ins_encode %{
14334 __ fabdd(as_FloatRegister($dst$$reg),
14335 as_FloatRegister($src1$$reg),
14336 as_FloatRegister($src2$$reg));
14337 %}
14338
14339 ins_pipe(fp_uop_d);
14340 %}
14341
14342 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14343 match(Set dst (SqrtD src));
14344
14345 ins_cost(INSN_COST * 50);
14346 format %{ "fsqrtd $dst, $src" %}
14347 ins_encode %{
14348 __ fsqrtd(as_FloatRegister($dst$$reg),
14349 as_FloatRegister($src$$reg));
14350 %}
14351
14352 ins_pipe(fp_div_s);
14353 %}
14354
14355 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14356 match(Set dst (SqrtF src));
14357
14358 ins_cost(INSN_COST * 50);
14359 format %{ "fsqrts $dst, $src" %}
14360 ins_encode %{
14361 __ fsqrts(as_FloatRegister($dst$$reg),
14362 as_FloatRegister($src$$reg));
14363 %}
14364
14365 ins_pipe(fp_div_d);
14366 %}
14367
14368 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14369 match(Set dst (SqrtHF src));
14370 format %{ "fsqrth $dst, $src" %}
14371 ins_encode %{
14372 __ fsqrth($dst$$FloatRegister,
14373 $src$$FloatRegister);
14374 %}
14375 ins_pipe(fp_div_s);
14376 %}
14377
14378 // Math.rint, floor, ceil
14379 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14380 match(Set dst (RoundDoubleMode src rmode));
14381 format %{ "frint $dst, $src, $rmode" %}
14382 ins_encode %{
14383 switch ($rmode$$constant) {
14384 case RoundDoubleModeNode::rmode_rint:
14385 __ frintnd(as_FloatRegister($dst$$reg),
14386 as_FloatRegister($src$$reg));
14387 break;
14388 case RoundDoubleModeNode::rmode_floor:
14389 __ frintmd(as_FloatRegister($dst$$reg),
14390 as_FloatRegister($src$$reg));
14391 break;
14392 case RoundDoubleModeNode::rmode_ceil:
14393 __ frintpd(as_FloatRegister($dst$$reg),
14394 as_FloatRegister($src$$reg));
14395 break;
14396 }
14397 %}
14398 ins_pipe(fp_uop_d);
14399 %}
14400
14401 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14402 match(Set dst (CopySignD src1 (Binary src2 zero)));
14403 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14404 format %{ "CopySignD $dst $src1 $src2" %}
14405 ins_encode %{
14406 FloatRegister dst = as_FloatRegister($dst$$reg),
14407 src1 = as_FloatRegister($src1$$reg),
14408 src2 = as_FloatRegister($src2$$reg),
14409 zero = as_FloatRegister($zero$$reg);
14410 __ fnegd(dst, zero);
14411 __ bsl(dst, __ T8B, src2, src1);
14412 %}
14413 ins_pipe(fp_uop_d);
14414 %}
14415
14416 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14417 match(Set dst (CopySignF src1 src2));
14418 effect(TEMP_DEF dst, USE src1, USE src2);
14419 format %{ "CopySignF $dst $src1 $src2" %}
14420 ins_encode %{
14421 FloatRegister dst = as_FloatRegister($dst$$reg),
14422 src1 = as_FloatRegister($src1$$reg),
14423 src2 = as_FloatRegister($src2$$reg);
14424 __ movi(dst, __ T2S, 0x80, 24);
14425 __ bsl(dst, __ T8B, src2, src1);
14426 %}
14427 ins_pipe(fp_uop_d);
14428 %}
14429
14430 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14431 match(Set dst (SignumD src (Binary zero one)));
14432 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14433 format %{ "signumD $dst, $src" %}
14434 ins_encode %{
14435 FloatRegister src = as_FloatRegister($src$$reg),
14436 dst = as_FloatRegister($dst$$reg),
14437 zero = as_FloatRegister($zero$$reg),
14438 one = as_FloatRegister($one$$reg);
14439 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14440 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14441 // Bit selection instruction gets bit from "one" for each enabled bit in
14442 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14443 // NaN the whole "src" will be copied because "dst" is zero. For all other
14444 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14445 // from "src", and all other bits are copied from 1.0.
14446 __ bsl(dst, __ T8B, one, src);
14447 %}
14448 ins_pipe(fp_uop_d);
14449 %}
14450
14451 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14452 match(Set dst (SignumF src (Binary zero one)));
14453 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14454 format %{ "signumF $dst, $src" %}
14455 ins_encode %{
14456 FloatRegister src = as_FloatRegister($src$$reg),
14457 dst = as_FloatRegister($dst$$reg),
14458 zero = as_FloatRegister($zero$$reg),
14459 one = as_FloatRegister($one$$reg);
14460 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14461 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14462 // Bit selection instruction gets bit from "one" for each enabled bit in
14463 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14464 // NaN the whole "src" will be copied because "dst" is zero. For all other
14465 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14466 // from "src", and all other bits are copied from 1.0.
14467 __ bsl(dst, __ T8B, one, src);
14468 %}
14469 ins_pipe(fp_uop_d);
14470 %}
14471
14472 instruct onspinwait() %{
14473 match(OnSpinWait);
14474 ins_cost(INSN_COST);
14475
14476 format %{ "onspinwait" %}
14477
14478 ins_encode %{
14479 __ spin_wait();
14480 %}
14481 ins_pipe(pipe_class_empty);
14482 %}
14483
14484 // ============================================================================
14485 // Logical Instructions
14486
14487 // Integer Logical Instructions
14488
14489 // And Instructions
14490
14491
14492 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14493 match(Set dst (AndI src1 src2));
14494
14495 format %{ "andw $dst, $src1, $src2\t# int" %}
14496
14497 ins_cost(INSN_COST);
14498 ins_encode %{
14499 __ andw(as_Register($dst$$reg),
14500 as_Register($src1$$reg),
14501 as_Register($src2$$reg));
14502 %}
14503
14504 ins_pipe(ialu_reg_reg);
14505 %}
14506
14507 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14508 match(Set dst (AndI src1 src2));
14509
14510 format %{ "andsw $dst, $src1, $src2\t# int" %}
14511
14512 ins_cost(INSN_COST);
14513 ins_encode %{
14514 __ andw(as_Register($dst$$reg),
14515 as_Register($src1$$reg),
14516 (uint64_t)($src2$$constant));
14517 %}
14518
14519 ins_pipe(ialu_reg_imm);
14520 %}
14521
14522 // Or Instructions
14523
14524 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14525 match(Set dst (OrI src1 src2));
14526
14527 format %{ "orrw $dst, $src1, $src2\t# int" %}
14528
14529 ins_cost(INSN_COST);
14530 ins_encode %{
14531 __ orrw(as_Register($dst$$reg),
14532 as_Register($src1$$reg),
14533 as_Register($src2$$reg));
14534 %}
14535
14536 ins_pipe(ialu_reg_reg);
14537 %}
14538
14539 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14540 match(Set dst (OrI src1 src2));
14541
14542 format %{ "orrw $dst, $src1, $src2\t# int" %}
14543
14544 ins_cost(INSN_COST);
14545 ins_encode %{
14546 __ orrw(as_Register($dst$$reg),
14547 as_Register($src1$$reg),
14548 (uint64_t)($src2$$constant));
14549 %}
14550
14551 ins_pipe(ialu_reg_imm);
14552 %}
14553
14554 // Xor Instructions
14555
14556 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14557 match(Set dst (XorI src1 src2));
14558
14559 format %{ "eorw $dst, $src1, $src2\t# int" %}
14560
14561 ins_cost(INSN_COST);
14562 ins_encode %{
14563 __ eorw(as_Register($dst$$reg),
14564 as_Register($src1$$reg),
14565 as_Register($src2$$reg));
14566 %}
14567
14568 ins_pipe(ialu_reg_reg);
14569 %}
14570
14571 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14572 match(Set dst (XorI src1 src2));
14573
14574 format %{ "eorw $dst, $src1, $src2\t# int" %}
14575
14576 ins_cost(INSN_COST);
14577 ins_encode %{
14578 __ eorw(as_Register($dst$$reg),
14579 as_Register($src1$$reg),
14580 (uint64_t)($src2$$constant));
14581 %}
14582
14583 ins_pipe(ialu_reg_imm);
14584 %}
14585
14586 // Long Logical Instructions
14587 // TODO
14588
14589 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14590 match(Set dst (AndL src1 src2));
14591
14592 format %{ "and $dst, $src1, $src2\t# int" %}
14593
14594 ins_cost(INSN_COST);
14595 ins_encode %{
14596 __ andr(as_Register($dst$$reg),
14597 as_Register($src1$$reg),
14598 as_Register($src2$$reg));
14599 %}
14600
14601 ins_pipe(ialu_reg_reg);
14602 %}
14603
14604 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14605 match(Set dst (AndL src1 src2));
14606
14607 format %{ "and $dst, $src1, $src2\t# int" %}
14608
14609 ins_cost(INSN_COST);
14610 ins_encode %{
14611 __ andr(as_Register($dst$$reg),
14612 as_Register($src1$$reg),
14613 (uint64_t)($src2$$constant));
14614 %}
14615
14616 ins_pipe(ialu_reg_imm);
14617 %}
14618
14619 // Or Instructions
14620
14621 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14622 match(Set dst (OrL src1 src2));
14623
14624 format %{ "orr $dst, $src1, $src2\t# int" %}
14625
14626 ins_cost(INSN_COST);
14627 ins_encode %{
14628 __ orr(as_Register($dst$$reg),
14629 as_Register($src1$$reg),
14630 as_Register($src2$$reg));
14631 %}
14632
14633 ins_pipe(ialu_reg_reg);
14634 %}
14635
14636 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14637 match(Set dst (OrL src1 src2));
14638
14639 format %{ "orr $dst, $src1, $src2\t# int" %}
14640
14641 ins_cost(INSN_COST);
14642 ins_encode %{
14643 __ orr(as_Register($dst$$reg),
14644 as_Register($src1$$reg),
14645 (uint64_t)($src2$$constant));
14646 %}
14647
14648 ins_pipe(ialu_reg_imm);
14649 %}
14650
14651 // Xor Instructions
14652
14653 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14654 match(Set dst (XorL src1 src2));
14655
14656 format %{ "eor $dst, $src1, $src2\t# int" %}
14657
14658 ins_cost(INSN_COST);
14659 ins_encode %{
14660 __ eor(as_Register($dst$$reg),
14661 as_Register($src1$$reg),
14662 as_Register($src2$$reg));
14663 %}
14664
14665 ins_pipe(ialu_reg_reg);
14666 %}
14667
14668 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14669 match(Set dst (XorL src1 src2));
14670
14671 ins_cost(INSN_COST);
14672 format %{ "eor $dst, $src1, $src2\t# int" %}
14673
14674 ins_encode %{
14675 __ eor(as_Register($dst$$reg),
14676 as_Register($src1$$reg),
14677 (uint64_t)($src2$$constant));
14678 %}
14679
14680 ins_pipe(ialu_reg_imm);
14681 %}
14682
14683 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14684 %{
14685 match(Set dst (ConvI2L src));
14686
14687 ins_cost(INSN_COST);
14688 format %{ "sxtw $dst, $src\t# i2l" %}
14689 ins_encode %{
14690 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14691 %}
14692 ins_pipe(ialu_reg_shift);
14693 %}
14694
14695 // this pattern occurs in bigmath arithmetic
14696 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14697 %{
14698 match(Set dst (AndL (ConvI2L src) mask));
14699
14700 ins_cost(INSN_COST);
14701 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14702 ins_encode %{
14703 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14704 %}
14705
14706 ins_pipe(ialu_reg_shift);
14707 %}
14708
14709 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14710 match(Set dst (ConvL2I src));
14711
14712 ins_cost(INSN_COST);
14713 format %{ "movw $dst, $src \t// l2i" %}
14714
14715 ins_encode %{
14716 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14717 %}
14718
14719 ins_pipe(ialu_reg);
14720 %}
14721
14722 instruct convD2F_reg(vRegF dst, vRegD src) %{
14723 match(Set dst (ConvD2F src));
14724
14725 ins_cost(INSN_COST * 5);
14726 format %{ "fcvtd $dst, $src \t// d2f" %}
14727
14728 ins_encode %{
14729 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14730 %}
14731
14732 ins_pipe(fp_d2f);
14733 %}
14734
14735 instruct convF2D_reg(vRegD dst, vRegF src) %{
14736 match(Set dst (ConvF2D src));
14737
14738 ins_cost(INSN_COST * 5);
14739 format %{ "fcvts $dst, $src \t// f2d" %}
14740
14741 ins_encode %{
14742 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14743 %}
14744
14745 ins_pipe(fp_f2d);
14746 %}
14747
14748 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14749 match(Set dst (ConvF2I src));
14750
14751 ins_cost(INSN_COST * 5);
14752 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14753
14754 ins_encode %{
14755 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14756 %}
14757
14758 ins_pipe(fp_f2i);
14759 %}
14760
14761 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14762 match(Set dst (ConvF2L src));
14763
14764 ins_cost(INSN_COST * 5);
14765 format %{ "fcvtzs $dst, $src \t// f2l" %}
14766
14767 ins_encode %{
14768 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14769 %}
14770
14771 ins_pipe(fp_f2l);
14772 %}
14773
14774 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14775 match(Set dst (ConvF2HF src));
14776 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14777 "smov $dst, $tmp\t# move result from $tmp to $dst"
14778 %}
14779 effect(TEMP tmp);
14780 ins_encode %{
14781 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14782 %}
14783 ins_pipe(pipe_slow);
14784 %}
14785
14786 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14787 match(Set dst (ConvHF2F src));
14788 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14789 "fcvt $dst, $tmp\t# convert half to single precision"
14790 %}
14791 effect(TEMP tmp);
14792 ins_encode %{
14793 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14794 %}
14795 ins_pipe(pipe_slow);
14796 %}
14797
14798 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14799 match(Set dst (ConvI2F src));
14800
14801 ins_cost(INSN_COST * 5);
14802 format %{ "scvtfws $dst, $src \t// i2f" %}
14803
14804 ins_encode %{
14805 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14806 %}
14807
14808 ins_pipe(fp_i2f);
14809 %}
14810
14811 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14812 match(Set dst (ConvL2F src));
14813
14814 ins_cost(INSN_COST * 5);
14815 format %{ "scvtfs $dst, $src \t// l2f" %}
14816
14817 ins_encode %{
14818 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14819 %}
14820
14821 ins_pipe(fp_l2f);
14822 %}
14823
14824 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14825 match(Set dst (ConvD2I src));
14826
14827 ins_cost(INSN_COST * 5);
14828 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14829
14830 ins_encode %{
14831 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14832 %}
14833
14834 ins_pipe(fp_d2i);
14835 %}
14836
14837 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14838 match(Set dst (ConvD2L src));
14839
14840 ins_cost(INSN_COST * 5);
14841 format %{ "fcvtzd $dst, $src \t// d2l" %}
14842
14843 ins_encode %{
14844 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14845 %}
14846
14847 ins_pipe(fp_d2l);
14848 %}
14849
14850 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14851 match(Set dst (ConvI2D src));
14852
14853 ins_cost(INSN_COST * 5);
14854 format %{ "scvtfwd $dst, $src \t// i2d" %}
14855
14856 ins_encode %{
14857 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14858 %}
14859
14860 ins_pipe(fp_i2d);
14861 %}
14862
14863 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14864 match(Set dst (ConvL2D src));
14865
14866 ins_cost(INSN_COST * 5);
14867 format %{ "scvtfd $dst, $src \t// l2d" %}
14868
14869 ins_encode %{
14870 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14871 %}
14872
14873 ins_pipe(fp_l2d);
14874 %}
14875
14876 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14877 %{
14878 match(Set dst (RoundD src));
14879 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14880 format %{ "java_round_double $dst,$src"%}
14881 ins_encode %{
14882 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14883 as_FloatRegister($ftmp$$reg));
14884 %}
14885 ins_pipe(pipe_slow);
14886 %}
14887
14888 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14889 %{
14890 match(Set dst (RoundF src));
14891 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14892 format %{ "java_round_float $dst,$src"%}
14893 ins_encode %{
14894 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14895 as_FloatRegister($ftmp$$reg));
14896 %}
14897 ins_pipe(pipe_slow);
14898 %}
14899
14900 // stack <-> reg and reg <-> reg shuffles with no conversion
14901
14902 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14903
14904 match(Set dst (MoveF2I src));
14905
14906 effect(DEF dst, USE src);
14907
14908 ins_cost(4 * INSN_COST);
14909
14910 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14911
14912 ins_encode %{
14913 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14914 %}
14915
14916 ins_pipe(iload_reg_reg);
14917
14918 %}
14919
14920 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14921
14922 match(Set dst (MoveI2F src));
14923
14924 effect(DEF dst, USE src);
14925
14926 ins_cost(4 * INSN_COST);
14927
14928 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14929
14930 ins_encode %{
14931 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14932 %}
14933
14934 ins_pipe(pipe_class_memory);
14935
14936 %}
14937
14938 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14939
14940 match(Set dst (MoveD2L src));
14941
14942 effect(DEF dst, USE src);
14943
14944 ins_cost(4 * INSN_COST);
14945
14946 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14947
14948 ins_encode %{
14949 __ ldr($dst$$Register, Address(sp, $src$$disp));
14950 %}
14951
14952 ins_pipe(iload_reg_reg);
14953
14954 %}
14955
14956 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14957
14958 match(Set dst (MoveL2D src));
14959
14960 effect(DEF dst, USE src);
14961
14962 ins_cost(4 * INSN_COST);
14963
14964 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14965
14966 ins_encode %{
14967 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14968 %}
14969
14970 ins_pipe(pipe_class_memory);
14971
14972 %}
14973
14974 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14975
14976 match(Set dst (MoveF2I src));
14977
14978 effect(DEF dst, USE src);
14979
14980 ins_cost(INSN_COST);
14981
14982 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14983
14984 ins_encode %{
14985 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14986 %}
14987
14988 ins_pipe(pipe_class_memory);
14989
14990 %}
14991
14992 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14993
14994 match(Set dst (MoveI2F src));
14995
14996 effect(DEF dst, USE src);
14997
14998 ins_cost(INSN_COST);
14999
15000 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
15001
15002 ins_encode %{
15003 __ strw($src$$Register, Address(sp, $dst$$disp));
15004 %}
15005
15006 ins_pipe(istore_reg_reg);
15007
15008 %}
15009
15010 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15011
15012 match(Set dst (MoveD2L src));
15013
15014 effect(DEF dst, USE src);
15015
15016 ins_cost(INSN_COST);
15017
15018 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15019
15020 ins_encode %{
15021 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15022 %}
15023
15024 ins_pipe(pipe_class_memory);
15025
15026 %}
15027
15028 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15029
15030 match(Set dst (MoveL2D src));
15031
15032 effect(DEF dst, USE src);
15033
15034 ins_cost(INSN_COST);
15035
15036 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15037
15038 ins_encode %{
15039 __ str($src$$Register, Address(sp, $dst$$disp));
15040 %}
15041
15042 ins_pipe(istore_reg_reg);
15043
15044 %}
15045
15046 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15047
15048 match(Set dst (MoveF2I src));
15049
15050 effect(DEF dst, USE src);
15051
15052 ins_cost(INSN_COST);
15053
15054 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15055
15056 ins_encode %{
15057 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15058 %}
15059
15060 ins_pipe(fp_f2i);
15061
15062 %}
15063
15064 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15065
15066 match(Set dst (MoveI2F src));
15067
15068 effect(DEF dst, USE src);
15069
15070 ins_cost(INSN_COST);
15071
15072 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15073
15074 ins_encode %{
15075 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15076 %}
15077
15078 ins_pipe(fp_i2f);
15079
15080 %}
15081
15082 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15083
15084 match(Set dst (MoveD2L src));
15085
15086 effect(DEF dst, USE src);
15087
15088 ins_cost(INSN_COST);
15089
15090 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15091
15092 ins_encode %{
15093 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15094 %}
15095
15096 ins_pipe(fp_d2l);
15097
15098 %}
15099
15100 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15101
15102 match(Set dst (MoveL2D src));
15103
15104 effect(DEF dst, USE src);
15105
15106 ins_cost(INSN_COST);
15107
15108 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15109
15110 ins_encode %{
15111 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15112 %}
15113
15114 ins_pipe(fp_l2d);
15115
15116 %}
15117
15118 // ============================================================================
15119 // clearing of an array
15120
15121 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
15122 %{
15123 match(Set dummy (ClearArray (Binary cnt base) zero));
15124 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15125
15126 ins_cost(4 * INSN_COST);
15127 format %{ "ClearArray $cnt, $base" %}
15128
15129 ins_encode %{
15130 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15131 if (tpc == nullptr) {
15132 ciEnv::current()->record_failure("CodeCache is full");
15133 return;
15134 }
15135 %}
15136
15137 ins_pipe(pipe_class_memory);
15138 %}
15139
15140 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
15141 %{
15142 predicate(((ClearArrayNode*)n)->word_copy_only());
15143 match(Set dummy (ClearArray (Binary cnt base) val));
15144 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15145
15146 ins_cost(4 * INSN_COST);
15147 format %{ "ClearArray $cnt, $base, $val" %}
15148
15149 ins_encode %{
15150 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
15151 %}
15152
15153 ins_pipe(pipe_class_memory);
15154 %}
15155
15156 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15157 %{
15158 predicate((uint64_t)n->in(2)->get_long()
15159 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
15160 && !((ClearArrayNode*)n)->word_copy_only());
15161 match(Set dummy (ClearArray cnt base));
15162 effect(TEMP temp, USE_KILL base, KILL cr);
15163
15164 ins_cost(4 * INSN_COST);
15165 format %{ "ClearArray $cnt, $base" %}
15166
15167 ins_encode %{
15168 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15169 if (tpc == nullptr) {
15170 ciEnv::current()->record_failure("CodeCache is full");
15171 return;
15172 }
15173 %}
15174
15175 ins_pipe(pipe_class_memory);
15176 %}
15177
15178 // ============================================================================
15179 // Overflow Math Instructions
15180
15181 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15182 %{
15183 match(Set cr (OverflowAddI op1 op2));
15184
15185 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15186 ins_cost(INSN_COST);
15187 ins_encode %{
15188 __ cmnw($op1$$Register, $op2$$Register);
15189 %}
15190
15191 ins_pipe(icmp_reg_reg);
15192 %}
15193
15194 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15195 %{
15196 match(Set cr (OverflowAddI op1 op2));
15197
15198 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15199 ins_cost(INSN_COST);
15200 ins_encode %{
15201 __ cmnw($op1$$Register, $op2$$constant);
15202 %}
15203
15204 ins_pipe(icmp_reg_imm);
15205 %}
15206
15207 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15208 %{
15209 match(Set cr (OverflowAddL op1 op2));
15210
15211 format %{ "cmn $op1, $op2\t# overflow check long" %}
15212 ins_cost(INSN_COST);
15213 ins_encode %{
15214 __ cmn($op1$$Register, $op2$$Register);
15215 %}
15216
15217 ins_pipe(icmp_reg_reg);
15218 %}
15219
15220 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15221 %{
15222 match(Set cr (OverflowAddL op1 op2));
15223
15224 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15225 ins_cost(INSN_COST);
15226 ins_encode %{
15227 __ adds(zr, $op1$$Register, $op2$$constant);
15228 %}
15229
15230 ins_pipe(icmp_reg_imm);
15231 %}
15232
15233 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15234 %{
15235 match(Set cr (OverflowSubI op1 op2));
15236
15237 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15238 ins_cost(INSN_COST);
15239 ins_encode %{
15240 __ cmpw($op1$$Register, $op2$$Register);
15241 %}
15242
15243 ins_pipe(icmp_reg_reg);
15244 %}
15245
15246 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15247 %{
15248 match(Set cr (OverflowSubI op1 op2));
15249
15250 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15251 ins_cost(INSN_COST);
15252 ins_encode %{
15253 __ cmpw($op1$$Register, $op2$$constant);
15254 %}
15255
15256 ins_pipe(icmp_reg_imm);
15257 %}
15258
15259 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15260 %{
15261 match(Set cr (OverflowSubL op1 op2));
15262
15263 format %{ "cmp $op1, $op2\t# overflow check long" %}
15264 ins_cost(INSN_COST);
15265 ins_encode %{
15266 __ cmp($op1$$Register, $op2$$Register);
15267 %}
15268
15269 ins_pipe(icmp_reg_reg);
15270 %}
15271
15272 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15273 %{
15274 match(Set cr (OverflowSubL op1 op2));
15275
15276 format %{ "cmp $op1, $op2\t# overflow check long" %}
15277 ins_cost(INSN_COST);
15278 ins_encode %{
15279 __ subs(zr, $op1$$Register, $op2$$constant);
15280 %}
15281
15282 ins_pipe(icmp_reg_imm);
15283 %}
15284
15285 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15286 %{
15287 match(Set cr (OverflowSubI zero op1));
15288
15289 format %{ "cmpw zr, $op1\t# overflow check int" %}
15290 ins_cost(INSN_COST);
15291 ins_encode %{
15292 __ cmpw(zr, $op1$$Register);
15293 %}
15294
15295 ins_pipe(icmp_reg_imm);
15296 %}
15297
15298 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15299 %{
15300 match(Set cr (OverflowSubL zero op1));
15301
15302 format %{ "cmp zr, $op1\t# overflow check long" %}
15303 ins_cost(INSN_COST);
15304 ins_encode %{
15305 __ cmp(zr, $op1$$Register);
15306 %}
15307
15308 ins_pipe(icmp_reg_imm);
15309 %}
15310
15311 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15312 %{
15313 match(Set cr (OverflowMulI op1 op2));
15314
15315 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15316 "cmp rscratch1, rscratch1, sxtw\n\t"
15317 "movw rscratch1, #0x80000000\n\t"
15318 "cselw rscratch1, rscratch1, zr, NE\n\t"
15319 "cmpw rscratch1, #1" %}
15320 ins_cost(5 * INSN_COST);
15321 ins_encode %{
15322 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15323 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15324 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15325 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15326 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15327 %}
15328
15329 ins_pipe(pipe_slow);
15330 %}
15331
15332 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15333 %{
15334 match(If cmp (OverflowMulI op1 op2));
15335 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15336 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15337 effect(USE labl, KILL cr);
15338
15339 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15340 "cmp rscratch1, rscratch1, sxtw\n\t"
15341 "b$cmp $labl" %}
15342 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15343 ins_encode %{
15344 Label* L = $labl$$label;
15345 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15346 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15347 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15348 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15349 %}
15350
15351 ins_pipe(pipe_serial);
15352 %}
15353
15354 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15355 %{
15356 match(Set cr (OverflowMulL op1 op2));
15357
15358 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15359 "smulh rscratch2, $op1, $op2\n\t"
15360 "cmp rscratch2, rscratch1, ASR #63\n\t"
15361 "movw rscratch1, #0x80000000\n\t"
15362 "cselw rscratch1, rscratch1, zr, NE\n\t"
15363 "cmpw rscratch1, #1" %}
15364 ins_cost(6 * INSN_COST);
15365 ins_encode %{
15366 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15367 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15368 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15369 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15370 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15371 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15372 %}
15373
15374 ins_pipe(pipe_slow);
15375 %}
15376
15377 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15378 %{
15379 match(If cmp (OverflowMulL op1 op2));
15380 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15381 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15382 effect(USE labl, KILL cr);
15383
15384 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15385 "smulh rscratch2, $op1, $op2\n\t"
15386 "cmp rscratch2, rscratch1, ASR #63\n\t"
15387 "b$cmp $labl" %}
15388 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15389 ins_encode %{
15390 Label* L = $labl$$label;
15391 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15392 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15393 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15394 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15395 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15396 %}
15397
15398 ins_pipe(pipe_serial);
15399 %}
15400
15401 // ============================================================================
15402 // Compare Instructions
15403
15404 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15405 %{
15406 match(Set cr (CmpI op1 op2));
15407
15408 effect(DEF cr, USE op1, USE op2);
15409
15410 ins_cost(INSN_COST);
15411 format %{ "cmpw $op1, $op2" %}
15412
15413 ins_encode(aarch64_enc_cmpw(op1, op2));
15414
15415 ins_pipe(icmp_reg_reg);
15416 %}
15417
15418 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15419 %{
15420 match(Set cr (CmpI op1 zero));
15421
15422 effect(DEF cr, USE op1);
15423
15424 ins_cost(INSN_COST);
15425 format %{ "cmpw $op1, 0" %}
15426
15427 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15428
15429 ins_pipe(icmp_reg_imm);
15430 %}
15431
15432 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15433 %{
15434 match(Set cr (CmpI op1 op2));
15435
15436 effect(DEF cr, USE op1);
15437
15438 ins_cost(INSN_COST);
15439 format %{ "cmpw $op1, $op2" %}
15440
15441 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15442
15443 ins_pipe(icmp_reg_imm);
15444 %}
15445
15446 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15447 %{
15448 match(Set cr (CmpI op1 op2));
15449
15450 effect(DEF cr, USE op1);
15451
15452 ins_cost(INSN_COST * 2);
15453 format %{ "cmpw $op1, $op2" %}
15454
15455 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15456
15457 ins_pipe(icmp_reg_imm);
15458 %}
15459
15460 // Unsigned compare Instructions; really, same as signed compare
15461 // except it should only be used to feed an If or a CMovI which takes a
15462 // cmpOpU.
15463
15464 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15465 %{
15466 match(Set cr (CmpU op1 op2));
15467
15468 effect(DEF cr, USE op1, USE op2);
15469
15470 ins_cost(INSN_COST);
15471 format %{ "cmpw $op1, $op2\t# unsigned" %}
15472
15473 ins_encode(aarch64_enc_cmpw(op1, op2));
15474
15475 ins_pipe(icmp_reg_reg);
15476 %}
15477
15478 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15479 %{
15480 match(Set cr (CmpU op1 zero));
15481
15482 effect(DEF cr, USE op1);
15483
15484 ins_cost(INSN_COST);
15485 format %{ "cmpw $op1, #0\t# unsigned" %}
15486
15487 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15488
15489 ins_pipe(icmp_reg_imm);
15490 %}
15491
15492 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15493 %{
15494 match(Set cr (CmpU op1 op2));
15495
15496 effect(DEF cr, USE op1);
15497
15498 ins_cost(INSN_COST);
15499 format %{ "cmpw $op1, $op2\t# unsigned" %}
15500
15501 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15502
15503 ins_pipe(icmp_reg_imm);
15504 %}
15505
15506 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15507 %{
15508 match(Set cr (CmpU op1 op2));
15509
15510 effect(DEF cr, USE op1);
15511
15512 ins_cost(INSN_COST * 2);
15513 format %{ "cmpw $op1, $op2\t# unsigned" %}
15514
15515 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15516
15517 ins_pipe(icmp_reg_imm);
15518 %}
15519
15520 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15521 %{
15522 match(Set cr (CmpL op1 op2));
15523
15524 effect(DEF cr, USE op1, USE op2);
15525
15526 ins_cost(INSN_COST);
15527 format %{ "cmp $op1, $op2" %}
15528
15529 ins_encode(aarch64_enc_cmp(op1, op2));
15530
15531 ins_pipe(icmp_reg_reg);
15532 %}
15533
15534 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15535 %{
15536 match(Set cr (CmpL op1 zero));
15537
15538 effect(DEF cr, USE op1);
15539
15540 ins_cost(INSN_COST);
15541 format %{ "tst $op1" %}
15542
15543 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15544
15545 ins_pipe(icmp_reg_imm);
15546 %}
15547
15548 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15549 %{
15550 match(Set cr (CmpL op1 op2));
15551
15552 effect(DEF cr, USE op1);
15553
15554 ins_cost(INSN_COST);
15555 format %{ "cmp $op1, $op2" %}
15556
15557 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15558
15559 ins_pipe(icmp_reg_imm);
15560 %}
15561
15562 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15563 %{
15564 match(Set cr (CmpL op1 op2));
15565
15566 effect(DEF cr, USE op1);
15567
15568 ins_cost(INSN_COST * 2);
15569 format %{ "cmp $op1, $op2" %}
15570
15571 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15572
15573 ins_pipe(icmp_reg_imm);
15574 %}
15575
15576 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15577 %{
15578 match(Set cr (CmpUL op1 op2));
15579
15580 effect(DEF cr, USE op1, USE op2);
15581
15582 ins_cost(INSN_COST);
15583 format %{ "cmp $op1, $op2" %}
15584
15585 ins_encode(aarch64_enc_cmp(op1, op2));
15586
15587 ins_pipe(icmp_reg_reg);
15588 %}
15589
15590 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15591 %{
15592 match(Set cr (CmpUL op1 zero));
15593
15594 effect(DEF cr, USE op1);
15595
15596 ins_cost(INSN_COST);
15597 format %{ "tst $op1" %}
15598
15599 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15600
15601 ins_pipe(icmp_reg_imm);
15602 %}
15603
15604 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15605 %{
15606 match(Set cr (CmpUL op1 op2));
15607
15608 effect(DEF cr, USE op1);
15609
15610 ins_cost(INSN_COST);
15611 format %{ "cmp $op1, $op2" %}
15612
15613 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15614
15615 ins_pipe(icmp_reg_imm);
15616 %}
15617
15618 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15619 %{
15620 match(Set cr (CmpUL op1 op2));
15621
15622 effect(DEF cr, USE op1);
15623
15624 ins_cost(INSN_COST * 2);
15625 format %{ "cmp $op1, $op2" %}
15626
15627 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15628
15629 ins_pipe(icmp_reg_imm);
15630 %}
15631
15632 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15633 %{
15634 match(Set cr (CmpP op1 op2));
15635
15636 effect(DEF cr, USE op1, USE op2);
15637
15638 ins_cost(INSN_COST);
15639 format %{ "cmp $op1, $op2\t // ptr" %}
15640
15641 ins_encode(aarch64_enc_cmpp(op1, op2));
15642
15643 ins_pipe(icmp_reg_reg);
15644 %}
15645
15646 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15647 %{
15648 match(Set cr (CmpN op1 op2));
15649
15650 effect(DEF cr, USE op1, USE op2);
15651
15652 ins_cost(INSN_COST);
15653 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15654
15655 ins_encode(aarch64_enc_cmpn(op1, op2));
15656
15657 ins_pipe(icmp_reg_reg);
15658 %}
15659
15660 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15661 %{
15662 match(Set cr (CmpP op1 zero));
15663
15664 effect(DEF cr, USE op1, USE zero);
15665
15666 ins_cost(INSN_COST);
15667 format %{ "cmp $op1, 0\t // ptr" %}
15668
15669 ins_encode(aarch64_enc_testp(op1));
15670
15671 ins_pipe(icmp_reg_imm);
15672 %}
15673
15674 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15675 %{
15676 match(Set cr (CmpN op1 zero));
15677
15678 effect(DEF cr, USE op1, USE zero);
15679
15680 ins_cost(INSN_COST);
15681 format %{ "cmp $op1, 0\t // compressed ptr" %}
15682
15683 ins_encode(aarch64_enc_testn(op1));
15684
15685 ins_pipe(icmp_reg_imm);
15686 %}
15687
15688 // FP comparisons
15689 //
15690 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15691 // using normal cmpOp. See declaration of rFlagsReg for details.
15692
15693 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15694 %{
15695 match(Set cr (CmpF src1 src2));
15696
15697 ins_cost(3 * INSN_COST);
15698 format %{ "fcmps $src1, $src2" %}
15699
15700 ins_encode %{
15701 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15702 %}
15703
15704 ins_pipe(pipe_class_compare);
15705 %}
15706
15707 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15708 %{
15709 match(Set cr (CmpF src1 src2));
15710
15711 ins_cost(3 * INSN_COST);
15712 format %{ "fcmps $src1, 0.0" %}
15713
15714 ins_encode %{
15715 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15716 %}
15717
15718 ins_pipe(pipe_class_compare);
15719 %}
15720 // FROM HERE
15721
15722 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15723 %{
15724 match(Set cr (CmpD src1 src2));
15725
15726 ins_cost(3 * INSN_COST);
15727 format %{ "fcmpd $src1, $src2" %}
15728
15729 ins_encode %{
15730 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15731 %}
15732
15733 ins_pipe(pipe_class_compare);
15734 %}
15735
15736 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15737 %{
15738 match(Set cr (CmpD src1 src2));
15739
15740 ins_cost(3 * INSN_COST);
15741 format %{ "fcmpd $src1, 0.0" %}
15742
15743 ins_encode %{
15744 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15745 %}
15746
15747 ins_pipe(pipe_class_compare);
15748 %}
15749
15750 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15751 %{
15752 match(Set dst (CmpF3 src1 src2));
15753 effect(KILL cr);
15754
15755 ins_cost(5 * INSN_COST);
15756 format %{ "fcmps $src1, $src2\n\t"
15757 "csinvw($dst, zr, zr, eq\n\t"
15758 "csnegw($dst, $dst, $dst, lt)"
15759 %}
15760
15761 ins_encode %{
15762 Label done;
15763 FloatRegister s1 = as_FloatRegister($src1$$reg);
15764 FloatRegister s2 = as_FloatRegister($src2$$reg);
15765 Register d = as_Register($dst$$reg);
15766 __ fcmps(s1, s2);
15767 // installs 0 if EQ else -1
15768 __ csinvw(d, zr, zr, Assembler::EQ);
15769 // keeps -1 if less or unordered else installs 1
15770 __ csnegw(d, d, d, Assembler::LT);
15771 __ bind(done);
15772 %}
15773
15774 ins_pipe(pipe_class_default);
15775
15776 %}
15777
15778 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15779 %{
15780 match(Set dst (CmpD3 src1 src2));
15781 effect(KILL cr);
15782
15783 ins_cost(5 * INSN_COST);
15784 format %{ "fcmpd $src1, $src2\n\t"
15785 "csinvw($dst, zr, zr, eq\n\t"
15786 "csnegw($dst, $dst, $dst, lt)"
15787 %}
15788
15789 ins_encode %{
15790 Label done;
15791 FloatRegister s1 = as_FloatRegister($src1$$reg);
15792 FloatRegister s2 = as_FloatRegister($src2$$reg);
15793 Register d = as_Register($dst$$reg);
15794 __ fcmpd(s1, s2);
15795 // installs 0 if EQ else -1
15796 __ csinvw(d, zr, zr, Assembler::EQ);
15797 // keeps -1 if less or unordered else installs 1
15798 __ csnegw(d, d, d, Assembler::LT);
15799 __ bind(done);
15800 %}
15801 ins_pipe(pipe_class_default);
15802
15803 %}
15804
15805 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15806 %{
15807 match(Set dst (CmpF3 src1 zero));
15808 effect(KILL cr);
15809
15810 ins_cost(5 * INSN_COST);
15811 format %{ "fcmps $src1, 0.0\n\t"
15812 "csinvw($dst, zr, zr, eq\n\t"
15813 "csnegw($dst, $dst, $dst, lt)"
15814 %}
15815
15816 ins_encode %{
15817 Label done;
15818 FloatRegister s1 = as_FloatRegister($src1$$reg);
15819 Register d = as_Register($dst$$reg);
15820 __ fcmps(s1, 0.0);
15821 // installs 0 if EQ else -1
15822 __ csinvw(d, zr, zr, Assembler::EQ);
15823 // keeps -1 if less or unordered else installs 1
15824 __ csnegw(d, d, d, Assembler::LT);
15825 __ bind(done);
15826 %}
15827
15828 ins_pipe(pipe_class_default);
15829
15830 %}
15831
15832 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15833 %{
15834 match(Set dst (CmpD3 src1 zero));
15835 effect(KILL cr);
15836
15837 ins_cost(5 * INSN_COST);
15838 format %{ "fcmpd $src1, 0.0\n\t"
15839 "csinvw($dst, zr, zr, eq\n\t"
15840 "csnegw($dst, $dst, $dst, lt)"
15841 %}
15842
15843 ins_encode %{
15844 Label done;
15845 FloatRegister s1 = as_FloatRegister($src1$$reg);
15846 Register d = as_Register($dst$$reg);
15847 __ fcmpd(s1, 0.0);
15848 // installs 0 if EQ else -1
15849 __ csinvw(d, zr, zr, Assembler::EQ);
15850 // keeps -1 if less or unordered else installs 1
15851 __ csnegw(d, d, d, Assembler::LT);
15852 __ bind(done);
15853 %}
15854 ins_pipe(pipe_class_default);
15855
15856 %}
15857
15858 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15859 %{
15860 match(Set dst (CmpLTMask p q));
15861 effect(KILL cr);
15862
15863 ins_cost(3 * INSN_COST);
15864
15865 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15866 "csetw $dst, lt\n\t"
15867 "subw $dst, zr, $dst"
15868 %}
15869
15870 ins_encode %{
15871 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15872 __ csetw(as_Register($dst$$reg), Assembler::LT);
15873 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15874 %}
15875
15876 ins_pipe(ialu_reg_reg);
15877 %}
15878
15879 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15880 %{
15881 match(Set dst (CmpLTMask src zero));
15882 effect(KILL cr);
15883
15884 ins_cost(INSN_COST);
15885
15886 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15887
15888 ins_encode %{
15889 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15890 %}
15891
15892 ins_pipe(ialu_reg_shift);
15893 %}
15894
15895 // ============================================================================
15896 // Max and Min
15897
15898 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15899
15900 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15901 %{
15902 effect(DEF cr, USE src);
15903 ins_cost(INSN_COST);
15904 format %{ "cmpw $src, 0" %}
15905
15906 ins_encode %{
15907 __ cmpw($src$$Register, 0);
15908 %}
15909 ins_pipe(icmp_reg_imm);
15910 %}
15911
15912 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15913 %{
15914 match(Set dst (MinI src1 src2));
15915 ins_cost(INSN_COST * 3);
15916
15917 expand %{
15918 rFlagsReg cr;
15919 compI_reg_reg(cr, src1, src2);
15920 cmovI_reg_reg_lt(dst, src1, src2, cr);
15921 %}
15922 %}
15923
15924 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15925 %{
15926 match(Set dst (MaxI src1 src2));
15927 ins_cost(INSN_COST * 3);
15928
15929 expand %{
15930 rFlagsReg cr;
15931 compI_reg_reg(cr, src1, src2);
15932 cmovI_reg_reg_gt(dst, src1, src2, cr);
15933 %}
15934 %}
15935
15936
15937 // ============================================================================
15938 // Branch Instructions
15939
15940 // Direct Branch.
15941 instruct branch(label lbl)
15942 %{
15943 match(Goto);
15944
15945 effect(USE lbl);
15946
15947 ins_cost(BRANCH_COST);
15948 format %{ "b $lbl" %}
15949
15950 ins_encode(aarch64_enc_b(lbl));
15951
15952 ins_pipe(pipe_branch);
15953 %}
15954
15955 // Conditional Near Branch
15956 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15957 %{
15958 // Same match rule as `branchConFar'.
15959 match(If cmp cr);
15960
15961 effect(USE lbl);
15962
15963 ins_cost(BRANCH_COST);
15964 // If set to 1 this indicates that the current instruction is a
15965 // short variant of a long branch. This avoids using this
15966 // instruction in first-pass matching. It will then only be used in
15967 // the `Shorten_branches' pass.
15968 // ins_short_branch(1);
15969 format %{ "b$cmp $lbl" %}
15970
15971 ins_encode(aarch64_enc_br_con(cmp, lbl));
15972
15973 ins_pipe(pipe_branch_cond);
15974 %}
15975
15976 // Conditional Near Branch Unsigned
15977 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15978 %{
15979 // Same match rule as `branchConFar'.
15980 match(If cmp cr);
15981
15982 effect(USE lbl);
15983
15984 ins_cost(BRANCH_COST);
15985 // If set to 1 this indicates that the current instruction is a
15986 // short variant of a long branch. This avoids using this
15987 // instruction in first-pass matching. It will then only be used in
15988 // the `Shorten_branches' pass.
15989 // ins_short_branch(1);
15990 format %{ "b$cmp $lbl\t# unsigned" %}
15991
15992 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15993
15994 ins_pipe(pipe_branch_cond);
15995 %}
15996
15997 // Make use of CBZ and CBNZ. These instructions, as well as being
15998 // shorter than (cmp; branch), have the additional benefit of not
15999 // killing the flags.
16000
16001 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
16002 match(If cmp (CmpI op1 op2));
16003 effect(USE labl);
16004
16005 ins_cost(BRANCH_COST);
16006 format %{ "cbw$cmp $op1, $labl" %}
16007 ins_encode %{
16008 Label* L = $labl$$label;
16009 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16010 if (cond == Assembler::EQ)
16011 __ cbzw($op1$$Register, *L);
16012 else
16013 __ cbnzw($op1$$Register, *L);
16014 %}
16015 ins_pipe(pipe_cmp_branch);
16016 %}
16017
16018 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
16019 match(If cmp (CmpL op1 op2));
16020 effect(USE labl);
16021
16022 ins_cost(BRANCH_COST);
16023 format %{ "cb$cmp $op1, $labl" %}
16024 ins_encode %{
16025 Label* L = $labl$$label;
16026 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16027 if (cond == Assembler::EQ)
16028 __ cbz($op1$$Register, *L);
16029 else
16030 __ cbnz($op1$$Register, *L);
16031 %}
16032 ins_pipe(pipe_cmp_branch);
16033 %}
16034
16035 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16036 match(If cmp (CmpP op1 op2));
16037 effect(USE labl);
16038
16039 ins_cost(BRANCH_COST);
16040 format %{ "cb$cmp $op1, $labl" %}
16041 ins_encode %{
16042 Label* L = $labl$$label;
16043 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16044 if (cond == Assembler::EQ)
16045 __ cbz($op1$$Register, *L);
16046 else
16047 __ cbnz($op1$$Register, *L);
16048 %}
16049 ins_pipe(pipe_cmp_branch);
16050 %}
16051
16052 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16053 match(If cmp (CmpN op1 op2));
16054 effect(USE labl);
16055
16056 ins_cost(BRANCH_COST);
16057 format %{ "cbw$cmp $op1, $labl" %}
16058 ins_encode %{
16059 Label* L = $labl$$label;
16060 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16061 if (cond == Assembler::EQ)
16062 __ cbzw($op1$$Register, *L);
16063 else
16064 __ cbnzw($op1$$Register, *L);
16065 %}
16066 ins_pipe(pipe_cmp_branch);
16067 %}
16068
16069 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16070 match(If cmp (CmpP (DecodeN oop) zero));
16071 effect(USE labl);
16072
16073 ins_cost(BRANCH_COST);
16074 format %{ "cb$cmp $oop, $labl" %}
16075 ins_encode %{
16076 Label* L = $labl$$label;
16077 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16078 if (cond == Assembler::EQ)
16079 __ cbzw($oop$$Register, *L);
16080 else
16081 __ cbnzw($oop$$Register, *L);
16082 %}
16083 ins_pipe(pipe_cmp_branch);
16084 %}
16085
16086 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16087 match(If cmp (CmpU op1 op2));
16088 effect(USE labl);
16089
16090 ins_cost(BRANCH_COST);
16091 format %{ "cbw$cmp $op1, $labl" %}
16092 ins_encode %{
16093 Label* L = $labl$$label;
16094 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16095 if (cond == Assembler::EQ || cond == Assembler::LS) {
16096 __ cbzw($op1$$Register, *L);
16097 } else {
16098 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16099 __ cbnzw($op1$$Register, *L);
16100 }
16101 %}
16102 ins_pipe(pipe_cmp_branch);
16103 %}
16104
16105 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16106 match(If cmp (CmpUL op1 op2));
16107 effect(USE labl);
16108
16109 ins_cost(BRANCH_COST);
16110 format %{ "cb$cmp $op1, $labl" %}
16111 ins_encode %{
16112 Label* L = $labl$$label;
16113 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16114 if (cond == Assembler::EQ || cond == Assembler::LS) {
16115 __ cbz($op1$$Register, *L);
16116 } else {
16117 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16118 __ cbnz($op1$$Register, *L);
16119 }
16120 %}
16121 ins_pipe(pipe_cmp_branch);
16122 %}
16123
16124 // Test bit and Branch
16125
16126 // Patterns for short (< 32KiB) variants
16127 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16128 match(If cmp (CmpL op1 op2));
16129 effect(USE labl);
16130
16131 ins_cost(BRANCH_COST);
16132 format %{ "cb$cmp $op1, $labl # long" %}
16133 ins_encode %{
16134 Label* L = $labl$$label;
16135 Assembler::Condition cond =
16136 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16137 __ tbr(cond, $op1$$Register, 63, *L);
16138 %}
16139 ins_pipe(pipe_cmp_branch);
16140 ins_short_branch(1);
16141 %}
16142
16143 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16144 match(If cmp (CmpI op1 op2));
16145 effect(USE labl);
16146
16147 ins_cost(BRANCH_COST);
16148 format %{ "cb$cmp $op1, $labl # int" %}
16149 ins_encode %{
16150 Label* L = $labl$$label;
16151 Assembler::Condition cond =
16152 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16153 __ tbr(cond, $op1$$Register, 31, *L);
16154 %}
16155 ins_pipe(pipe_cmp_branch);
16156 ins_short_branch(1);
16157 %}
16158
16159 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16160 match(If cmp (CmpL (AndL op1 op2) op3));
16161 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16162 effect(USE labl);
16163
16164 ins_cost(BRANCH_COST);
16165 format %{ "tb$cmp $op1, $op2, $labl" %}
16166 ins_encode %{
16167 Label* L = $labl$$label;
16168 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16169 int bit = exact_log2_long($op2$$constant);
16170 __ tbr(cond, $op1$$Register, bit, *L);
16171 %}
16172 ins_pipe(pipe_cmp_branch);
16173 ins_short_branch(1);
16174 %}
16175
16176 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16177 match(If cmp (CmpI (AndI op1 op2) op3));
16178 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16179 effect(USE labl);
16180
16181 ins_cost(BRANCH_COST);
16182 format %{ "tb$cmp $op1, $op2, $labl" %}
16183 ins_encode %{
16184 Label* L = $labl$$label;
16185 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16186 int bit = exact_log2((juint)$op2$$constant);
16187 __ tbr(cond, $op1$$Register, bit, *L);
16188 %}
16189 ins_pipe(pipe_cmp_branch);
16190 ins_short_branch(1);
16191 %}
16192
16193 // And far variants
16194 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16195 match(If cmp (CmpL op1 op2));
16196 effect(USE labl);
16197
16198 ins_cost(BRANCH_COST);
16199 format %{ "cb$cmp $op1, $labl # long" %}
16200 ins_encode %{
16201 Label* L = $labl$$label;
16202 Assembler::Condition cond =
16203 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16204 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16205 %}
16206 ins_pipe(pipe_cmp_branch);
16207 %}
16208
16209 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16210 match(If cmp (CmpI op1 op2));
16211 effect(USE labl);
16212
16213 ins_cost(BRANCH_COST);
16214 format %{ "cb$cmp $op1, $labl # int" %}
16215 ins_encode %{
16216 Label* L = $labl$$label;
16217 Assembler::Condition cond =
16218 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16219 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16220 %}
16221 ins_pipe(pipe_cmp_branch);
16222 %}
16223
16224 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16225 match(If cmp (CmpL (AndL op1 op2) op3));
16226 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16227 effect(USE labl);
16228
16229 ins_cost(BRANCH_COST);
16230 format %{ "tb$cmp $op1, $op2, $labl" %}
16231 ins_encode %{
16232 Label* L = $labl$$label;
16233 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16234 int bit = exact_log2_long($op2$$constant);
16235 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16236 %}
16237 ins_pipe(pipe_cmp_branch);
16238 %}
16239
16240 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16241 match(If cmp (CmpI (AndI op1 op2) op3));
16242 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16243 effect(USE labl);
16244
16245 ins_cost(BRANCH_COST);
16246 format %{ "tb$cmp $op1, $op2, $labl" %}
16247 ins_encode %{
16248 Label* L = $labl$$label;
16249 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16250 int bit = exact_log2((juint)$op2$$constant);
16251 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16252 %}
16253 ins_pipe(pipe_cmp_branch);
16254 %}
16255
16256 // Test bits
16257
16258 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16259 match(Set cr (CmpL (AndL op1 op2) op3));
16260 predicate(Assembler::operand_valid_for_logical_immediate
16261 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16262
16263 ins_cost(INSN_COST);
16264 format %{ "tst $op1, $op2 # long" %}
16265 ins_encode %{
16266 __ tst($op1$$Register, $op2$$constant);
16267 %}
16268 ins_pipe(ialu_reg_reg);
16269 %}
16270
16271 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16272 match(Set cr (CmpI (AndI op1 op2) op3));
16273 predicate(Assembler::operand_valid_for_logical_immediate
16274 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16275
16276 ins_cost(INSN_COST);
16277 format %{ "tst $op1, $op2 # int" %}
16278 ins_encode %{
16279 __ tstw($op1$$Register, $op2$$constant);
16280 %}
16281 ins_pipe(ialu_reg_reg);
16282 %}
16283
16284 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16285 match(Set cr (CmpL (AndL op1 op2) op3));
16286
16287 ins_cost(INSN_COST);
16288 format %{ "tst $op1, $op2 # long" %}
16289 ins_encode %{
16290 __ tst($op1$$Register, $op2$$Register);
16291 %}
16292 ins_pipe(ialu_reg_reg);
16293 %}
16294
16295 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16296 match(Set cr (CmpI (AndI op1 op2) op3));
16297
16298 ins_cost(INSN_COST);
16299 format %{ "tstw $op1, $op2 # int" %}
16300 ins_encode %{
16301 __ tstw($op1$$Register, $op2$$Register);
16302 %}
16303 ins_pipe(ialu_reg_reg);
16304 %}
16305
16306
16307 // Conditional Far Branch
16308 // Conditional Far Branch Unsigned
16309 // TODO: fixme
16310
16311 // counted loop end branch near
16312 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16313 %{
16314 match(CountedLoopEnd cmp cr);
16315
16316 effect(USE lbl);
16317
16318 ins_cost(BRANCH_COST);
16319 // short variant.
16320 // ins_short_branch(1);
16321 format %{ "b$cmp $lbl \t// counted loop end" %}
16322
16323 ins_encode(aarch64_enc_br_con(cmp, lbl));
16324
16325 ins_pipe(pipe_branch);
16326 %}
16327
16328 // counted loop end branch far
16329 // TODO: fixme
16330
16331 // ============================================================================
16332 // inlined locking and unlocking
16333
16334 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16335 %{
16336 match(Set cr (FastLock object box));
16337 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16338
16339 ins_cost(5 * INSN_COST);
16340 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16341
16342 ins_encode %{
16343 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16344 %}
16345
16346 ins_pipe(pipe_serial);
16347 %}
16348
16349 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16350 %{
16351 match(Set cr (FastUnlock object box));
16352 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16353
16354 ins_cost(5 * INSN_COST);
16355 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16356
16357 ins_encode %{
16358 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16359 %}
16360
16361 ins_pipe(pipe_serial);
16362 %}
16363
16364 // ============================================================================
16365 // Safepoint Instructions
16366
16367 // TODO
16368 // provide a near and far version of this code
16369
16370 instruct safePoint(rFlagsReg cr, iRegP poll)
16371 %{
16372 match(SafePoint poll);
16373 effect(KILL cr);
16374
16375 format %{
16376 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16377 %}
16378 ins_encode %{
16379 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16380 %}
16381 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16382 %}
16383
16384
16385 // ============================================================================
16386 // Procedure Call/Return Instructions
16387
16388 // Call Java Static Instruction
16389
16390 instruct CallStaticJavaDirect(method meth)
16391 %{
16392 match(CallStaticJava);
16393
16394 effect(USE meth);
16395
16396 ins_cost(CALL_COST);
16397
16398 format %{ "call,static $meth \t// ==> " %}
16399
16400 ins_encode(aarch64_enc_java_static_call(meth),
16401 aarch64_enc_call_epilog);
16402
16403 ins_pipe(pipe_class_call);
16404 %}
16405
16406 // TO HERE
16407
16408 // Call Java Dynamic Instruction
16409 instruct CallDynamicJavaDirect(method meth)
16410 %{
16411 match(CallDynamicJava);
16412
16413 effect(USE meth);
16414
16415 ins_cost(CALL_COST);
16416
16417 format %{ "CALL,dynamic $meth \t// ==> " %}
16418
16419 ins_encode(aarch64_enc_java_dynamic_call(meth),
16420 aarch64_enc_call_epilog);
16421
16422 ins_pipe(pipe_class_call);
16423 %}
16424
16425 // Call Runtime Instruction
16426
16427 instruct CallRuntimeDirect(method meth)
16428 %{
16429 match(CallRuntime);
16430
16431 effect(USE meth);
16432
16433 ins_cost(CALL_COST);
16434
16435 format %{ "CALL, runtime $meth" %}
16436
16437 ins_encode( aarch64_enc_java_to_runtime(meth) );
16438
16439 ins_pipe(pipe_class_call);
16440 %}
16441
16442 // Call Runtime Instruction
16443
16444 instruct CallLeafDirect(method meth)
16445 %{
16446 match(CallLeaf);
16447
16448 effect(USE meth);
16449
16450 ins_cost(CALL_COST);
16451
16452 format %{ "CALL, runtime leaf $meth" %}
16453
16454 ins_encode( aarch64_enc_java_to_runtime(meth) );
16455
16456 ins_pipe(pipe_class_call);
16457 %}
16458
16459 // Call Runtime Instruction without safepoint and with vector arguments
16460 instruct CallLeafDirectVector(method meth)
16461 %{
16462 match(CallLeafVector);
16463
16464 effect(USE meth);
16465
16466 ins_cost(CALL_COST);
16467
16468 format %{ "CALL, runtime leaf vector $meth" %}
16469
16470 ins_encode(aarch64_enc_java_to_runtime(meth));
16471
16472 ins_pipe(pipe_class_call);
16473 %}
16474
16475 // Call Runtime Instruction
16476
16477 // entry point is null, target holds the address to call
16478 instruct CallLeafNoFPIndirect(iRegP target)
16479 %{
16480 predicate(n->as_Call()->entry_point() == nullptr);
16481
16482 match(CallLeafNoFP target);
16483
16484 ins_cost(CALL_COST);
16485
16486 format %{ "CALL, runtime leaf nofp indirect $target" %}
16487
16488 ins_encode %{
16489 __ blr($target$$Register);
16490 %}
16491
16492 ins_pipe(pipe_class_call);
16493 %}
16494
16495 instruct CallLeafNoFPDirect(method meth)
16496 %{
16497 predicate(n->as_Call()->entry_point() != nullptr);
16498
16499 match(CallLeafNoFP);
16500
16501 effect(USE meth);
16502
16503 ins_cost(CALL_COST);
16504
16505 format %{ "CALL, runtime leaf nofp $meth" %}
16506
16507 ins_encode( aarch64_enc_java_to_runtime(meth) );
16508
16509 ins_pipe(pipe_class_call);
16510 %}
16511
16512 // Tail Call; Jump from runtime stub to Java code.
16513 // Also known as an 'interprocedural jump'.
16514 // Target of jump will eventually return to caller.
16515 // TailJump below removes the return address.
16516 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16517 // emitted just above the TailCall which has reset rfp to the caller state.
16518 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16519 %{
16520 match(TailCall jump_target method_ptr);
16521
16522 ins_cost(CALL_COST);
16523
16524 format %{ "br $jump_target\t# $method_ptr holds method" %}
16525
16526 ins_encode(aarch64_enc_tail_call(jump_target));
16527
16528 ins_pipe(pipe_class_call);
16529 %}
16530
16531 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16532 %{
16533 match(TailJump jump_target ex_oop);
16534
16535 ins_cost(CALL_COST);
16536
16537 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16538
16539 ins_encode(aarch64_enc_tail_jmp(jump_target));
16540
16541 ins_pipe(pipe_class_call);
16542 %}
16543
16544 // Forward exception.
16545 instruct ForwardExceptionjmp()
16546 %{
16547 match(ForwardException);
16548 ins_cost(CALL_COST);
16549
16550 format %{ "b forward_exception_stub" %}
16551 ins_encode %{
16552 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16553 %}
16554 ins_pipe(pipe_class_call);
16555 %}
16556
16557 // Create exception oop: created by stack-crawling runtime code.
16558 // Created exception is now available to this handler, and is setup
16559 // just prior to jumping to this handler. No code emitted.
16560 // TODO check
16561 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16562 instruct CreateException(iRegP_R0 ex_oop)
16563 %{
16564 match(Set ex_oop (CreateEx));
16565
16566 format %{ " -- \t// exception oop; no code emitted" %}
16567
16568 size(0);
16569
16570 ins_encode( /*empty*/ );
16571
16572 ins_pipe(pipe_class_empty);
16573 %}
16574
16575 // Rethrow exception: The exception oop will come in the first
16576 // argument position. Then JUMP (not call) to the rethrow stub code.
16577 instruct RethrowException() %{
16578 match(Rethrow);
16579 ins_cost(CALL_COST);
16580
16581 format %{ "b rethrow_stub" %}
16582
16583 ins_encode( aarch64_enc_rethrow() );
16584
16585 ins_pipe(pipe_class_call);
16586 %}
16587
16588
16589 // Return Instruction
16590 // epilog node loads ret address into lr as part of frame pop
16591 instruct Ret()
16592 %{
16593 match(Return);
16594
16595 format %{ "ret\t// return register" %}
16596
16597 ins_encode( aarch64_enc_ret() );
16598
16599 ins_pipe(pipe_branch);
16600 %}
16601
16602 // Die now.
16603 instruct ShouldNotReachHere() %{
16604 match(Halt);
16605
16606 ins_cost(CALL_COST);
16607 format %{ "ShouldNotReachHere" %}
16608
16609 ins_encode %{
16610 if (is_reachable()) {
16611 const char* str = __ code_string(_halt_reason);
16612 __ stop(str);
16613 }
16614 %}
16615
16616 ins_pipe(pipe_class_default);
16617 %}
16618
16619 // ============================================================================
16620 // Partial Subtype Check
16621 //
16622 // superklass array for an instance of the superklass. Set a hidden
16623 // internal cache on a hit (cache is checked with exposed code in
16624 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16625 // encoding ALSO sets flags.
16626
16627 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16628 %{
16629 match(Set result (PartialSubtypeCheck sub super));
16630 predicate(!UseSecondarySupersTable);
16631 effect(KILL cr, KILL temp);
16632
16633 ins_cost(20 * INSN_COST); // slightly larger than the next version
16634 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16635
16636 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16637
16638 opcode(0x1); // Force zero of result reg on hit
16639
16640 ins_pipe(pipe_class_memory);
16641 %}
16642
16643 // Two versions of partialSubtypeCheck, both used when we need to
16644 // search for a super class in the secondary supers array. The first
16645 // is used when we don't know _a priori_ the class being searched
16646 // for. The second, far more common, is used when we do know: this is
16647 // used for instanceof, checkcast, and any case where C2 can determine
16648 // it by constant propagation.
16649
16650 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16651 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16652 rFlagsReg cr)
16653 %{
16654 match(Set result (PartialSubtypeCheck sub super));
16655 predicate(UseSecondarySupersTable);
16656 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16657
16658 ins_cost(10 * INSN_COST); // slightly larger than the next version
16659 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16660
16661 ins_encode %{
16662 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16663 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16664 $vtemp$$FloatRegister,
16665 $result$$Register, /*L_success*/nullptr);
16666 %}
16667
16668 ins_pipe(pipe_class_memory);
16669 %}
16670
16671 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16672 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16673 rFlagsReg cr)
16674 %{
16675 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16676 predicate(UseSecondarySupersTable);
16677 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16678
16679 ins_cost(5 * INSN_COST); // smaller than the next version
16680 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16681
16682 ins_encode %{
16683 bool success = false;
16684 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16685 if (InlineSecondarySupersTest) {
16686 success =
16687 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16688 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16689 $vtemp$$FloatRegister,
16690 $result$$Register,
16691 super_klass_slot);
16692 } else {
16693 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16694 success = (call != nullptr);
16695 }
16696 if (!success) {
16697 ciEnv::current()->record_failure("CodeCache is full");
16698 return;
16699 }
16700 %}
16701
16702 ins_pipe(pipe_class_memory);
16703 %}
16704
16705 // Intrisics for String.compareTo()
16706
16707 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16708 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16709 %{
16710 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16711 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16712 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16713
16714 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16715 ins_encode %{
16716 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16717 __ string_compare($str1$$Register, $str2$$Register,
16718 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16719 $tmp1$$Register, $tmp2$$Register,
16720 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16721 %}
16722 ins_pipe(pipe_class_memory);
16723 %}
16724
16725 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16726 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16727 %{
16728 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16729 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16730 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16731
16732 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16733 ins_encode %{
16734 __ string_compare($str1$$Register, $str2$$Register,
16735 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16736 $tmp1$$Register, $tmp2$$Register,
16737 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16738 %}
16739 ins_pipe(pipe_class_memory);
16740 %}
16741
16742 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16743 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16744 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16745 %{
16746 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16747 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16748 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16749 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16750
16751 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16752 ins_encode %{
16753 __ string_compare($str1$$Register, $str2$$Register,
16754 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16755 $tmp1$$Register, $tmp2$$Register,
16756 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16757 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16758 %}
16759 ins_pipe(pipe_class_memory);
16760 %}
16761
16762 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16763 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16764 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16765 %{
16766 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16767 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16768 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16769 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16770
16771 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16772 ins_encode %{
16773 __ string_compare($str1$$Register, $str2$$Register,
16774 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16775 $tmp1$$Register, $tmp2$$Register,
16776 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16777 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16778 %}
16779 ins_pipe(pipe_class_memory);
16780 %}
16781
16782 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16783 // these string_compare variants as NEON register type for convenience so that the prototype of
16784 // string_compare can be shared with all variants.
16785
16786 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16787 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16788 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16789 pRegGov_P1 pgtmp2, rFlagsReg cr)
16790 %{
16791 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16792 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16793 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16794 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16795
16796 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16797 ins_encode %{
16798 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16799 __ string_compare($str1$$Register, $str2$$Register,
16800 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16801 $tmp1$$Register, $tmp2$$Register,
16802 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16803 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16804 StrIntrinsicNode::LL);
16805 %}
16806 ins_pipe(pipe_class_memory);
16807 %}
16808
16809 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16810 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16811 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16812 pRegGov_P1 pgtmp2, rFlagsReg cr)
16813 %{
16814 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16815 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16816 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16817 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16818
16819 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16820 ins_encode %{
16821 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16822 __ string_compare($str1$$Register, $str2$$Register,
16823 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16824 $tmp1$$Register, $tmp2$$Register,
16825 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16826 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16827 StrIntrinsicNode::LU);
16828 %}
16829 ins_pipe(pipe_class_memory);
16830 %}
16831
16832 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16833 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16834 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16835 pRegGov_P1 pgtmp2, rFlagsReg cr)
16836 %{
16837 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16838 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16839 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16840 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16841
16842 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16843 ins_encode %{
16844 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16845 __ string_compare($str1$$Register, $str2$$Register,
16846 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16847 $tmp1$$Register, $tmp2$$Register,
16848 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16849 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16850 StrIntrinsicNode::UL);
16851 %}
16852 ins_pipe(pipe_class_memory);
16853 %}
16854
16855 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16856 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16857 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16858 pRegGov_P1 pgtmp2, rFlagsReg cr)
16859 %{
16860 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16861 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16862 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16863 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16864
16865 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16866 ins_encode %{
16867 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16868 __ string_compare($str1$$Register, $str2$$Register,
16869 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16870 $tmp1$$Register, $tmp2$$Register,
16871 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16872 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16873 StrIntrinsicNode::UU);
16874 %}
16875 ins_pipe(pipe_class_memory);
16876 %}
16877
16878 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16879 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16880 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16881 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16882 %{
16883 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16884 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16885 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16886 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16887 TEMP vtmp0, TEMP vtmp1, KILL cr);
16888 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16889 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16890
16891 ins_encode %{
16892 __ string_indexof($str1$$Register, $str2$$Register,
16893 $cnt1$$Register, $cnt2$$Register,
16894 $tmp1$$Register, $tmp2$$Register,
16895 $tmp3$$Register, $tmp4$$Register,
16896 $tmp5$$Register, $tmp6$$Register,
16897 -1, $result$$Register, StrIntrinsicNode::UU);
16898 %}
16899 ins_pipe(pipe_class_memory);
16900 %}
16901
16902 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16903 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16904 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16905 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16906 %{
16907 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16908 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16909 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16910 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16911 TEMP vtmp0, TEMP vtmp1, KILL cr);
16912 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16913 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16914
16915 ins_encode %{
16916 __ string_indexof($str1$$Register, $str2$$Register,
16917 $cnt1$$Register, $cnt2$$Register,
16918 $tmp1$$Register, $tmp2$$Register,
16919 $tmp3$$Register, $tmp4$$Register,
16920 $tmp5$$Register, $tmp6$$Register,
16921 -1, $result$$Register, StrIntrinsicNode::LL);
16922 %}
16923 ins_pipe(pipe_class_memory);
16924 %}
16925
16926 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16927 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16928 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16929 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16930 %{
16931 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16932 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16933 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16934 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16935 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16936 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16937 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16938
16939 ins_encode %{
16940 __ string_indexof($str1$$Register, $str2$$Register,
16941 $cnt1$$Register, $cnt2$$Register,
16942 $tmp1$$Register, $tmp2$$Register,
16943 $tmp3$$Register, $tmp4$$Register,
16944 $tmp5$$Register, $tmp6$$Register,
16945 -1, $result$$Register, StrIntrinsicNode::UL);
16946 %}
16947 ins_pipe(pipe_class_memory);
16948 %}
16949
16950 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16951 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16952 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16953 %{
16954 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16955 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16956 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16957 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16958 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16959 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16960
16961 ins_encode %{
16962 int icnt2 = (int)$int_cnt2$$constant;
16963 __ string_indexof($str1$$Register, $str2$$Register,
16964 $cnt1$$Register, zr,
16965 $tmp1$$Register, $tmp2$$Register,
16966 $tmp3$$Register, $tmp4$$Register, zr, zr,
16967 icnt2, $result$$Register, StrIntrinsicNode::UU);
16968 %}
16969 ins_pipe(pipe_class_memory);
16970 %}
16971
16972 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16973 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16974 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16975 %{
16976 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16977 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16978 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16979 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16980 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16981 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16982
16983 ins_encode %{
16984 int icnt2 = (int)$int_cnt2$$constant;
16985 __ string_indexof($str1$$Register, $str2$$Register,
16986 $cnt1$$Register, zr,
16987 $tmp1$$Register, $tmp2$$Register,
16988 $tmp3$$Register, $tmp4$$Register, zr, zr,
16989 icnt2, $result$$Register, StrIntrinsicNode::LL);
16990 %}
16991 ins_pipe(pipe_class_memory);
16992 %}
16993
16994 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16995 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16996 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16997 %{
16998 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16999 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
17000 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
17001 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
17002 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
17003 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
17004
17005 ins_encode %{
17006 int icnt2 = (int)$int_cnt2$$constant;
17007 __ string_indexof($str1$$Register, $str2$$Register,
17008 $cnt1$$Register, zr,
17009 $tmp1$$Register, $tmp2$$Register,
17010 $tmp3$$Register, $tmp4$$Register, zr, zr,
17011 icnt2, $result$$Register, StrIntrinsicNode::UL);
17012 %}
17013 ins_pipe(pipe_class_memory);
17014 %}
17015
17016 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17017 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17018 iRegINoSp tmp3, rFlagsReg cr)
17019 %{
17020 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17021 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
17022 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17023 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17024
17025 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17026
17027 ins_encode %{
17028 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17029 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17030 $tmp3$$Register);
17031 %}
17032 ins_pipe(pipe_class_memory);
17033 %}
17034
17035 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17036 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
17037 iRegINoSp tmp3, rFlagsReg cr)
17038 %{
17039 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17040 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
17041 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
17042 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
17043
17044 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17045
17046 ins_encode %{
17047 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17048 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17049 $tmp3$$Register);
17050 %}
17051 ins_pipe(pipe_class_memory);
17052 %}
17053
17054 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17055 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17056 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17057 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17058 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17059 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17060 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17061 ins_encode %{
17062 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17063 $result$$Register, $ztmp1$$FloatRegister,
17064 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17065 $ptmp$$PRegister, true /* isL */);
17066 %}
17067 ins_pipe(pipe_class_memory);
17068 %}
17069
17070 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17071 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17072 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17073 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17074 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17075 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17076 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17077 ins_encode %{
17078 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17079 $result$$Register, $ztmp1$$FloatRegister,
17080 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17081 $ptmp$$PRegister, false /* isL */);
17082 %}
17083 ins_pipe(pipe_class_memory);
17084 %}
17085
17086 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17087 iRegI_R0 result, rFlagsReg cr)
17088 %{
17089 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17090 match(Set result (StrEquals (Binary str1 str2) cnt));
17091 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17092
17093 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17094 ins_encode %{
17095 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17096 __ string_equals($str1$$Register, $str2$$Register,
17097 $result$$Register, $cnt$$Register);
17098 %}
17099 ins_pipe(pipe_class_memory);
17100 %}
17101
17102 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17103 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17104 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17105 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17106 iRegP_R10 tmp, rFlagsReg cr)
17107 %{
17108 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17109 match(Set result (AryEq ary1 ary2));
17110 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17111 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17112 TEMP vtmp6, TEMP vtmp7, KILL cr);
17113
17114 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17115 ins_encode %{
17116 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17117 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17118 $result$$Register, $tmp$$Register, 1);
17119 if (tpc == nullptr) {
17120 ciEnv::current()->record_failure("CodeCache is full");
17121 return;
17122 }
17123 %}
17124 ins_pipe(pipe_class_memory);
17125 %}
17126
17127 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17128 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17129 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17130 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17131 iRegP_R10 tmp, rFlagsReg cr)
17132 %{
17133 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17134 match(Set result (AryEq ary1 ary2));
17135 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17136 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17137 TEMP vtmp6, TEMP vtmp7, KILL cr);
17138
17139 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17140 ins_encode %{
17141 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17142 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17143 $result$$Register, $tmp$$Register, 2);
17144 if (tpc == nullptr) {
17145 ciEnv::current()->record_failure("CodeCache is full");
17146 return;
17147 }
17148 %}
17149 ins_pipe(pipe_class_memory);
17150 %}
17151
17152 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17153 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17154 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17155 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17156 %{
17157 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17158 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17159 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17160
17161 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
17162 ins_encode %{
17163 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17164 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17165 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17166 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17167 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17168 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17169 (BasicType)$basic_type$$constant);
17170 if (tpc == nullptr) {
17171 ciEnv::current()->record_failure("CodeCache is full");
17172 return;
17173 }
17174 %}
17175 ins_pipe(pipe_class_memory);
17176 %}
17177
17178 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17179 %{
17180 match(Set result (CountPositives ary1 len));
17181 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17182 format %{ "count positives byte[] $ary1,$len -> $result" %}
17183 ins_encode %{
17184 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17185 if (tpc == nullptr) {
17186 ciEnv::current()->record_failure("CodeCache is full");
17187 return;
17188 }
17189 %}
17190 ins_pipe( pipe_slow );
17191 %}
17192
17193 // fast char[] to byte[] compression
17194 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17195 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17196 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17197 iRegI_R0 result, rFlagsReg cr)
17198 %{
17199 match(Set result (StrCompressedCopy src (Binary dst len)));
17200 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17201 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17202
17203 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17204 ins_encode %{
17205 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17206 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17207 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17208 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17209 %}
17210 ins_pipe(pipe_slow);
17211 %}
17212
17213 // fast byte[] to char[] inflation
17214 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17215 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17216 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17217 %{
17218 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17219 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17220 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17221 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17222
17223 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17224 ins_encode %{
17225 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17226 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17227 $vtmp2$$FloatRegister, $tmp$$Register);
17228 if (tpc == nullptr) {
17229 ciEnv::current()->record_failure("CodeCache is full");
17230 return;
17231 }
17232 %}
17233 ins_pipe(pipe_class_memory);
17234 %}
17235
17236 // encode char[] to byte[] in ISO_8859_1
17237 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17238 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17239 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17240 iRegI_R0 result, rFlagsReg cr)
17241 %{
17242 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17243 match(Set result (EncodeISOArray src (Binary dst len)));
17244 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17245 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17246
17247 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17248 ins_encode %{
17249 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17250 $result$$Register, false,
17251 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17252 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17253 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17254 %}
17255 ins_pipe(pipe_class_memory);
17256 %}
17257
17258 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17259 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17260 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17261 iRegI_R0 result, rFlagsReg cr)
17262 %{
17263 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17264 match(Set result (EncodeISOArray src (Binary dst len)));
17265 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17266 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17267
17268 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17269 ins_encode %{
17270 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17271 $result$$Register, true,
17272 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17273 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17274 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17275 %}
17276 ins_pipe(pipe_class_memory);
17277 %}
17278
17279 //----------------------------- CompressBits/ExpandBits ------------------------
17280
17281 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17282 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17283 match(Set dst (CompressBits src mask));
17284 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17285 format %{ "mov $tsrc, $src\n\t"
17286 "mov $tmask, $mask\n\t"
17287 "bext $tdst, $tsrc, $tmask\n\t"
17288 "mov $dst, $tdst"
17289 %}
17290 ins_encode %{
17291 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17292 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17293 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17294 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17295 %}
17296 ins_pipe(pipe_slow);
17297 %}
17298
17299 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17300 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17301 match(Set dst (CompressBits (LoadI mem) mask));
17302 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17303 format %{ "ldrs $tsrc, $mem\n\t"
17304 "ldrs $tmask, $mask\n\t"
17305 "bext $tdst, $tsrc, $tmask\n\t"
17306 "mov $dst, $tdst"
17307 %}
17308 ins_encode %{
17309 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17310 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17311 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17312 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17313 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17314 %}
17315 ins_pipe(pipe_slow);
17316 %}
17317
17318 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17319 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17320 match(Set dst (CompressBits src mask));
17321 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17322 format %{ "mov $tsrc, $src\n\t"
17323 "mov $tmask, $mask\n\t"
17324 "bext $tdst, $tsrc, $tmask\n\t"
17325 "mov $dst, $tdst"
17326 %}
17327 ins_encode %{
17328 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17329 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17330 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17331 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17332 %}
17333 ins_pipe(pipe_slow);
17334 %}
17335
17336 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17337 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17338 match(Set dst (CompressBits (LoadL mem) mask));
17339 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17340 format %{ "ldrd $tsrc, $mem\n\t"
17341 "ldrd $tmask, $mask\n\t"
17342 "bext $tdst, $tsrc, $tmask\n\t"
17343 "mov $dst, $tdst"
17344 %}
17345 ins_encode %{
17346 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17347 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17348 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17349 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17350 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17351 %}
17352 ins_pipe(pipe_slow);
17353 %}
17354
17355 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17356 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17357 match(Set dst (ExpandBits src mask));
17358 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17359 format %{ "mov $tsrc, $src\n\t"
17360 "mov $tmask, $mask\n\t"
17361 "bdep $tdst, $tsrc, $tmask\n\t"
17362 "mov $dst, $tdst"
17363 %}
17364 ins_encode %{
17365 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17366 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17367 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17368 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17369 %}
17370 ins_pipe(pipe_slow);
17371 %}
17372
17373 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17374 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17375 match(Set dst (ExpandBits (LoadI mem) mask));
17376 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17377 format %{ "ldrs $tsrc, $mem\n\t"
17378 "ldrs $tmask, $mask\n\t"
17379 "bdep $tdst, $tsrc, $tmask\n\t"
17380 "mov $dst, $tdst"
17381 %}
17382 ins_encode %{
17383 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17384 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17385 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17386 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17387 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17388 %}
17389 ins_pipe(pipe_slow);
17390 %}
17391
17392 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17393 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17394 match(Set dst (ExpandBits src mask));
17395 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17396 format %{ "mov $tsrc, $src\n\t"
17397 "mov $tmask, $mask\n\t"
17398 "bdep $tdst, $tsrc, $tmask\n\t"
17399 "mov $dst, $tdst"
17400 %}
17401 ins_encode %{
17402 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17403 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17404 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17405 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17406 %}
17407 ins_pipe(pipe_slow);
17408 %}
17409
17410
17411 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17412 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17413 match(Set dst (ExpandBits (LoadL mem) mask));
17414 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17415 format %{ "ldrd $tsrc, $mem\n\t"
17416 "ldrd $tmask, $mask\n\t"
17417 "bdep $tdst, $tsrc, $tmask\n\t"
17418 "mov $dst, $tdst"
17419 %}
17420 ins_encode %{
17421 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17422 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17423 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17424 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17425 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17426 %}
17427 ins_pipe(pipe_slow);
17428 %}
17429
17430 //----------------------------- Reinterpret ----------------------------------
17431 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17432 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17433 match(Set dst (ReinterpretHF2S src));
17434 format %{ "reinterpretHF2S $dst, $src" %}
17435 ins_encode %{
17436 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17437 %}
17438 ins_pipe(pipe_slow);
17439 %}
17440
17441 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17442 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17443 match(Set dst (ReinterpretS2HF src));
17444 format %{ "reinterpretS2HF $dst, $src" %}
17445 ins_encode %{
17446 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17447 %}
17448 ins_pipe(pipe_slow);
17449 %}
17450
17451 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17452 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17453 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
17454 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
17455 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
17456 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17457 // can be omitted in this pattern, resulting in -
17458 // fcvt $dst, $src // Convert float to half-precision float
17459 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17460 %{
17461 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17462 format %{ "convF2HFAndS2HF $dst, $src" %}
17463 ins_encode %{
17464 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17465 %}
17466 ins_pipe(pipe_slow);
17467 %}
17468
17469 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17470 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17471 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
17472 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17473 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
17474 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17475 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17476 // resulting in -
17477 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
17478 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17479 %{
17480 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17481 format %{ "convHF2SAndHF2F $dst, $src" %}
17482 ins_encode %{
17483 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17484 %}
17485 ins_pipe(pipe_slow);
17486 %}
17487
17488 // ============================================================================
17489 // This name is KNOWN by the ADLC and cannot be changed.
17490 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17491 // for this guy.
17492 instruct tlsLoadP(thread_RegP dst)
17493 %{
17494 match(Set dst (ThreadLocal));
17495
17496 ins_cost(0);
17497
17498 format %{ " -- \t// $dst=Thread::current(), empty" %}
17499
17500 size(0);
17501
17502 ins_encode( /*empty*/ );
17503
17504 ins_pipe(pipe_class_empty);
17505 %}
17506
17507 //----------PEEPHOLE RULES-----------------------------------------------------
17508 // These must follow all instruction definitions as they use the names
17509 // defined in the instructions definitions.
17510 //
17511 // peepmatch ( root_instr_name [preceding_instruction]* );
17512 //
17513 // peepconstraint %{
17514 // (instruction_number.operand_name relational_op instruction_number.operand_name
17515 // [, ...] );
17516 // // instruction numbers are zero-based using left to right order in peepmatch
17517 //
17518 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17519 // // provide an instruction_number.operand_name for each operand that appears
17520 // // in the replacement instruction's match rule
17521 //
17522 // ---------VM FLAGS---------------------------------------------------------
17523 //
17524 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17525 //
17526 // Each peephole rule is given an identifying number starting with zero and
17527 // increasing by one in the order seen by the parser. An individual peephole
17528 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17529 // on the command-line.
17530 //
17531 // ---------CURRENT LIMITATIONS----------------------------------------------
17532 //
17533 // Only match adjacent instructions in same basic block
17534 // Only equality constraints
17535 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17536 // Only one replacement instruction
17537 //
17538 // ---------EXAMPLE----------------------------------------------------------
17539 //
17540 // // pertinent parts of existing instructions in architecture description
17541 // instruct movI(iRegINoSp dst, iRegI src)
17542 // %{
17543 // match(Set dst (CopyI src));
17544 // %}
17545 //
17546 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17547 // %{
17548 // match(Set dst (AddI dst src));
17549 // effect(KILL cr);
17550 // %}
17551 //
17552 // // Change (inc mov) to lea
17553 // peephole %{
17554 // // increment preceded by register-register move
17555 // peepmatch ( incI_iReg movI );
17556 // // require that the destination register of the increment
17557 // // match the destination register of the move
17558 // peepconstraint ( 0.dst == 1.dst );
17559 // // construct a replacement instruction that sets
17560 // // the destination to ( move's source register + one )
17561 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17562 // %}
17563 //
17564
17565 // Implementation no longer uses movX instructions since
17566 // machine-independent system no longer uses CopyX nodes.
17567 //
17568 // peephole
17569 // %{
17570 // peepmatch (incI_iReg movI);
17571 // peepconstraint (0.dst == 1.dst);
17572 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17573 // %}
17574
17575 // peephole
17576 // %{
17577 // peepmatch (decI_iReg movI);
17578 // peepconstraint (0.dst == 1.dst);
17579 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17580 // %}
17581
17582 // peephole
17583 // %{
17584 // peepmatch (addI_iReg_imm movI);
17585 // peepconstraint (0.dst == 1.dst);
17586 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17587 // %}
17588
17589 // peephole
17590 // %{
17591 // peepmatch (incL_iReg movL);
17592 // peepconstraint (0.dst == 1.dst);
17593 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17594 // %}
17595
17596 // peephole
17597 // %{
17598 // peepmatch (decL_iReg movL);
17599 // peepconstraint (0.dst == 1.dst);
17600 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17601 // %}
17602
17603 // peephole
17604 // %{
17605 // peepmatch (addL_iReg_imm movL);
17606 // peepconstraint (0.dst == 1.dst);
17607 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17608 // %}
17609
17610 // peephole
17611 // %{
17612 // peepmatch (addP_iReg_imm movP);
17613 // peepconstraint (0.dst == 1.dst);
17614 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17615 // %}
17616
17617 // // Change load of spilled value to only a spill
17618 // instruct storeI(memory mem, iRegI src)
17619 // %{
17620 // match(Set mem (StoreI mem src));
17621 // %}
17622 //
17623 // instruct loadI(iRegINoSp dst, memory mem)
17624 // %{
17625 // match(Set dst (LoadI mem));
17626 // %}
17627 //
17628
17629 //----------SMARTSPILL RULES---------------------------------------------------
17630 // These must follow all instruction definitions as they use the names
17631 // defined in the instructions definitions.
17632
17633 // Local Variables:
17634 // mode: c++
17635 // End: