1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BootTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else {
1694 return 6 * NativeInstruction::instruction_size;
1695 }
1696 }
1697
1698 //=============================================================================
1699
1700 #ifndef PRODUCT
1701 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1702 st->print("BREAKPOINT");
1703 }
1704 #endif
1705
1706 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1707 __ brk(0);
1708 }
1709
1710 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1711 return MachNode::size(ra_);
1712 }
1713
1714 //=============================================================================
1715
1716 #ifndef PRODUCT
1717 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1718 st->print("nop \t# %d bytes pad for loops and calls", _count);
1719 }
1720 #endif
1721
1722 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1723 for (int i = 0; i < _count; i++) {
1724 __ nop();
1725 }
1726 }
1727
1728 uint MachNopNode::size(PhaseRegAlloc*) const {
1729 return _count * NativeInstruction::instruction_size;
1730 }
1731
1732 //=============================================================================
1733 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1734
1735 int ConstantTable::calculate_table_base_offset() const {
1736 return 0; // absolute addressing, no offset
1737 }
1738
1739 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1740 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1741 ShouldNotReachHere();
1742 }
1743
1744 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1745 // Empty encoding
1746 }
1747
1748 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1749 return 0;
1750 }
1751
1752 #ifndef PRODUCT
1753 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1754 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1755 }
1756 #endif
1757
1758 #ifndef PRODUCT
1759 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1760 Compile* C = ra_->C;
1761
1762 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1763
1764 if (C->output()->need_stack_bang(framesize))
1765 st->print("# stack bang size=%d\n\t", framesize);
1766
1767 if (VM_Version::use_rop_protection()) {
1768 st->print("ldr zr, [lr]\n\t");
1769 st->print("paciaz\n\t");
1770 }
1771 if (framesize < ((1 << 9) + 2 * wordSize)) {
1772 st->print("sub sp, sp, #%d\n\t", framesize);
1773 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1774 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1775 } else {
1776 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1777 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1778 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1779 st->print("sub sp, sp, rscratch1");
1780 }
1781 if (C->stub_function() == nullptr) {
1782 st->print("\n\t");
1783 st->print("ldr rscratch1, [guard]\n\t");
1784 st->print("dmb ishld\n\t");
1785 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1786 st->print("cmp rscratch1, rscratch2\n\t");
1787 st->print("b.eq skip");
1788 st->print("\n\t");
1789 st->print("blr #nmethod_entry_barrier_stub\n\t");
1790 st->print("b skip\n\t");
1791 st->print("guard: int\n\t");
1792 st->print("\n\t");
1793 st->print("skip:\n\t");
1794 }
1795 }
1796 #endif
1797
1798 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1799 Compile* C = ra_->C;
1800
1801 // n.b. frame size includes space for return pc and rfp
1802 const int framesize = C->output()->frame_size_in_bytes();
1803
1804 if (C->clinit_barrier_on_entry()) {
1805 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1806
1807 Label L_skip_barrier;
1808
1809 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1810 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1811 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1812 __ bind(L_skip_barrier);
1813 }
1814
1815 if (C->max_vector_size() > 0) {
1816 __ reinitialize_ptrue();
1817 }
1818
1819 int bangsize = C->output()->bang_size_in_bytes();
1820 if (C->output()->need_stack_bang(bangsize))
1821 __ generate_stack_overflow_check(bangsize);
1822
1823 __ build_frame(framesize);
1824
1825 if (C->stub_function() == nullptr) {
1826 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1827 // Dummy labels for just measuring the code size
1828 Label dummy_slow_path;
1829 Label dummy_continuation;
1830 Label dummy_guard;
1831 Label* slow_path = &dummy_slow_path;
1832 Label* continuation = &dummy_continuation;
1833 Label* guard = &dummy_guard;
1834 if (!Compile::current()->output()->in_scratch_emit_size()) {
1835 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1836 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1837 Compile::current()->output()->add_stub(stub);
1838 slow_path = &stub->entry();
1839 continuation = &stub->continuation();
1840 guard = &stub->guard();
1841 }
1842 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1843 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1844 }
1845
1846 if (VerifyStackAtCalls) {
1847 Unimplemented();
1848 }
1849
1850 C->output()->set_frame_complete(__ offset());
1851
1852 if (C->has_mach_constant_base_node()) {
1853 // NOTE: We set the table base offset here because users might be
1854 // emitted before MachConstantBaseNode.
1855 ConstantTable& constant_table = C->output()->constant_table();
1856 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1857 }
1858 }
1859
1860 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1861 {
1862 return MachNode::size(ra_); // too many variables; just compute it
1863 // the hard way
1864 }
1865
1866 int MachPrologNode::reloc() const
1867 {
1868 return 0;
1869 }
1870
1871 //=============================================================================
1872
1873 #ifndef PRODUCT
1874 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 st->print("# pop frame %d\n\t",framesize);
1879
1880 if (framesize == 0) {
1881 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1882 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1883 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1884 st->print("add sp, sp, #%d\n\t", framesize);
1885 } else {
1886 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1887 st->print("add sp, sp, rscratch1\n\t");
1888 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1889 }
1890 if (VM_Version::use_rop_protection()) {
1891 st->print("autiaz\n\t");
1892 st->print("ldr zr, [lr]\n\t");
1893 }
1894
1895 if (do_polling() && C->is_method_compilation()) {
1896 st->print("# test polling word\n\t");
1897 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1898 st->print("cmp sp, rscratch1\n\t");
1899 st->print("bhi #slow_path");
1900 }
1901 }
1902 #endif
1903
1904 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1905 Compile* C = ra_->C;
1906 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1907
1908 __ remove_frame(framesize);
1909
1910 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1911 __ reserved_stack_check();
1912 }
1913
1914 if (do_polling() && C->is_method_compilation()) {
1915 Label dummy_label;
1916 Label* code_stub = &dummy_label;
1917 if (!C->output()->in_scratch_emit_size()) {
1918 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1919 C->output()->add_stub(stub);
1920 code_stub = &stub->entry();
1921 }
1922 __ relocate(relocInfo::poll_return_type);
1923 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1924 }
1925 }
1926
1927 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1928 // Variable size. Determine dynamically.
1929 return MachNode::size(ra_);
1930 }
1931
1932 int MachEpilogNode::reloc() const {
1933 // Return number of relocatable values contained in this instruction.
1934 return 1; // 1 for polling page.
1935 }
1936
1937 const Pipeline * MachEpilogNode::pipeline() const {
1938 return MachNode::pipeline_class();
1939 }
1940
1941 //=============================================================================
1942
1943 static enum RC rc_class(OptoReg::Name reg) {
1944
1945 if (reg == OptoReg::Bad) {
1946 return rc_bad;
1947 }
1948
1949 // we have 32 int registers * 2 halves
1950 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1951
1952 if (reg < slots_of_int_registers) {
1953 return rc_int;
1954 }
1955
1956 // we have 32 float register * 8 halves
1957 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1958 if (reg < slots_of_int_registers + slots_of_float_registers) {
1959 return rc_float;
1960 }
1961
1962 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1963 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1964 return rc_predicate;
1965 }
1966
1967 // Between predicate regs & stack is the flags.
1968 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1969
1970 return rc_stack;
1971 }
1972
1973 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1974 Compile* C = ra_->C;
1975
1976 // Get registers to move.
1977 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1978 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1979 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1980 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1981
1982 enum RC src_hi_rc = rc_class(src_hi);
1983 enum RC src_lo_rc = rc_class(src_lo);
1984 enum RC dst_hi_rc = rc_class(dst_hi);
1985 enum RC dst_lo_rc = rc_class(dst_lo);
1986
1987 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1988
1989 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1990 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1991 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1992 "expected aligned-adjacent pairs");
1993 }
1994
1995 if (src_lo == dst_lo && src_hi == dst_hi) {
1996 return 0; // Self copy, no move.
1997 }
1998
1999 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2000 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2001 int src_offset = ra_->reg2offset(src_lo);
2002 int dst_offset = ra_->reg2offset(dst_lo);
2003
2004 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2005 uint ireg = ideal_reg();
2006 if (ireg == Op_VecA && masm) {
2007 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2008 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2009 // stack->stack
2010 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2011 sve_vector_reg_size_in_bytes);
2012 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2013 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2017 sve_vector_reg_size_in_bytes);
2018 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2019 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2020 as_FloatRegister(Matcher::_regEncode[src_lo]),
2021 as_FloatRegister(Matcher::_regEncode[src_lo]));
2022 } else {
2023 ShouldNotReachHere();
2024 }
2025 } else if (masm) {
2026 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2027 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2028 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2029 // stack->stack
2030 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2031 if (ireg == Op_VecD) {
2032 __ unspill(rscratch1, true, src_offset);
2033 __ spill(rscratch1, true, dst_offset);
2034 } else {
2035 __ spill_copy128(src_offset, dst_offset);
2036 }
2037 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2038 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2039 ireg == Op_VecD ? __ T8B : __ T16B,
2040 as_FloatRegister(Matcher::_regEncode[src_lo]));
2041 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2042 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2043 ireg == Op_VecD ? __ D : __ Q,
2044 ra_->reg2offset(dst_lo));
2045 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2046 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2047 ireg == Op_VecD ? __ D : __ Q,
2048 ra_->reg2offset(src_lo));
2049 } else {
2050 ShouldNotReachHere();
2051 }
2052 }
2053 } else if (masm) {
2054 switch (src_lo_rc) {
2055 case rc_int:
2056 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2057 if (is64) {
2058 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2059 as_Register(Matcher::_regEncode[src_lo]));
2060 } else {
2061 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2062 as_Register(Matcher::_regEncode[src_lo]));
2063 }
2064 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2065 if (is64) {
2066 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2067 as_Register(Matcher::_regEncode[src_lo]));
2068 } else {
2069 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2070 as_Register(Matcher::_regEncode[src_lo]));
2071 }
2072 } else { // gpr --> stack spill
2073 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2074 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2075 }
2076 break;
2077 case rc_float:
2078 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2079 if (is64) {
2080 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2081 as_FloatRegister(Matcher::_regEncode[src_lo]));
2082 } else {
2083 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2084 as_FloatRegister(Matcher::_regEncode[src_lo]));
2085 }
2086 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2087 if (is64) {
2088 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2089 as_FloatRegister(Matcher::_regEncode[src_lo]));
2090 } else {
2091 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2092 as_FloatRegister(Matcher::_regEncode[src_lo]));
2093 }
2094 } else { // fpr --> stack spill
2095 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2096 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2097 is64 ? __ D : __ S, dst_offset);
2098 }
2099 break;
2100 case rc_stack:
2101 if (dst_lo_rc == rc_int) { // stack --> gpr load
2102 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2103 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2104 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2105 is64 ? __ D : __ S, src_offset);
2106 } else if (dst_lo_rc == rc_predicate) {
2107 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2108 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2109 } else { // stack --> stack copy
2110 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2111 if (ideal_reg() == Op_RegVectMask) {
2112 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2113 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2114 } else {
2115 __ unspill(rscratch1, is64, src_offset);
2116 __ spill(rscratch1, is64, dst_offset);
2117 }
2118 }
2119 break;
2120 case rc_predicate:
2121 if (dst_lo_rc == rc_predicate) {
2122 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2123 } else if (dst_lo_rc == rc_stack) {
2124 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2125 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2126 } else {
2127 assert(false, "bad src and dst rc_class combination.");
2128 ShouldNotReachHere();
2129 }
2130 break;
2131 default:
2132 assert(false, "bad rc_class for spill");
2133 ShouldNotReachHere();
2134 }
2135 }
2136
2137 if (st) {
2138 st->print("spill ");
2139 if (src_lo_rc == rc_stack) {
2140 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2141 } else {
2142 st->print("%s -> ", Matcher::regName[src_lo]);
2143 }
2144 if (dst_lo_rc == rc_stack) {
2145 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2146 } else {
2147 st->print("%s", Matcher::regName[dst_lo]);
2148 }
2149 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2150 int vsize = 0;
2151 switch (ideal_reg()) {
2152 case Op_VecD:
2153 vsize = 64;
2154 break;
2155 case Op_VecX:
2156 vsize = 128;
2157 break;
2158 case Op_VecA:
2159 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2160 break;
2161 default:
2162 assert(false, "bad register type for spill");
2163 ShouldNotReachHere();
2164 }
2165 st->print("\t# vector spill size = %d", vsize);
2166 } else if (ideal_reg() == Op_RegVectMask) {
2167 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2168 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2169 st->print("\t# predicate spill size = %d", vsize);
2170 } else {
2171 st->print("\t# spill size = %d", is64 ? 64 : 32);
2172 }
2173 }
2174
2175 return 0;
2176
2177 }
2178
2179 #ifndef PRODUCT
2180 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2181 if (!ra_)
2182 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2183 else
2184 implementation(nullptr, ra_, false, st);
2185 }
2186 #endif
2187
2188 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2189 implementation(masm, ra_, false, nullptr);
2190 }
2191
2192 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2193 return MachNode::size(ra_);
2194 }
2195
2196 //=============================================================================
2197
2198 #ifndef PRODUCT
2199 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2200 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2201 int reg = ra_->get_reg_first(this);
2202 st->print("add %s, rsp, #%d]\t# box lock",
2203 Matcher::regName[reg], offset);
2204 }
2205 #endif
2206
2207 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2208 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2209 int reg = ra_->get_encode(this);
2210
2211 // This add will handle any 24-bit signed offset. 24 bits allows an
2212 // 8 megabyte stack frame.
2213 __ add(as_Register(reg), sp, offset);
2214 }
2215
2216 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2217 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2218 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2219
2220 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2221 return NativeInstruction::instruction_size;
2222 } else {
2223 return 2 * NativeInstruction::instruction_size;
2224 }
2225 }
2226
2227 //=============================================================================
2228
2229 #ifndef PRODUCT
2230 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2231 {
2232 st->print_cr("# MachUEPNode");
2233 if (UseCompressedClassPointers) {
2234 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2235 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2236 st->print_cr("\tcmpw rscratch1, r10");
2237 } else {
2238 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2239 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2240 st->print_cr("\tcmp rscratch1, r10");
2241 }
2242 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2243 }
2244 #endif
2245
2246 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2247 {
2248 __ ic_check(InteriorEntryAlignment);
2249 }
2250
2251 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2252 {
2253 return MachNode::size(ra_);
2254 }
2255
2256 // REQUIRED EMIT CODE
2257
2258 //=============================================================================
2259
2260 // Emit deopt handler code.
2261 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2262 {
2263 // Note that the code buffer's insts_mark is always relative to insts.
2264 // That's why we must use the macroassembler to generate a handler.
2265 address base = __ start_a_stub(size_deopt_handler());
2266 if (base == nullptr) {
2267 ciEnv::current()->record_failure("CodeCache is full");
2268 return 0; // CodeBuffer::expand failed
2269 }
2270
2271 int offset = __ offset();
2272 Label start;
2273 __ bind(start);
2274 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2275
2276 int entry_offset = __ offset();
2277 __ b(start);
2278
2279 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2280 assert(__ offset() - entry_offset >= NativePostCallNop::first_check_size,
2281 "out of bounds read in post-call NOP check");
2282 __ end_a_stub();
2283 return entry_offset;
2284 }
2285
2286 // REQUIRED MATCHER CODE
2287
2288 //=============================================================================
2289
2290 bool Matcher::match_rule_supported(int opcode) {
2291 if (!has_match_rule(opcode))
2292 return false;
2293
2294 switch (opcode) {
2295 case Op_OnSpinWait:
2296 return VM_Version::supports_on_spin_wait();
2297 case Op_CacheWB:
2298 case Op_CacheWBPreSync:
2299 case Op_CacheWBPostSync:
2300 if (!VM_Version::supports_data_cache_line_flush()) {
2301 return false;
2302 }
2303 break;
2304 case Op_ExpandBits:
2305 case Op_CompressBits:
2306 if (!VM_Version::supports_svebitperm()) {
2307 return false;
2308 }
2309 break;
2310 case Op_FmaF:
2311 case Op_FmaD:
2312 case Op_FmaVF:
2313 case Op_FmaVD:
2314 if (!UseFMA) {
2315 return false;
2316 }
2317 break;
2318 case Op_FmaHF:
2319 // UseFMA flag also needs to be checked along with FEAT_FP16
2320 if (!UseFMA || !is_feat_fp16_supported()) {
2321 return false;
2322 }
2323 break;
2324 case Op_AddHF:
2325 case Op_SubHF:
2326 case Op_MulHF:
2327 case Op_DivHF:
2328 case Op_MinHF:
2329 case Op_MaxHF:
2330 case Op_SqrtHF:
2331 // Half-precision floating point scalar operations require FEAT_FP16
2332 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2333 // features are supported.
2334 if (!is_feat_fp16_supported()) {
2335 return false;
2336 }
2337 break;
2338 }
2339
2340 return true; // Per default match rules are supported.
2341 }
2342
2343 const RegMask* Matcher::predicate_reg_mask(void) {
2344 return &_PR_REG_mask;
2345 }
2346
2347 bool Matcher::supports_vector_calling_convention(void) {
2348 return EnableVectorSupport;
2349 }
2350
2351 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2352 assert(EnableVectorSupport, "sanity");
2353 int lo = V0_num;
2354 int hi = V0_H_num;
2355 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2356 hi = V0_K_num;
2357 }
2358 return OptoRegPair(hi, lo);
2359 }
2360
2361 // Is this branch offset short enough that a short branch can be used?
2362 //
2363 // NOTE: If the platform does not provide any short branch variants, then
2364 // this method should return false for offset 0.
2365 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2366 // The passed offset is relative to address of the branch.
2367
2368 return (-32768 <= offset && offset < 32768);
2369 }
2370
2371 // Vector width in bytes.
2372 int Matcher::vector_width_in_bytes(BasicType bt) {
2373 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2374 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2375 // Minimum 2 values in vector
2376 if (size < 2*type2aelembytes(bt)) size = 0;
2377 // But never < 4
2378 if (size < 4) size = 0;
2379 return size;
2380 }
2381
2382 // Limits on vector size (number of elements) loaded into vector.
2383 int Matcher::max_vector_size(const BasicType bt) {
2384 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2385 }
2386
2387 int Matcher::min_vector_size(const BasicType bt) {
2388 // Usually, the shortest vector length supported by AArch64 ISA and
2389 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2390 // vectors in a few special cases.
2391 int size;
2392 switch(bt) {
2393 case T_BOOLEAN:
2394 // Load/store a vector mask with only 2 elements for vector types
2395 // such as "2I/2F/2L/2D".
2396 size = 2;
2397 break;
2398 case T_BYTE:
2399 // Generate a "4B" vector, to support vector cast between "8B/16B"
2400 // and "4S/4I/4L/4F/4D".
2401 size = 4;
2402 break;
2403 case T_SHORT:
2404 // Generate a "2S" vector, to support vector cast between "4S/8S"
2405 // and "2I/2L/2F/2D".
2406 size = 2;
2407 break;
2408 default:
2409 // Limit the min vector length to 64-bit.
2410 size = 8 / type2aelembytes(bt);
2411 // The number of elements in a vector should be at least 2.
2412 size = MAX2(size, 2);
2413 }
2414
2415 int max_size = max_vector_size(bt);
2416 return MIN2(size, max_size);
2417 }
2418
2419 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2420 return Matcher::max_vector_size(bt);
2421 }
2422
2423 // Actual max scalable vector register length.
2424 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2425 return Matcher::max_vector_size(bt);
2426 }
2427
2428 // Vector ideal reg.
2429 uint Matcher::vector_ideal_reg(int len) {
2430 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2431 return Op_VecA;
2432 }
2433 switch(len) {
2434 // For 16-bit/32-bit mask vector, reuse VecD.
2435 case 2:
2436 case 4:
2437 case 8: return Op_VecD;
2438 case 16: return Op_VecX;
2439 }
2440 ShouldNotReachHere();
2441 return 0;
2442 }
2443
2444 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2445 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2446 switch (ideal_reg) {
2447 case Op_VecA: return new vecAOper();
2448 case Op_VecD: return new vecDOper();
2449 case Op_VecX: return new vecXOper();
2450 }
2451 ShouldNotReachHere();
2452 return nullptr;
2453 }
2454
2455 bool Matcher::is_reg2reg_move(MachNode* m) {
2456 return false;
2457 }
2458
2459 bool Matcher::is_generic_vector(MachOper* opnd) {
2460 return opnd->opcode() == VREG;
2461 }
2462
2463 // Return whether or not this register is ever used as an argument.
2464 // This function is used on startup to build the trampoline stubs in
2465 // generateOptoStub. Registers not mentioned will be killed by the VM
2466 // call in the trampoline, and arguments in those registers not be
2467 // available to the callee.
2468 bool Matcher::can_be_java_arg(int reg)
2469 {
2470 return
2471 reg == R0_num || reg == R0_H_num ||
2472 reg == R1_num || reg == R1_H_num ||
2473 reg == R2_num || reg == R2_H_num ||
2474 reg == R3_num || reg == R3_H_num ||
2475 reg == R4_num || reg == R4_H_num ||
2476 reg == R5_num || reg == R5_H_num ||
2477 reg == R6_num || reg == R6_H_num ||
2478 reg == R7_num || reg == R7_H_num ||
2479 reg == V0_num || reg == V0_H_num ||
2480 reg == V1_num || reg == V1_H_num ||
2481 reg == V2_num || reg == V2_H_num ||
2482 reg == V3_num || reg == V3_H_num ||
2483 reg == V4_num || reg == V4_H_num ||
2484 reg == V5_num || reg == V5_H_num ||
2485 reg == V6_num || reg == V6_H_num ||
2486 reg == V7_num || reg == V7_H_num;
2487 }
2488
2489 bool Matcher::is_spillable_arg(int reg)
2490 {
2491 return can_be_java_arg(reg);
2492 }
2493
2494 uint Matcher::int_pressure_limit()
2495 {
2496 // JDK-8183543: When taking the number of available registers as int
2497 // register pressure threshold, the jtreg test:
2498 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2499 // failed due to C2 compilation failure with
2500 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2501 //
2502 // A derived pointer is live at CallNode and then is flagged by RA
2503 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2504 // derived pointers and lastly fail to spill after reaching maximum
2505 // number of iterations. Lowering the default pressure threshold to
2506 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2507 // a high register pressure area of the code so that split_DEF can
2508 // generate DefinitionSpillCopy for the derived pointer.
2509 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2510 if (!PreserveFramePointer) {
2511 // When PreserveFramePointer is off, frame pointer is allocatable,
2512 // but different from other SOC registers, it is excluded from
2513 // fatproj's mask because its save type is No-Save. Decrease 1 to
2514 // ensure high pressure at fatproj when PreserveFramePointer is off.
2515 // See check_pressure_at_fatproj().
2516 default_int_pressure_threshold--;
2517 }
2518 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2519 }
2520
2521 uint Matcher::float_pressure_limit()
2522 {
2523 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2524 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2525 }
2526
2527 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2528 return false;
2529 }
2530
2531 const RegMask& Matcher::divI_proj_mask() {
2532 ShouldNotReachHere();
2533 return RegMask::EMPTY;
2534 }
2535
2536 // Register for MODI projection of divmodI.
2537 const RegMask& Matcher::modI_proj_mask() {
2538 ShouldNotReachHere();
2539 return RegMask::EMPTY;
2540 }
2541
2542 // Register for DIVL projection of divmodL.
2543 const RegMask& Matcher::divL_proj_mask() {
2544 ShouldNotReachHere();
2545 return RegMask::EMPTY;
2546 }
2547
2548 // Register for MODL projection of divmodL.
2549 const RegMask& Matcher::modL_proj_mask() {
2550 ShouldNotReachHere();
2551 return RegMask::EMPTY;
2552 }
2553
2554 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2555 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2556 Node* u = addp->fast_out(i);
2557 if (u->is_LoadStore()) {
2558 // On AArch64, LoadStoreNodes (i.e. compare and swap
2559 // instructions) only take register indirect as an operand, so
2560 // any attempt to use an AddPNode as an input to a LoadStoreNode
2561 // must fail.
2562 return false;
2563 }
2564 if (u->is_Mem()) {
2565 int opsize = u->as_Mem()->memory_size();
2566 assert(opsize > 0, "unexpected memory operand size");
2567 if (u->as_Mem()->memory_size() != (1<<shift)) {
2568 return false;
2569 }
2570 }
2571 }
2572 return true;
2573 }
2574
2575 // Convert BootTest condition to Assembler condition.
2576 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2577 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2578 Assembler::Condition result;
2579 switch(cond) {
2580 case BoolTest::eq:
2581 result = Assembler::EQ; break;
2582 case BoolTest::ne:
2583 result = Assembler::NE; break;
2584 case BoolTest::le:
2585 result = Assembler::LE; break;
2586 case BoolTest::ge:
2587 result = Assembler::GE; break;
2588 case BoolTest::lt:
2589 result = Assembler::LT; break;
2590 case BoolTest::gt:
2591 result = Assembler::GT; break;
2592 case BoolTest::ule:
2593 result = Assembler::LS; break;
2594 case BoolTest::uge:
2595 result = Assembler::HS; break;
2596 case BoolTest::ult:
2597 result = Assembler::LO; break;
2598 case BoolTest::ugt:
2599 result = Assembler::HI; break;
2600 case BoolTest::overflow:
2601 result = Assembler::VS; break;
2602 case BoolTest::no_overflow:
2603 result = Assembler::VC; break;
2604 default:
2605 ShouldNotReachHere();
2606 return Assembler::Condition(-1);
2607 }
2608
2609 // Check conversion
2610 if (cond & BoolTest::unsigned_compare) {
2611 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2612 } else {
2613 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2614 }
2615
2616 return result;
2617 }
2618
2619 // Binary src (Replicate con)
2620 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2621 if (n == nullptr || m == nullptr) {
2622 return false;
2623 }
2624
2625 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2626 return false;
2627 }
2628
2629 Node* imm_node = m->in(1);
2630 if (!imm_node->is_Con()) {
2631 return false;
2632 }
2633
2634 const Type* t = imm_node->bottom_type();
2635 if (!(t->isa_int() || t->isa_long())) {
2636 return false;
2637 }
2638
2639 switch (n->Opcode()) {
2640 case Op_AndV:
2641 case Op_OrV:
2642 case Op_XorV: {
2643 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2644 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2645 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2646 }
2647 case Op_AddVB:
2648 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2649 case Op_AddVS:
2650 case Op_AddVI:
2651 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2652 case Op_AddVL:
2653 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2654 default:
2655 return false;
2656 }
2657 }
2658
2659 // (XorV src (Replicate m1))
2660 // (XorVMask src (MaskAll m1))
2661 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2662 if (n != nullptr && m != nullptr) {
2663 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2664 VectorNode::is_all_ones_vector(m);
2665 }
2666 return false;
2667 }
2668
2669 // Should the matcher clone input 'm' of node 'n'?
2670 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2671 if (is_vshift_con_pattern(n, m) ||
2672 is_vector_bitwise_not_pattern(n, m) ||
2673 is_valid_sve_arith_imm_pattern(n, m) ||
2674 is_encode_and_store_pattern(n, m)) {
2675 mstack.push(m, Visit);
2676 return true;
2677 }
2678 return false;
2679 }
2680
2681 // Should the Matcher clone shifts on addressing modes, expecting them
2682 // to be subsumed into complex addressing expressions or compute them
2683 // into registers?
2684 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2685
2686 // Loads and stores with indirect memory input (e.g., volatile loads and
2687 // stores) do not subsume the input into complex addressing expressions. If
2688 // the addressing expression is input to at least one such load or store, do
2689 // not clone the addressing expression. Query needs_acquiring_load and
2690 // needs_releasing_store as a proxy for indirect memory input, as it is not
2691 // possible to directly query for indirect memory input at this stage.
2692 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2693 Node* n = m->fast_out(i);
2694 if (n->is_Load() && needs_acquiring_load(n)) {
2695 return false;
2696 }
2697 if (n->is_Store() && needs_releasing_store(n)) {
2698 return false;
2699 }
2700 }
2701
2702 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2703 return true;
2704 }
2705
2706 Node *off = m->in(AddPNode::Offset);
2707 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2708 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2709 // Are there other uses besides address expressions?
2710 !is_visited(off)) {
2711 address_visited.set(off->_idx); // Flag as address_visited
2712 mstack.push(off->in(2), Visit);
2713 Node *conv = off->in(1);
2714 if (conv->Opcode() == Op_ConvI2L &&
2715 // Are there other uses besides address expressions?
2716 !is_visited(conv)) {
2717 address_visited.set(conv->_idx); // Flag as address_visited
2718 mstack.push(conv->in(1), Pre_Visit);
2719 } else {
2720 mstack.push(conv, Pre_Visit);
2721 }
2722 address_visited.test_set(m->_idx); // Flag as address_visited
2723 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2724 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2725 return true;
2726 } else if (off->Opcode() == Op_ConvI2L &&
2727 // Are there other uses besides address expressions?
2728 !is_visited(off)) {
2729 address_visited.test_set(m->_idx); // Flag as address_visited
2730 address_visited.set(off->_idx); // Flag as address_visited
2731 mstack.push(off->in(1), Pre_Visit);
2732 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2733 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2734 return true;
2735 }
2736 return false;
2737 }
2738
2739 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2740 { \
2741 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2742 guarantee(DISP == 0, "mode not permitted for volatile"); \
2743 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2744 __ INSN(REG, as_Register(BASE)); \
2745 }
2746
2747
2748 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2749 {
2750 Address::extend scale;
2751
2752 // Hooboy, this is fugly. We need a way to communicate to the
2753 // encoder that the index needs to be sign extended, so we have to
2754 // enumerate all the cases.
2755 switch (opcode) {
2756 case INDINDEXSCALEDI2L:
2757 case INDINDEXSCALEDI2LN:
2758 case INDINDEXI2L:
2759 case INDINDEXI2LN:
2760 scale = Address::sxtw(size);
2761 break;
2762 default:
2763 scale = Address::lsl(size);
2764 }
2765
2766 if (index == -1) {
2767 return Address(base, disp);
2768 } else {
2769 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2770 return Address(base, as_Register(index), scale);
2771 }
2772 }
2773
2774
2775 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2776 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2777 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2778 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2779 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2780
2781 // Used for all non-volatile memory accesses. The use of
2782 // $mem->opcode() to discover whether this pattern uses sign-extended
2783 // offsets is something of a kludge.
2784 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2785 Register reg, int opcode,
2786 Register base, int index, int scale, int disp,
2787 int size_in_memory)
2788 {
2789 Address addr = mem2address(opcode, base, index, scale, disp);
2790 if (addr.getMode() == Address::base_plus_offset) {
2791 /* Fix up any out-of-range offsets. */
2792 assert_different_registers(rscratch1, base);
2793 assert_different_registers(rscratch1, reg);
2794 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2795 }
2796 (masm->*insn)(reg, addr);
2797 }
2798
2799 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2800 FloatRegister reg, int opcode,
2801 Register base, int index, int size, int disp,
2802 int size_in_memory)
2803 {
2804 Address::extend scale;
2805
2806 switch (opcode) {
2807 case INDINDEXSCALEDI2L:
2808 case INDINDEXSCALEDI2LN:
2809 scale = Address::sxtw(size);
2810 break;
2811 default:
2812 scale = Address::lsl(size);
2813 }
2814
2815 if (index == -1) {
2816 // Fix up any out-of-range offsets.
2817 assert_different_registers(rscratch1, base);
2818 Address addr = Address(base, disp);
2819 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2820 (masm->*insn)(reg, addr);
2821 } else {
2822 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2823 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2824 }
2825 }
2826
2827 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2828 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2829 int opcode, Register base, int index, int size, int disp)
2830 {
2831 if (index == -1) {
2832 (masm->*insn)(reg, T, Address(base, disp));
2833 } else {
2834 assert(disp == 0, "unsupported address mode");
2835 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2836 }
2837 }
2838
2839 %}
2840
2841
2842
2843 //----------ENCODING BLOCK-----------------------------------------------------
2844 // This block specifies the encoding classes used by the compiler to
2845 // output byte streams. Encoding classes are parameterized macros
2846 // used by Machine Instruction Nodes in order to generate the bit
2847 // encoding of the instruction. Operands specify their base encoding
2848 // interface with the interface keyword. There are currently
2849 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2850 // COND_INTER. REG_INTER causes an operand to generate a function
2851 // which returns its register number when queried. CONST_INTER causes
2852 // an operand to generate a function which returns the value of the
2853 // constant when queried. MEMORY_INTER causes an operand to generate
2854 // four functions which return the Base Register, the Index Register,
2855 // the Scale Value, and the Offset Value of the operand when queried.
2856 // COND_INTER causes an operand to generate six functions which return
2857 // the encoding code (ie - encoding bits for the instruction)
2858 // associated with each basic boolean condition for a conditional
2859 // instruction.
2860 //
2861 // Instructions specify two basic values for encoding. Again, a
2862 // function is available to check if the constant displacement is an
2863 // oop. They use the ins_encode keyword to specify their encoding
2864 // classes (which must be a sequence of enc_class names, and their
2865 // parameters, specified in the encoding block), and they use the
2866 // opcode keyword to specify, in order, their primary, secondary, and
2867 // tertiary opcode. Only the opcode sections which a particular
2868 // instruction needs for encoding need to be specified.
2869 encode %{
2870 // Build emit functions for each basic byte or larger field in the
2871 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2872 // from C++ code in the enc_class source block. Emit functions will
2873 // live in the main source block for now. In future, we can
2874 // generalize this by adding a syntax that specifies the sizes of
2875 // fields in an order, so that the adlc can build the emit functions
2876 // automagically
2877
2878 // catch all for unimplemented encodings
2879 enc_class enc_unimplemented %{
2880 __ unimplemented("C2 catch all");
2881 %}
2882
2883 // BEGIN Non-volatile memory access
2884
2885 // This encoding class is generated automatically from ad_encode.m4.
2886 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2887 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2888 Register dst_reg = as_Register($dst$$reg);
2889 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2890 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2891 %}
2892
2893 // This encoding class is generated automatically from ad_encode.m4.
2894 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2895 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2896 Register dst_reg = as_Register($dst$$reg);
2897 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2898 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2899 %}
2900
2901 // This encoding class is generated automatically from ad_encode.m4.
2902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2903 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2904 Register dst_reg = as_Register($dst$$reg);
2905 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2906 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2907 %}
2908
2909 // This encoding class is generated automatically from ad_encode.m4.
2910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2911 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2912 Register dst_reg = as_Register($dst$$reg);
2913 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2914 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2915 %}
2916
2917 // This encoding class is generated automatically from ad_encode.m4.
2918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2919 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2920 Register dst_reg = as_Register($dst$$reg);
2921 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2922 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2923 %}
2924
2925 // This encoding class is generated automatically from ad_encode.m4.
2926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2927 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2928 Register dst_reg = as_Register($dst$$reg);
2929 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2931 %}
2932
2933 // This encoding class is generated automatically from ad_encode.m4.
2934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2935 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2936 Register dst_reg = as_Register($dst$$reg);
2937 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2944 Register dst_reg = as_Register($dst$$reg);
2945 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2946 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2947 %}
2948
2949 // This encoding class is generated automatically from ad_encode.m4.
2950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2951 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2952 Register dst_reg = as_Register($dst$$reg);
2953 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2954 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2955 %}
2956
2957 // This encoding class is generated automatically from ad_encode.m4.
2958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2959 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2960 Register dst_reg = as_Register($dst$$reg);
2961 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2962 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2963 %}
2964
2965 // This encoding class is generated automatically from ad_encode.m4.
2966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2967 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2968 Register dst_reg = as_Register($dst$$reg);
2969 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2970 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2971 %}
2972
2973 // This encoding class is generated automatically from ad_encode.m4.
2974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2975 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2976 Register dst_reg = as_Register($dst$$reg);
2977 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2978 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2979 %}
2980
2981 // This encoding class is generated automatically from ad_encode.m4.
2982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2983 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2984 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2985 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2986 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2987 %}
2988
2989 // This encoding class is generated automatically from ad_encode.m4.
2990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2991 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2992 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2993 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2994 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2995 %}
2996
2997 // This encoding class is generated automatically from ad_encode.m4.
2998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2999 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3000 Register src_reg = as_Register($src$$reg);
3001 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3002 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3003 %}
3004
3005 // This encoding class is generated automatically from ad_encode.m4.
3006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3007 enc_class aarch64_enc_strb0(memory1 mem) %{
3008 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3009 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3010 %}
3011
3012 // This encoding class is generated automatically from ad_encode.m4.
3013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3014 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3015 Register src_reg = as_Register($src$$reg);
3016 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3017 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3018 %}
3019
3020 // This encoding class is generated automatically from ad_encode.m4.
3021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3022 enc_class aarch64_enc_strh0(memory2 mem) %{
3023 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3024 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3025 %}
3026
3027 // This encoding class is generated automatically from ad_encode.m4.
3028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3029 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3030 Register src_reg = as_Register($src$$reg);
3031 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3032 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3033 %}
3034
3035 // This encoding class is generated automatically from ad_encode.m4.
3036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3037 enc_class aarch64_enc_strw0(memory4 mem) %{
3038 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3039 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3040 %}
3041
3042 // This encoding class is generated automatically from ad_encode.m4.
3043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3044 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3045 Register src_reg = as_Register($src$$reg);
3046 // we sometimes get asked to store the stack pointer into the
3047 // current thread -- we cannot do that directly on AArch64
3048 if (src_reg == r31_sp) {
3049 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3050 __ mov(rscratch2, sp);
3051 src_reg = rscratch2;
3052 }
3053 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3054 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3055 %}
3056
3057 // This encoding class is generated automatically from ad_encode.m4.
3058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3059 enc_class aarch64_enc_str0(memory8 mem) %{
3060 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3061 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3062 %}
3063
3064 // This encoding class is generated automatically from ad_encode.m4.
3065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3066 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3067 FloatRegister src_reg = as_FloatRegister($src$$reg);
3068 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3069 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3070 %}
3071
3072 // This encoding class is generated automatically from ad_encode.m4.
3073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3074 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3075 FloatRegister src_reg = as_FloatRegister($src$$reg);
3076 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3077 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3078 %}
3079
3080 // This encoding class is generated automatically from ad_encode.m4.
3081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3082 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3083 __ membar(Assembler::StoreStore);
3084 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3085 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3086 %}
3087
3088 // END Non-volatile memory access
3089
3090 // Vector loads and stores
3091 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3092 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3093 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3094 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3095 %}
3096
3097 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3098 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3099 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3100 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3101 %}
3102
3103 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3104 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3105 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3106 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3107 %}
3108
3109 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3110 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3111 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3112 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3113 %}
3114
3115 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3116 FloatRegister src_reg = as_FloatRegister($src$$reg);
3117 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3118 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3119 %}
3120
3121 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3122 FloatRegister src_reg = as_FloatRegister($src$$reg);
3123 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3124 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3125 %}
3126
3127 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3128 FloatRegister src_reg = as_FloatRegister($src$$reg);
3129 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3130 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3131 %}
3132
3133 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3134 FloatRegister src_reg = as_FloatRegister($src$$reg);
3135 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3136 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3137 %}
3138
3139 // volatile loads and stores
3140
3141 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3142 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3143 rscratch1, stlrb);
3144 %}
3145
3146 enc_class aarch64_enc_stlrb0(memory mem) %{
3147 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3148 rscratch1, stlrb);
3149 %}
3150
3151 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3152 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3153 rscratch1, stlrh);
3154 %}
3155
3156 enc_class aarch64_enc_stlrh0(memory mem) %{
3157 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3158 rscratch1, stlrh);
3159 %}
3160
3161 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3162 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3163 rscratch1, stlrw);
3164 %}
3165
3166 enc_class aarch64_enc_stlrw0(memory mem) %{
3167 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3168 rscratch1, stlrw);
3169 %}
3170
3171 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3172 Register dst_reg = as_Register($dst$$reg);
3173 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3174 rscratch1, ldarb);
3175 __ sxtbw(dst_reg, dst_reg);
3176 %}
3177
3178 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3179 Register dst_reg = as_Register($dst$$reg);
3180 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3181 rscratch1, ldarb);
3182 __ sxtb(dst_reg, dst_reg);
3183 %}
3184
3185 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3186 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3187 rscratch1, ldarb);
3188 %}
3189
3190 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3191 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3192 rscratch1, ldarb);
3193 %}
3194
3195 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3196 Register dst_reg = as_Register($dst$$reg);
3197 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3198 rscratch1, ldarh);
3199 __ sxthw(dst_reg, dst_reg);
3200 %}
3201
3202 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3203 Register dst_reg = as_Register($dst$$reg);
3204 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3205 rscratch1, ldarh);
3206 __ sxth(dst_reg, dst_reg);
3207 %}
3208
3209 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3210 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3211 rscratch1, ldarh);
3212 %}
3213
3214 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3215 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3216 rscratch1, ldarh);
3217 %}
3218
3219 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3220 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3221 rscratch1, ldarw);
3222 %}
3223
3224 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3225 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, ldarw);
3227 %}
3228
3229 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3230 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3231 rscratch1, ldar);
3232 %}
3233
3234 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3235 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3236 rscratch1, ldarw);
3237 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3238 %}
3239
3240 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3241 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3242 rscratch1, ldar);
3243 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3244 %}
3245
3246 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3247 Register src_reg = as_Register($src$$reg);
3248 // we sometimes get asked to store the stack pointer into the
3249 // current thread -- we cannot do that directly on AArch64
3250 if (src_reg == r31_sp) {
3251 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3252 __ mov(rscratch2, sp);
3253 src_reg = rscratch2;
3254 }
3255 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3256 rscratch1, stlr);
3257 %}
3258
3259 enc_class aarch64_enc_stlr0(memory mem) %{
3260 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3261 rscratch1, stlr);
3262 %}
3263
3264 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3265 {
3266 FloatRegister src_reg = as_FloatRegister($src$$reg);
3267 __ fmovs(rscratch2, src_reg);
3268 }
3269 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3270 rscratch1, stlrw);
3271 %}
3272
3273 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3274 {
3275 FloatRegister src_reg = as_FloatRegister($src$$reg);
3276 __ fmovd(rscratch2, src_reg);
3277 }
3278 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3279 rscratch1, stlr);
3280 %}
3281
3282 // synchronized read/update encodings
3283
3284 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3285 Register dst_reg = as_Register($dst$$reg);
3286 Register base = as_Register($mem$$base);
3287 int index = $mem$$index;
3288 int scale = $mem$$scale;
3289 int disp = $mem$$disp;
3290 if (index == -1) {
3291 if (disp != 0) {
3292 __ lea(rscratch1, Address(base, disp));
3293 __ ldaxr(dst_reg, rscratch1);
3294 } else {
3295 // TODO
3296 // should we ever get anything other than this case?
3297 __ ldaxr(dst_reg, base);
3298 }
3299 } else {
3300 Register index_reg = as_Register(index);
3301 if (disp == 0) {
3302 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3303 __ ldaxr(dst_reg, rscratch1);
3304 } else {
3305 __ lea(rscratch1, Address(base, disp));
3306 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3307 __ ldaxr(dst_reg, rscratch1);
3308 }
3309 }
3310 %}
3311
3312 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3313 Register src_reg = as_Register($src$$reg);
3314 Register base = as_Register($mem$$base);
3315 int index = $mem$$index;
3316 int scale = $mem$$scale;
3317 int disp = $mem$$disp;
3318 if (index == -1) {
3319 if (disp != 0) {
3320 __ lea(rscratch2, Address(base, disp));
3321 __ stlxr(rscratch1, src_reg, rscratch2);
3322 } else {
3323 // TODO
3324 // should we ever get anything other than this case?
3325 __ stlxr(rscratch1, src_reg, base);
3326 }
3327 } else {
3328 Register index_reg = as_Register(index);
3329 if (disp == 0) {
3330 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3331 __ stlxr(rscratch1, src_reg, rscratch2);
3332 } else {
3333 __ lea(rscratch2, Address(base, disp));
3334 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3335 __ stlxr(rscratch1, src_reg, rscratch2);
3336 }
3337 }
3338 __ cmpw(rscratch1, zr);
3339 %}
3340
3341 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3342 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3343 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3344 Assembler::xword, /*acquire*/ false, /*release*/ true,
3345 /*weak*/ false, noreg);
3346 %}
3347
3348 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3349 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3350 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3351 Assembler::word, /*acquire*/ false, /*release*/ true,
3352 /*weak*/ false, noreg);
3353 %}
3354
3355 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3356 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3357 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3358 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3359 /*weak*/ false, noreg);
3360 %}
3361
3362 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3363 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3364 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3365 Assembler::byte, /*acquire*/ false, /*release*/ true,
3366 /*weak*/ false, noreg);
3367 %}
3368
3369
3370 // The only difference between aarch64_enc_cmpxchg and
3371 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3372 // CompareAndSwap sequence to serve as a barrier on acquiring a
3373 // lock.
3374 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegL oldval, iRegL newval) %{
3375 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3376 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3377 Assembler::xword, /*acquire*/ true, /*release*/ true,
3378 /*weak*/ false, noreg);
3379 %}
3380
3381 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegI oldval, iRegI newval) %{
3382 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3383 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3384 Assembler::word, /*acquire*/ true, /*release*/ true,
3385 /*weak*/ false, noreg);
3386 %}
3387
3388 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegI oldval, iRegI newval) %{
3389 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3390 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3391 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3392 /*weak*/ false, noreg);
3393 %}
3394
3395 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegI oldval, iRegI newval) %{
3396 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3397 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3398 Assembler::byte, /*acquire*/ true, /*release*/ true,
3399 /*weak*/ false, noreg);
3400 %}
3401
3402 // auxiliary used for CompareAndSwapX to set result register
3403 enc_class aarch64_enc_cset_eq(iRegI res) %{
3404 Register res_reg = as_Register($res$$reg);
3405 __ cset(res_reg, Assembler::EQ);
3406 %}
3407
3408 // prefetch encodings
3409
3410 enc_class aarch64_enc_prefetchw(memory mem) %{
3411 Register base = as_Register($mem$$base);
3412 int index = $mem$$index;
3413 int scale = $mem$$scale;
3414 int disp = $mem$$disp;
3415 if (index == -1) {
3416 // Fix up any out-of-range offsets.
3417 assert_different_registers(rscratch1, base);
3418 Address addr = Address(base, disp);
3419 addr = __ legitimize_address(addr, 8, rscratch1);
3420 __ prfm(addr, PSTL1KEEP);
3421 } else {
3422 Register index_reg = as_Register(index);
3423 if (disp == 0) {
3424 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3425 } else {
3426 __ lea(rscratch1, Address(base, disp));
3427 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3428 }
3429 }
3430 %}
3431
3432 // mov encodings
3433
3434 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3435 uint32_t con = (uint32_t)$src$$constant;
3436 Register dst_reg = as_Register($dst$$reg);
3437 if (con == 0) {
3438 __ movw(dst_reg, zr);
3439 } else {
3440 __ movw(dst_reg, con);
3441 }
3442 %}
3443
3444 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3445 Register dst_reg = as_Register($dst$$reg);
3446 uint64_t con = (uint64_t)$src$$constant;
3447 if (con == 0) {
3448 __ mov(dst_reg, zr);
3449 } else {
3450 __ mov(dst_reg, con);
3451 }
3452 %}
3453
3454 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3455 Register dst_reg = as_Register($dst$$reg);
3456 address con = (address)$src$$constant;
3457 if (con == nullptr || con == (address)1) {
3458 ShouldNotReachHere();
3459 } else {
3460 relocInfo::relocType rtype = $src->constant_reloc();
3461 if (rtype == relocInfo::oop_type) {
3462 __ movoop(dst_reg, (jobject)con);
3463 } else if (rtype == relocInfo::metadata_type) {
3464 __ mov_metadata(dst_reg, (Metadata*)con);
3465 } else {
3466 assert(rtype == relocInfo::none, "unexpected reloc type");
3467 if (! __ is_valid_AArch64_address(con) ||
3468 con < (address)(uintptr_t)os::vm_page_size()) {
3469 __ mov(dst_reg, con);
3470 } else {
3471 uint64_t offset;
3472 __ adrp(dst_reg, con, offset);
3473 __ add(dst_reg, dst_reg, offset);
3474 }
3475 }
3476 }
3477 %}
3478
3479 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3480 Register dst_reg = as_Register($dst$$reg);
3481 __ mov(dst_reg, zr);
3482 %}
3483
3484 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3485 Register dst_reg = as_Register($dst$$reg);
3486 __ mov(dst_reg, (uint64_t)1);
3487 %}
3488
3489 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3490 Register dst_reg = as_Register($dst$$reg);
3491 address con = (address)$src$$constant;
3492 if (con == nullptr) {
3493 ShouldNotReachHere();
3494 } else {
3495 relocInfo::relocType rtype = $src->constant_reloc();
3496 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3497 __ set_narrow_oop(dst_reg, (jobject)con);
3498 }
3499 %}
3500
3501 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3502 Register dst_reg = as_Register($dst$$reg);
3503 __ mov(dst_reg, zr);
3504 %}
3505
3506 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3507 Register dst_reg = as_Register($dst$$reg);
3508 address con = (address)$src$$constant;
3509 if (con == nullptr) {
3510 ShouldNotReachHere();
3511 } else {
3512 relocInfo::relocType rtype = $src->constant_reloc();
3513 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3514 __ set_narrow_klass(dst_reg, (Klass *)con);
3515 }
3516 %}
3517
3518 // arithmetic encodings
3519
3520 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3521 Register dst_reg = as_Register($dst$$reg);
3522 Register src_reg = as_Register($src1$$reg);
3523 int32_t con = (int32_t)$src2$$constant;
3524 // add has primary == 0, subtract has primary == 1
3525 if ($primary) { con = -con; }
3526 if (con < 0) {
3527 __ subw(dst_reg, src_reg, -con);
3528 } else {
3529 __ addw(dst_reg, src_reg, con);
3530 }
3531 %}
3532
3533 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3534 Register dst_reg = as_Register($dst$$reg);
3535 Register src_reg = as_Register($src1$$reg);
3536 int32_t con = (int32_t)$src2$$constant;
3537 // add has primary == 0, subtract has primary == 1
3538 if ($primary) { con = -con; }
3539 if (con < 0) {
3540 __ sub(dst_reg, src_reg, -con);
3541 } else {
3542 __ add(dst_reg, src_reg, con);
3543 }
3544 %}
3545
3546 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3547 Register dst_reg = as_Register($dst$$reg);
3548 Register src1_reg = as_Register($src1$$reg);
3549 Register src2_reg = as_Register($src2$$reg);
3550 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3551 %}
3552
3553 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3554 Register dst_reg = as_Register($dst$$reg);
3555 Register src1_reg = as_Register($src1$$reg);
3556 Register src2_reg = as_Register($src2$$reg);
3557 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3558 %}
3559
3560 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3561 Register dst_reg = as_Register($dst$$reg);
3562 Register src1_reg = as_Register($src1$$reg);
3563 Register src2_reg = as_Register($src2$$reg);
3564 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3565 %}
3566
3567 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3568 Register dst_reg = as_Register($dst$$reg);
3569 Register src1_reg = as_Register($src1$$reg);
3570 Register src2_reg = as_Register($src2$$reg);
3571 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3572 %}
3573
3574 // compare instruction encodings
3575
3576 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3577 Register reg1 = as_Register($src1$$reg);
3578 Register reg2 = as_Register($src2$$reg);
3579 __ cmpw(reg1, reg2);
3580 %}
3581
3582 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3583 Register reg = as_Register($src1$$reg);
3584 int32_t val = $src2$$constant;
3585 if (val >= 0) {
3586 __ subsw(zr, reg, val);
3587 } else {
3588 __ addsw(zr, reg, -val);
3589 }
3590 %}
3591
3592 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3593 Register reg1 = as_Register($src1$$reg);
3594 uint32_t val = (uint32_t)$src2$$constant;
3595 __ movw(rscratch1, val);
3596 __ cmpw(reg1, rscratch1);
3597 %}
3598
3599 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3600 Register reg1 = as_Register($src1$$reg);
3601 Register reg2 = as_Register($src2$$reg);
3602 __ cmp(reg1, reg2);
3603 %}
3604
3605 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3606 Register reg = as_Register($src1$$reg);
3607 int64_t val = $src2$$constant;
3608 if (val >= 0) {
3609 __ subs(zr, reg, val);
3610 } else if (val != -val) {
3611 __ adds(zr, reg, -val);
3612 } else {
3613 // aargh, Long.MIN_VALUE is a special case
3614 __ orr(rscratch1, zr, (uint64_t)val);
3615 __ subs(zr, reg, rscratch1);
3616 }
3617 %}
3618
3619 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3620 Register reg1 = as_Register($src1$$reg);
3621 uint64_t val = (uint64_t)$src2$$constant;
3622 __ mov(rscratch1, val);
3623 __ cmp(reg1, rscratch1);
3624 %}
3625
3626 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3627 Register reg1 = as_Register($src1$$reg);
3628 Register reg2 = as_Register($src2$$reg);
3629 __ cmp(reg1, reg2);
3630 %}
3631
3632 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3633 Register reg1 = as_Register($src1$$reg);
3634 Register reg2 = as_Register($src2$$reg);
3635 __ cmpw(reg1, reg2);
3636 %}
3637
3638 enc_class aarch64_enc_testp(iRegP src) %{
3639 Register reg = as_Register($src$$reg);
3640 __ cmp(reg, zr);
3641 %}
3642
3643 enc_class aarch64_enc_testn(iRegN src) %{
3644 Register reg = as_Register($src$$reg);
3645 __ cmpw(reg, zr);
3646 %}
3647
3648 enc_class aarch64_enc_b(label lbl) %{
3649 Label *L = $lbl$$label;
3650 __ b(*L);
3651 %}
3652
3653 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3654 Label *L = $lbl$$label;
3655 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3656 %}
3657
3658 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3659 Label *L = $lbl$$label;
3660 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3661 %}
3662
3663 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3664 %{
3665 Register sub_reg = as_Register($sub$$reg);
3666 Register super_reg = as_Register($super$$reg);
3667 Register temp_reg = as_Register($temp$$reg);
3668 Register result_reg = as_Register($result$$reg);
3669
3670 Label miss;
3671 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3672 nullptr, &miss,
3673 /*set_cond_codes:*/ true);
3674 if ($primary) {
3675 __ mov(result_reg, zr);
3676 }
3677 __ bind(miss);
3678 %}
3679
3680 enc_class aarch64_enc_java_static_call(method meth) %{
3681 address addr = (address)$meth$$method;
3682 address call;
3683 if (!_method) {
3684 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3685 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3686 if (call == nullptr) {
3687 ciEnv::current()->record_failure("CodeCache is full");
3688 return;
3689 }
3690 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3691 // The NOP here is purely to ensure that eliding a call to
3692 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3693 __ nop();
3694 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3695 } else {
3696 int method_index = resolved_method_index(masm);
3697 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3698 : static_call_Relocation::spec(method_index);
3699 call = __ trampoline_call(Address(addr, rspec));
3700 if (call == nullptr) {
3701 ciEnv::current()->record_failure("CodeCache is full");
3702 return;
3703 }
3704 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3705 // Calls of the same statically bound method can share
3706 // a stub to the interpreter.
3707 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3708 } else {
3709 // Emit stub for static call
3710 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3711 if (stub == nullptr) {
3712 ciEnv::current()->record_failure("CodeCache is full");
3713 return;
3714 }
3715 }
3716 }
3717
3718 __ post_call_nop();
3719
3720 // Only non uncommon_trap calls need to reinitialize ptrue.
3721 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3722 __ reinitialize_ptrue();
3723 }
3724 %}
3725
3726 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3727 int method_index = resolved_method_index(masm);
3728 address call = __ ic_call((address)$meth$$method, method_index);
3729 if (call == nullptr) {
3730 ciEnv::current()->record_failure("CodeCache is full");
3731 return;
3732 }
3733 __ post_call_nop();
3734 if (Compile::current()->max_vector_size() > 0) {
3735 __ reinitialize_ptrue();
3736 }
3737 %}
3738
3739 enc_class aarch64_enc_call_epilog() %{
3740 if (VerifyStackAtCalls) {
3741 // Check that stack depth is unchanged: find majik cookie on stack
3742 __ call_Unimplemented();
3743 }
3744 %}
3745
3746 enc_class aarch64_enc_java_to_runtime(method meth) %{
3747 // some calls to generated routines (arraycopy code) are scheduled
3748 // by C2 as runtime calls. if so we can call them using a br (they
3749 // will be in a reachable segment) otherwise we have to use a blr
3750 // which loads the absolute address into a register.
3751 address entry = (address)$meth$$method;
3752 CodeBlob *cb = CodeCache::find_blob(entry);
3753 if (cb) {
3754 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3755 if (call == nullptr) {
3756 ciEnv::current()->record_failure("CodeCache is full");
3757 return;
3758 }
3759 __ post_call_nop();
3760 } else {
3761 Label retaddr;
3762 // Make the anchor frame walkable
3763 __ adr(rscratch2, retaddr);
3764 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3765 __ lea(rscratch1, RuntimeAddress(entry));
3766 __ blr(rscratch1);
3767 __ bind(retaddr);
3768 __ post_call_nop();
3769 }
3770 if (Compile::current()->max_vector_size() > 0) {
3771 __ reinitialize_ptrue();
3772 }
3773 %}
3774
3775 enc_class aarch64_enc_rethrow() %{
3776 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3777 %}
3778
3779 enc_class aarch64_enc_ret() %{
3780 #ifdef ASSERT
3781 if (Compile::current()->max_vector_size() > 0) {
3782 __ verify_ptrue();
3783 }
3784 #endif
3785 __ ret(lr);
3786 %}
3787
3788 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3789 Register target_reg = as_Register($jump_target$$reg);
3790 __ br(target_reg);
3791 %}
3792
3793 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3794 Register target_reg = as_Register($jump_target$$reg);
3795 // exception oop should be in r0
3796 // ret addr has been popped into lr
3797 // callee expects it in r3
3798 __ mov(r3, lr);
3799 __ br(target_reg);
3800 %}
3801
3802 %}
3803
3804 //----------FRAME--------------------------------------------------------------
3805 // Definition of frame structure and management information.
3806 //
3807 // S T A C K L A Y O U T Allocators stack-slot number
3808 // | (to get allocators register number
3809 // G Owned by | | v add OptoReg::stack0())
3810 // r CALLER | |
3811 // o | +--------+ pad to even-align allocators stack-slot
3812 // w V | pad0 | numbers; owned by CALLER
3813 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3814 // h ^ | in | 5
3815 // | | args | 4 Holes in incoming args owned by SELF
3816 // | | | | 3
3817 // | | +--------+
3818 // V | | old out| Empty on Intel, window on Sparc
3819 // | old |preserve| Must be even aligned.
3820 // | SP-+--------+----> Matcher::_old_SP, even aligned
3821 // | | in | 3 area for Intel ret address
3822 // Owned by |preserve| Empty on Sparc.
3823 // SELF +--------+
3824 // | | pad2 | 2 pad to align old SP
3825 // | +--------+ 1
3826 // | | locks | 0
3827 // | +--------+----> OptoReg::stack0(), even aligned
3828 // | | pad1 | 11 pad to align new SP
3829 // | +--------+
3830 // | | | 10
3831 // | | spills | 9 spills
3832 // V | | 8 (pad0 slot for callee)
3833 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3834 // ^ | out | 7
3835 // | | args | 6 Holes in outgoing args owned by CALLEE
3836 // Owned by +--------+
3837 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3838 // | new |preserve| Must be even-aligned.
3839 // | SP-+--------+----> Matcher::_new_SP, even aligned
3840 // | | |
3841 //
3842 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3843 // known from SELF's arguments and the Java calling convention.
3844 // Region 6-7 is determined per call site.
3845 // Note 2: If the calling convention leaves holes in the incoming argument
3846 // area, those holes are owned by SELF. Holes in the outgoing area
3847 // are owned by the CALLEE. Holes should not be necessary in the
3848 // incoming area, as the Java calling convention is completely under
3849 // the control of the AD file. Doubles can be sorted and packed to
3850 // avoid holes. Holes in the outgoing arguments may be necessary for
3851 // varargs C calling conventions.
3852 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3853 // even aligned with pad0 as needed.
3854 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3855 // (the latter is true on Intel but is it false on AArch64?)
3856 // region 6-11 is even aligned; it may be padded out more so that
3857 // the region from SP to FP meets the minimum stack alignment.
3858 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3859 // alignment. Region 11, pad1, may be dynamically extended so that
3860 // SP meets the minimum alignment.
3861
3862 frame %{
3863 // These three registers define part of the calling convention
3864 // between compiled code and the interpreter.
3865
3866 // Inline Cache Register or Method for I2C.
3867 inline_cache_reg(R12);
3868
3869 // Number of stack slots consumed by locking an object
3870 sync_stack_slots(2);
3871
3872 // Compiled code's Frame Pointer
3873 frame_pointer(R31);
3874
3875 // Interpreter stores its frame pointer in a register which is
3876 // stored to the stack by I2CAdaptors.
3877 // I2CAdaptors convert from interpreted java to compiled java.
3878 interpreter_frame_pointer(R29);
3879
3880 // Stack alignment requirement
3881 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3882
3883 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3884 // for calls to C. Supports the var-args backing area for register parms.
3885 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3886
3887 // The after-PROLOG location of the return address. Location of
3888 // return address specifies a type (REG or STACK) and a number
3889 // representing the register number (i.e. - use a register name) or
3890 // stack slot.
3891 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3892 // Otherwise, it is above the locks and verification slot and alignment word
3893 // TODO this may well be correct but need to check why that - 2 is there
3894 // ppc port uses 0 but we definitely need to allow for fixed_slots
3895 // which folds in the space used for monitors
3896 return_addr(STACK - 2 +
3897 align_up((Compile::current()->in_preserve_stack_slots() +
3898 Compile::current()->fixed_slots()),
3899 stack_alignment_in_slots()));
3900
3901 // Location of compiled Java return values. Same as C for now.
3902 return_value
3903 %{
3904 // TODO do we allow ideal_reg == Op_RegN???
3905 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3906 "only return normal values");
3907
3908 static const int lo[Op_RegL + 1] = { // enum name
3909 0, // Op_Node
3910 0, // Op_Set
3911 R0_num, // Op_RegN
3912 R0_num, // Op_RegI
3913 R0_num, // Op_RegP
3914 V0_num, // Op_RegF
3915 V0_num, // Op_RegD
3916 R0_num // Op_RegL
3917 };
3918
3919 static const int hi[Op_RegL + 1] = { // enum name
3920 0, // Op_Node
3921 0, // Op_Set
3922 OptoReg::Bad, // Op_RegN
3923 OptoReg::Bad, // Op_RegI
3924 R0_H_num, // Op_RegP
3925 OptoReg::Bad, // Op_RegF
3926 V0_H_num, // Op_RegD
3927 R0_H_num // Op_RegL
3928 };
3929
3930 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3931 %}
3932 %}
3933
3934 //----------ATTRIBUTES---------------------------------------------------------
3935 //----------Operand Attributes-------------------------------------------------
3936 op_attrib op_cost(1); // Required cost attribute
3937
3938 //----------Instruction Attributes---------------------------------------------
3939 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3940 ins_attrib ins_size(32); // Required size attribute (in bits)
3941 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3942 // a non-matching short branch variant
3943 // of some long branch?
3944 ins_attrib ins_alignment(4); // Required alignment attribute (must
3945 // be a power of 2) specifies the
3946 // alignment that some part of the
3947 // instruction (not necessarily the
3948 // start) requires. If > 1, a
3949 // compute_padding() function must be
3950 // provided for the instruction
3951
3952 // Whether this node is expanded during code emission into a sequence of
3953 // instructions and the first instruction can perform an implicit null check.
3954 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3955
3956 //----------OPERANDS-----------------------------------------------------------
3957 // Operand definitions must precede instruction definitions for correct parsing
3958 // in the ADLC because operands constitute user defined types which are used in
3959 // instruction definitions.
3960
3961 //----------Simple Operands----------------------------------------------------
3962
3963 // Integer operands 32 bit
3964 // 32 bit immediate
3965 operand immI()
3966 %{
3967 match(ConI);
3968
3969 op_cost(0);
3970 format %{ %}
3971 interface(CONST_INTER);
3972 %}
3973
3974 // 32 bit zero
3975 operand immI0()
3976 %{
3977 predicate(n->get_int() == 0);
3978 match(ConI);
3979
3980 op_cost(0);
3981 format %{ %}
3982 interface(CONST_INTER);
3983 %}
3984
3985 // 32 bit unit increment
3986 operand immI_1()
3987 %{
3988 predicate(n->get_int() == 1);
3989 match(ConI);
3990
3991 op_cost(0);
3992 format %{ %}
3993 interface(CONST_INTER);
3994 %}
3995
3996 // 32 bit unit decrement
3997 operand immI_M1()
3998 %{
3999 predicate(n->get_int() == -1);
4000 match(ConI);
4001
4002 op_cost(0);
4003 format %{ %}
4004 interface(CONST_INTER);
4005 %}
4006
4007 // Shift values for add/sub extension shift
4008 operand immIExt()
4009 %{
4010 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4011 match(ConI);
4012
4013 op_cost(0);
4014 format %{ %}
4015 interface(CONST_INTER);
4016 %}
4017
4018 operand immI_gt_1()
4019 %{
4020 predicate(n->get_int() > 1);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 operand immI_le_4()
4029 %{
4030 predicate(n->get_int() <= 4);
4031 match(ConI);
4032
4033 op_cost(0);
4034 format %{ %}
4035 interface(CONST_INTER);
4036 %}
4037
4038 operand immI_16()
4039 %{
4040 predicate(n->get_int() == 16);
4041 match(ConI);
4042
4043 op_cost(0);
4044 format %{ %}
4045 interface(CONST_INTER);
4046 %}
4047
4048 operand immI_24()
4049 %{
4050 predicate(n->get_int() == 24);
4051 match(ConI);
4052
4053 op_cost(0);
4054 format %{ %}
4055 interface(CONST_INTER);
4056 %}
4057
4058 operand immI_32()
4059 %{
4060 predicate(n->get_int() == 32);
4061 match(ConI);
4062
4063 op_cost(0);
4064 format %{ %}
4065 interface(CONST_INTER);
4066 %}
4067
4068 operand immI_48()
4069 %{
4070 predicate(n->get_int() == 48);
4071 match(ConI);
4072
4073 op_cost(0);
4074 format %{ %}
4075 interface(CONST_INTER);
4076 %}
4077
4078 operand immI_56()
4079 %{
4080 predicate(n->get_int() == 56);
4081 match(ConI);
4082
4083 op_cost(0);
4084 format %{ %}
4085 interface(CONST_INTER);
4086 %}
4087
4088 operand immI_255()
4089 %{
4090 predicate(n->get_int() == 255);
4091 match(ConI);
4092
4093 op_cost(0);
4094 format %{ %}
4095 interface(CONST_INTER);
4096 %}
4097
4098 operand immI_65535()
4099 %{
4100 predicate(n->get_int() == 65535);
4101 match(ConI);
4102
4103 op_cost(0);
4104 format %{ %}
4105 interface(CONST_INTER);
4106 %}
4107
4108 operand immI_positive()
4109 %{
4110 predicate(n->get_int() > 0);
4111 match(ConI);
4112
4113 op_cost(0);
4114 format %{ %}
4115 interface(CONST_INTER);
4116 %}
4117
4118 // BoolTest condition for signed compare
4119 operand immI_cmp_cond()
4120 %{
4121 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4122 match(ConI);
4123
4124 op_cost(0);
4125 format %{ %}
4126 interface(CONST_INTER);
4127 %}
4128
4129 // BoolTest condition for unsigned compare
4130 operand immI_cmpU_cond()
4131 %{
4132 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4133 match(ConI);
4134
4135 op_cost(0);
4136 format %{ %}
4137 interface(CONST_INTER);
4138 %}
4139
4140 operand immL_255()
4141 %{
4142 predicate(n->get_long() == 255L);
4143 match(ConL);
4144
4145 op_cost(0);
4146 format %{ %}
4147 interface(CONST_INTER);
4148 %}
4149
4150 operand immL_65535()
4151 %{
4152 predicate(n->get_long() == 65535L);
4153 match(ConL);
4154
4155 op_cost(0);
4156 format %{ %}
4157 interface(CONST_INTER);
4158 %}
4159
4160 operand immL_4294967295()
4161 %{
4162 predicate(n->get_long() == 4294967295L);
4163 match(ConL);
4164
4165 op_cost(0);
4166 format %{ %}
4167 interface(CONST_INTER);
4168 %}
4169
4170 operand immL_bitmask()
4171 %{
4172 predicate((n->get_long() != 0)
4173 && ((n->get_long() & 0xc000000000000000l) == 0)
4174 && is_power_of_2(n->get_long() + 1));
4175 match(ConL);
4176
4177 op_cost(0);
4178 format %{ %}
4179 interface(CONST_INTER);
4180 %}
4181
4182 operand immI_bitmask()
4183 %{
4184 predicate((n->get_int() != 0)
4185 && ((n->get_int() & 0xc0000000) == 0)
4186 && is_power_of_2(n->get_int() + 1));
4187 match(ConI);
4188
4189 op_cost(0);
4190 format %{ %}
4191 interface(CONST_INTER);
4192 %}
4193
4194 operand immL_positive_bitmaskI()
4195 %{
4196 predicate((n->get_long() != 0)
4197 && ((julong)n->get_long() < 0x80000000ULL)
4198 && is_power_of_2(n->get_long() + 1));
4199 match(ConL);
4200
4201 op_cost(0);
4202 format %{ %}
4203 interface(CONST_INTER);
4204 %}
4205
4206 // Scale values for scaled offset addressing modes (up to long but not quad)
4207 operand immIScale()
4208 %{
4209 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4210 match(ConI);
4211
4212 op_cost(0);
4213 format %{ %}
4214 interface(CONST_INTER);
4215 %}
4216
4217 // 5 bit signed integer
4218 operand immI5()
4219 %{
4220 predicate(Assembler::is_simm(n->get_int(), 5));
4221 match(ConI);
4222
4223 op_cost(0);
4224 format %{ %}
4225 interface(CONST_INTER);
4226 %}
4227
4228 // 7 bit unsigned integer
4229 operand immIU7()
4230 %{
4231 predicate(Assembler::is_uimm(n->get_int(), 7));
4232 match(ConI);
4233
4234 op_cost(0);
4235 format %{ %}
4236 interface(CONST_INTER);
4237 %}
4238
4239 // Offset for scaled or unscaled immediate loads and stores
4240 operand immIOffset()
4241 %{
4242 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4243 match(ConI);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immIOffset1()
4251 %{
4252 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4253 match(ConI);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immIOffset2()
4261 %{
4262 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4263 match(ConI);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immIOffset4()
4271 %{
4272 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4273 match(ConI);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immIOffset8()
4281 %{
4282 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4283 match(ConI);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 operand immIOffset16()
4291 %{
4292 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4293 match(ConI);
4294
4295 op_cost(0);
4296 format %{ %}
4297 interface(CONST_INTER);
4298 %}
4299
4300 operand immLOffset()
4301 %{
4302 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4303 match(ConL);
4304
4305 op_cost(0);
4306 format %{ %}
4307 interface(CONST_INTER);
4308 %}
4309
4310 operand immLoffset1()
4311 %{
4312 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4313 match(ConL);
4314
4315 op_cost(0);
4316 format %{ %}
4317 interface(CONST_INTER);
4318 %}
4319
4320 operand immLoffset2()
4321 %{
4322 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4323 match(ConL);
4324
4325 op_cost(0);
4326 format %{ %}
4327 interface(CONST_INTER);
4328 %}
4329
4330 operand immLoffset4()
4331 %{
4332 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4333 match(ConL);
4334
4335 op_cost(0);
4336 format %{ %}
4337 interface(CONST_INTER);
4338 %}
4339
4340 operand immLoffset8()
4341 %{
4342 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4343 match(ConL);
4344
4345 op_cost(0);
4346 format %{ %}
4347 interface(CONST_INTER);
4348 %}
4349
4350 operand immLoffset16()
4351 %{
4352 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4353 match(ConL);
4354
4355 op_cost(0);
4356 format %{ %}
4357 interface(CONST_INTER);
4358 %}
4359
4360 // 5 bit signed long integer
4361 operand immL5()
4362 %{
4363 predicate(Assembler::is_simm(n->get_long(), 5));
4364 match(ConL);
4365
4366 op_cost(0);
4367 format %{ %}
4368 interface(CONST_INTER);
4369 %}
4370
4371 // 7 bit unsigned long integer
4372 operand immLU7()
4373 %{
4374 predicate(Assembler::is_uimm(n->get_long(), 7));
4375 match(ConL);
4376
4377 op_cost(0);
4378 format %{ %}
4379 interface(CONST_INTER);
4380 %}
4381
4382 // 8 bit signed value.
4383 operand immI8()
4384 %{
4385 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4386 match(ConI);
4387
4388 op_cost(0);
4389 format %{ %}
4390 interface(CONST_INTER);
4391 %}
4392
4393 // 8 bit signed value (simm8), or #simm8 LSL 8.
4394 operand immIDupV()
4395 %{
4396 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4397 match(ConI);
4398
4399 op_cost(0);
4400 format %{ %}
4401 interface(CONST_INTER);
4402 %}
4403
4404 // 8 bit signed value (simm8), or #simm8 LSL 8.
4405 operand immLDupV()
4406 %{
4407 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4408 match(ConL);
4409
4410 op_cost(0);
4411 format %{ %}
4412 interface(CONST_INTER);
4413 %}
4414
4415 // 8 bit signed value (simm8), or #simm8 LSL 8.
4416 operand immHDupV()
4417 %{
4418 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4419 match(ConH);
4420
4421 op_cost(0);
4422 format %{ %}
4423 interface(CONST_INTER);
4424 %}
4425
4426 // 8 bit integer valid for vector add sub immediate
4427 operand immBAddSubV()
4428 %{
4429 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4430 match(ConI);
4431
4432 op_cost(0);
4433 format %{ %}
4434 interface(CONST_INTER);
4435 %}
4436
4437 // 32 bit integer valid for add sub immediate
4438 operand immIAddSub()
4439 %{
4440 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4441 match(ConI);
4442 op_cost(0);
4443 format %{ %}
4444 interface(CONST_INTER);
4445 %}
4446
4447 // 32 bit integer valid for vector add sub immediate
4448 operand immIAddSubV()
4449 %{
4450 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4451 match(ConI);
4452
4453 op_cost(0);
4454 format %{ %}
4455 interface(CONST_INTER);
4456 %}
4457
4458 // 32 bit unsigned integer valid for logical immediate
4459
4460 operand immBLog()
4461 %{
4462 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4463 match(ConI);
4464
4465 op_cost(0);
4466 format %{ %}
4467 interface(CONST_INTER);
4468 %}
4469
4470 operand immSLog()
4471 %{
4472 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4473 match(ConI);
4474
4475 op_cost(0);
4476 format %{ %}
4477 interface(CONST_INTER);
4478 %}
4479
4480 operand immILog()
4481 %{
4482 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4483 match(ConI);
4484
4485 op_cost(0);
4486 format %{ %}
4487 interface(CONST_INTER);
4488 %}
4489
4490 // Integer operands 64 bit
4491 // 64 bit immediate
4492 operand immL()
4493 %{
4494 match(ConL);
4495
4496 op_cost(0);
4497 format %{ %}
4498 interface(CONST_INTER);
4499 %}
4500
4501 // 64 bit zero
4502 operand immL0()
4503 %{
4504 predicate(n->get_long() == 0);
4505 match(ConL);
4506
4507 op_cost(0);
4508 format %{ %}
4509 interface(CONST_INTER);
4510 %}
4511
4512 // 64 bit unit decrement
4513 operand immL_M1()
4514 %{
4515 predicate(n->get_long() == -1);
4516 match(ConL);
4517
4518 op_cost(0);
4519 format %{ %}
4520 interface(CONST_INTER);
4521 %}
4522
4523 // 64 bit integer valid for add sub immediate
4524 operand immLAddSub()
4525 %{
4526 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4527 match(ConL);
4528 op_cost(0);
4529 format %{ %}
4530 interface(CONST_INTER);
4531 %}
4532
4533 // 64 bit integer valid for addv subv immediate
4534 operand immLAddSubV()
4535 %{
4536 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4537 match(ConL);
4538
4539 op_cost(0);
4540 format %{ %}
4541 interface(CONST_INTER);
4542 %}
4543
4544 // 64 bit integer valid for logical immediate
4545 operand immLLog()
4546 %{
4547 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4548 match(ConL);
4549 op_cost(0);
4550 format %{ %}
4551 interface(CONST_INTER);
4552 %}
4553
4554 // Long Immediate: low 32-bit mask
4555 operand immL_32bits()
4556 %{
4557 predicate(n->get_long() == 0xFFFFFFFFL);
4558 match(ConL);
4559 op_cost(0);
4560 format %{ %}
4561 interface(CONST_INTER);
4562 %}
4563
4564 // Pointer operands
4565 // Pointer Immediate
4566 operand immP()
4567 %{
4568 match(ConP);
4569
4570 op_cost(0);
4571 format %{ %}
4572 interface(CONST_INTER);
4573 %}
4574
4575 // nullptr Pointer Immediate
4576 operand immP0()
4577 %{
4578 predicate(n->get_ptr() == 0);
4579 match(ConP);
4580
4581 op_cost(0);
4582 format %{ %}
4583 interface(CONST_INTER);
4584 %}
4585
4586 // Pointer Immediate One
4587 // this is used in object initialization (initial object header)
4588 operand immP_1()
4589 %{
4590 predicate(n->get_ptr() == 1);
4591 match(ConP);
4592
4593 op_cost(0);
4594 format %{ %}
4595 interface(CONST_INTER);
4596 %}
4597
4598 // Float and Double operands
4599 // Double Immediate
4600 operand immD()
4601 %{
4602 match(ConD);
4603 op_cost(0);
4604 format %{ %}
4605 interface(CONST_INTER);
4606 %}
4607
4608 // Double Immediate: +0.0d
4609 operand immD0()
4610 %{
4611 predicate(jlong_cast(n->getd()) == 0);
4612 match(ConD);
4613
4614 op_cost(0);
4615 format %{ %}
4616 interface(CONST_INTER);
4617 %}
4618
4619 // constant 'double +0.0'.
4620 operand immDPacked()
4621 %{
4622 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4623 match(ConD);
4624 op_cost(0);
4625 format %{ %}
4626 interface(CONST_INTER);
4627 %}
4628
4629 // Float Immediate
4630 operand immF()
4631 %{
4632 match(ConF);
4633 op_cost(0);
4634 format %{ %}
4635 interface(CONST_INTER);
4636 %}
4637
4638 // Float Immediate: +0.0f.
4639 operand immF0()
4640 %{
4641 predicate(jint_cast(n->getf()) == 0);
4642 match(ConF);
4643
4644 op_cost(0);
4645 format %{ %}
4646 interface(CONST_INTER);
4647 %}
4648
4649 // Half Float (FP16) Immediate
4650 operand immH()
4651 %{
4652 match(ConH);
4653 op_cost(0);
4654 format %{ %}
4655 interface(CONST_INTER);
4656 %}
4657
4658 //
4659 operand immFPacked()
4660 %{
4661 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4662 match(ConF);
4663 op_cost(0);
4664 format %{ %}
4665 interface(CONST_INTER);
4666 %}
4667
4668 // Narrow pointer operands
4669 // Narrow Pointer Immediate
4670 operand immN()
4671 %{
4672 match(ConN);
4673
4674 op_cost(0);
4675 format %{ %}
4676 interface(CONST_INTER);
4677 %}
4678
4679 // Narrow nullptr Pointer Immediate
4680 operand immN0()
4681 %{
4682 predicate(n->get_narrowcon() == 0);
4683 match(ConN);
4684
4685 op_cost(0);
4686 format %{ %}
4687 interface(CONST_INTER);
4688 %}
4689
4690 operand immNKlass()
4691 %{
4692 match(ConNKlass);
4693
4694 op_cost(0);
4695 format %{ %}
4696 interface(CONST_INTER);
4697 %}
4698
4699 // Integer 32 bit Register Operands
4700 // Integer 32 bitRegister (excludes SP)
4701 operand iRegI()
4702 %{
4703 constraint(ALLOC_IN_RC(any_reg32));
4704 match(RegI);
4705 match(iRegINoSp);
4706 op_cost(0);
4707 format %{ %}
4708 interface(REG_INTER);
4709 %}
4710
4711 // Integer 32 bit Register not Special
4712 operand iRegINoSp()
4713 %{
4714 constraint(ALLOC_IN_RC(no_special_reg32));
4715 match(RegI);
4716 op_cost(0);
4717 format %{ %}
4718 interface(REG_INTER);
4719 %}
4720
4721 // Integer 64 bit Register Operands
4722 // Integer 64 bit Register (includes SP)
4723 operand iRegL()
4724 %{
4725 constraint(ALLOC_IN_RC(any_reg));
4726 match(RegL);
4727 match(iRegLNoSp);
4728 op_cost(0);
4729 format %{ %}
4730 interface(REG_INTER);
4731 %}
4732
4733 // Integer 64 bit Register not Special
4734 operand iRegLNoSp()
4735 %{
4736 constraint(ALLOC_IN_RC(no_special_reg));
4737 match(RegL);
4738 match(iRegL_R0);
4739 format %{ %}
4740 interface(REG_INTER);
4741 %}
4742
4743 // Pointer Register Operands
4744 // Pointer Register
4745 operand iRegP()
4746 %{
4747 constraint(ALLOC_IN_RC(ptr_reg));
4748 match(RegP);
4749 match(iRegPNoSp);
4750 match(iRegP_R0);
4751 //match(iRegP_R2);
4752 //match(iRegP_R4);
4753 match(iRegP_R5);
4754 match(thread_RegP);
4755 op_cost(0);
4756 format %{ %}
4757 interface(REG_INTER);
4758 %}
4759
4760 // Pointer 64 bit Register not Special
4761 operand iRegPNoSp()
4762 %{
4763 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4764 match(RegP);
4765 // match(iRegP);
4766 // match(iRegP_R0);
4767 // match(iRegP_R2);
4768 // match(iRegP_R4);
4769 // match(iRegP_R5);
4770 // match(thread_RegP);
4771 op_cost(0);
4772 format %{ %}
4773 interface(REG_INTER);
4774 %}
4775
4776 // This operand is not allowed to use rfp even if
4777 // rfp is not used to hold the frame pointer.
4778 operand iRegPNoSpNoRfp()
4779 %{
4780 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4781 match(RegP);
4782 match(iRegPNoSp);
4783 op_cost(0);
4784 format %{ %}
4785 interface(REG_INTER);
4786 %}
4787
4788 // Pointer 64 bit Register R0 only
4789 operand iRegP_R0()
4790 %{
4791 constraint(ALLOC_IN_RC(r0_reg));
4792 match(RegP);
4793 // match(iRegP);
4794 match(iRegPNoSp);
4795 op_cost(0);
4796 format %{ %}
4797 interface(REG_INTER);
4798 %}
4799
4800 // Pointer 64 bit Register R1 only
4801 operand iRegP_R1()
4802 %{
4803 constraint(ALLOC_IN_RC(r1_reg));
4804 match(RegP);
4805 // match(iRegP);
4806 match(iRegPNoSp);
4807 op_cost(0);
4808 format %{ %}
4809 interface(REG_INTER);
4810 %}
4811
4812 // Pointer 64 bit Register R2 only
4813 operand iRegP_R2()
4814 %{
4815 constraint(ALLOC_IN_RC(r2_reg));
4816 match(RegP);
4817 // match(iRegP);
4818 match(iRegPNoSp);
4819 op_cost(0);
4820 format %{ %}
4821 interface(REG_INTER);
4822 %}
4823
4824 // Pointer 64 bit Register R3 only
4825 operand iRegP_R3()
4826 %{
4827 constraint(ALLOC_IN_RC(r3_reg));
4828 match(RegP);
4829 // match(iRegP);
4830 match(iRegPNoSp);
4831 op_cost(0);
4832 format %{ %}
4833 interface(REG_INTER);
4834 %}
4835
4836 // Pointer 64 bit Register R4 only
4837 operand iRegP_R4()
4838 %{
4839 constraint(ALLOC_IN_RC(r4_reg));
4840 match(RegP);
4841 // match(iRegP);
4842 match(iRegPNoSp);
4843 op_cost(0);
4844 format %{ %}
4845 interface(REG_INTER);
4846 %}
4847
4848 // Pointer 64 bit Register R5 only
4849 operand iRegP_R5()
4850 %{
4851 constraint(ALLOC_IN_RC(r5_reg));
4852 match(RegP);
4853 // match(iRegP);
4854 match(iRegPNoSp);
4855 op_cost(0);
4856 format %{ %}
4857 interface(REG_INTER);
4858 %}
4859
4860 // Pointer 64 bit Register R10 only
4861 operand iRegP_R10()
4862 %{
4863 constraint(ALLOC_IN_RC(r10_reg));
4864 match(RegP);
4865 // match(iRegP);
4866 match(iRegPNoSp);
4867 op_cost(0);
4868 format %{ %}
4869 interface(REG_INTER);
4870 %}
4871
4872 // Long 64 bit Register R0 only
4873 operand iRegL_R0()
4874 %{
4875 constraint(ALLOC_IN_RC(r0_reg));
4876 match(RegL);
4877 match(iRegLNoSp);
4878 op_cost(0);
4879 format %{ %}
4880 interface(REG_INTER);
4881 %}
4882
4883 // Long 64 bit Register R11 only
4884 operand iRegL_R11()
4885 %{
4886 constraint(ALLOC_IN_RC(r11_reg));
4887 match(RegL);
4888 match(iRegLNoSp);
4889 op_cost(0);
4890 format %{ %}
4891 interface(REG_INTER);
4892 %}
4893
4894 // Register R0 only
4895 operand iRegI_R0()
4896 %{
4897 constraint(ALLOC_IN_RC(int_r0_reg));
4898 match(RegI);
4899 match(iRegINoSp);
4900 op_cost(0);
4901 format %{ %}
4902 interface(REG_INTER);
4903 %}
4904
4905 // Register R2 only
4906 operand iRegI_R2()
4907 %{
4908 constraint(ALLOC_IN_RC(int_r2_reg));
4909 match(RegI);
4910 match(iRegINoSp);
4911 op_cost(0);
4912 format %{ %}
4913 interface(REG_INTER);
4914 %}
4915
4916 // Register R3 only
4917 operand iRegI_R3()
4918 %{
4919 constraint(ALLOC_IN_RC(int_r3_reg));
4920 match(RegI);
4921 match(iRegINoSp);
4922 op_cost(0);
4923 format %{ %}
4924 interface(REG_INTER);
4925 %}
4926
4927
4928 // Register R4 only
4929 operand iRegI_R4()
4930 %{
4931 constraint(ALLOC_IN_RC(int_r4_reg));
4932 match(RegI);
4933 match(iRegINoSp);
4934 op_cost(0);
4935 format %{ %}
4936 interface(REG_INTER);
4937 %}
4938
4939
4940 // Pointer Register Operands
4941 // Narrow Pointer Register
4942 operand iRegN()
4943 %{
4944 constraint(ALLOC_IN_RC(any_reg32));
4945 match(RegN);
4946 match(iRegNNoSp);
4947 op_cost(0);
4948 format %{ %}
4949 interface(REG_INTER);
4950 %}
4951
4952 // Integer 64 bit Register not Special
4953 operand iRegNNoSp()
4954 %{
4955 constraint(ALLOC_IN_RC(no_special_reg32));
4956 match(RegN);
4957 op_cost(0);
4958 format %{ %}
4959 interface(REG_INTER);
4960 %}
4961
4962 // Float Register
4963 // Float register operands
4964 operand vRegF()
4965 %{
4966 constraint(ALLOC_IN_RC(float_reg));
4967 match(RegF);
4968
4969 op_cost(0);
4970 format %{ %}
4971 interface(REG_INTER);
4972 %}
4973
4974 // Double Register
4975 // Double register operands
4976 operand vRegD()
4977 %{
4978 constraint(ALLOC_IN_RC(double_reg));
4979 match(RegD);
4980
4981 op_cost(0);
4982 format %{ %}
4983 interface(REG_INTER);
4984 %}
4985
4986 // Generic vector class. This will be used for
4987 // all vector operands, including NEON and SVE.
4988 operand vReg()
4989 %{
4990 constraint(ALLOC_IN_RC(dynamic));
4991 match(VecA);
4992 match(VecD);
4993 match(VecX);
4994
4995 op_cost(0);
4996 format %{ %}
4997 interface(REG_INTER);
4998 %}
4999
5000 operand vReg_V10()
5001 %{
5002 constraint(ALLOC_IN_RC(v10_veca_reg));
5003 match(vReg);
5004
5005 op_cost(0);
5006 format %{ %}
5007 interface(REG_INTER);
5008 %}
5009
5010 operand vReg_V11()
5011 %{
5012 constraint(ALLOC_IN_RC(v11_veca_reg));
5013 match(vReg);
5014
5015 op_cost(0);
5016 format %{ %}
5017 interface(REG_INTER);
5018 %}
5019
5020 operand vReg_V12()
5021 %{
5022 constraint(ALLOC_IN_RC(v12_veca_reg));
5023 match(vReg);
5024
5025 op_cost(0);
5026 format %{ %}
5027 interface(REG_INTER);
5028 %}
5029
5030 operand vReg_V13()
5031 %{
5032 constraint(ALLOC_IN_RC(v13_veca_reg));
5033 match(vReg);
5034
5035 op_cost(0);
5036 format %{ %}
5037 interface(REG_INTER);
5038 %}
5039
5040 operand vReg_V17()
5041 %{
5042 constraint(ALLOC_IN_RC(v17_veca_reg));
5043 match(vReg);
5044
5045 op_cost(0);
5046 format %{ %}
5047 interface(REG_INTER);
5048 %}
5049
5050 operand vReg_V18()
5051 %{
5052 constraint(ALLOC_IN_RC(v18_veca_reg));
5053 match(vReg);
5054
5055 op_cost(0);
5056 format %{ %}
5057 interface(REG_INTER);
5058 %}
5059
5060 operand vReg_V23()
5061 %{
5062 constraint(ALLOC_IN_RC(v23_veca_reg));
5063 match(vReg);
5064
5065 op_cost(0);
5066 format %{ %}
5067 interface(REG_INTER);
5068 %}
5069
5070 operand vReg_V24()
5071 %{
5072 constraint(ALLOC_IN_RC(v24_veca_reg));
5073 match(vReg);
5074
5075 op_cost(0);
5076 format %{ %}
5077 interface(REG_INTER);
5078 %}
5079
5080 operand vecA()
5081 %{
5082 constraint(ALLOC_IN_RC(vectora_reg));
5083 match(VecA);
5084
5085 op_cost(0);
5086 format %{ %}
5087 interface(REG_INTER);
5088 %}
5089
5090 operand vecD()
5091 %{
5092 constraint(ALLOC_IN_RC(vectord_reg));
5093 match(VecD);
5094
5095 op_cost(0);
5096 format %{ %}
5097 interface(REG_INTER);
5098 %}
5099
5100 operand vecX()
5101 %{
5102 constraint(ALLOC_IN_RC(vectorx_reg));
5103 match(VecX);
5104
5105 op_cost(0);
5106 format %{ %}
5107 interface(REG_INTER);
5108 %}
5109
5110 operand vRegD_V0()
5111 %{
5112 constraint(ALLOC_IN_RC(v0_reg));
5113 match(RegD);
5114 op_cost(0);
5115 format %{ %}
5116 interface(REG_INTER);
5117 %}
5118
5119 operand vRegD_V1()
5120 %{
5121 constraint(ALLOC_IN_RC(v1_reg));
5122 match(RegD);
5123 op_cost(0);
5124 format %{ %}
5125 interface(REG_INTER);
5126 %}
5127
5128 operand vRegD_V2()
5129 %{
5130 constraint(ALLOC_IN_RC(v2_reg));
5131 match(RegD);
5132 op_cost(0);
5133 format %{ %}
5134 interface(REG_INTER);
5135 %}
5136
5137 operand vRegD_V3()
5138 %{
5139 constraint(ALLOC_IN_RC(v3_reg));
5140 match(RegD);
5141 op_cost(0);
5142 format %{ %}
5143 interface(REG_INTER);
5144 %}
5145
5146 operand vRegD_V4()
5147 %{
5148 constraint(ALLOC_IN_RC(v4_reg));
5149 match(RegD);
5150 op_cost(0);
5151 format %{ %}
5152 interface(REG_INTER);
5153 %}
5154
5155 operand vRegD_V5()
5156 %{
5157 constraint(ALLOC_IN_RC(v5_reg));
5158 match(RegD);
5159 op_cost(0);
5160 format %{ %}
5161 interface(REG_INTER);
5162 %}
5163
5164 operand vRegD_V6()
5165 %{
5166 constraint(ALLOC_IN_RC(v6_reg));
5167 match(RegD);
5168 op_cost(0);
5169 format %{ %}
5170 interface(REG_INTER);
5171 %}
5172
5173 operand vRegD_V7()
5174 %{
5175 constraint(ALLOC_IN_RC(v7_reg));
5176 match(RegD);
5177 op_cost(0);
5178 format %{ %}
5179 interface(REG_INTER);
5180 %}
5181
5182 operand vRegD_V12()
5183 %{
5184 constraint(ALLOC_IN_RC(v12_reg));
5185 match(RegD);
5186 op_cost(0);
5187 format %{ %}
5188 interface(REG_INTER);
5189 %}
5190
5191 operand vRegD_V13()
5192 %{
5193 constraint(ALLOC_IN_RC(v13_reg));
5194 match(RegD);
5195 op_cost(0);
5196 format %{ %}
5197 interface(REG_INTER);
5198 %}
5199
5200 operand pReg()
5201 %{
5202 constraint(ALLOC_IN_RC(pr_reg));
5203 match(RegVectMask);
5204 match(pRegGov);
5205 op_cost(0);
5206 format %{ %}
5207 interface(REG_INTER);
5208 %}
5209
5210 operand pRegGov()
5211 %{
5212 constraint(ALLOC_IN_RC(gov_pr));
5213 match(RegVectMask);
5214 match(pReg);
5215 op_cost(0);
5216 format %{ %}
5217 interface(REG_INTER);
5218 %}
5219
5220 operand pRegGov_P0()
5221 %{
5222 constraint(ALLOC_IN_RC(p0_reg));
5223 match(RegVectMask);
5224 op_cost(0);
5225 format %{ %}
5226 interface(REG_INTER);
5227 %}
5228
5229 operand pRegGov_P1()
5230 %{
5231 constraint(ALLOC_IN_RC(p1_reg));
5232 match(RegVectMask);
5233 op_cost(0);
5234 format %{ %}
5235 interface(REG_INTER);
5236 %}
5237
5238 // Flags register, used as output of signed compare instructions
5239
5240 // note that on AArch64 we also use this register as the output for
5241 // for floating point compare instructions (CmpF CmpD). this ensures
5242 // that ordered inequality tests use GT, GE, LT or LE none of which
5243 // pass through cases where the result is unordered i.e. one or both
5244 // inputs to the compare is a NaN. this means that the ideal code can
5245 // replace e.g. a GT with an LE and not end up capturing the NaN case
5246 // (where the comparison should always fail). EQ and NE tests are
5247 // always generated in ideal code so that unordered folds into the NE
5248 // case, matching the behaviour of AArch64 NE.
5249 //
5250 // This differs from x86 where the outputs of FP compares use a
5251 // special FP flags registers and where compares based on this
5252 // register are distinguished into ordered inequalities (cmpOpUCF) and
5253 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5254 // to explicitly handle the unordered case in branches. x86 also has
5255 // to include extra CMoveX rules to accept a cmpOpUCF input.
5256
5257 operand rFlagsReg()
5258 %{
5259 constraint(ALLOC_IN_RC(int_flags));
5260 match(RegFlags);
5261
5262 op_cost(0);
5263 format %{ "RFLAGS" %}
5264 interface(REG_INTER);
5265 %}
5266
5267 // Flags register, used as output of unsigned compare instructions
5268 operand rFlagsRegU()
5269 %{
5270 constraint(ALLOC_IN_RC(int_flags));
5271 match(RegFlags);
5272
5273 op_cost(0);
5274 format %{ "RFLAGSU" %}
5275 interface(REG_INTER);
5276 %}
5277
5278 // Special Registers
5279
5280 // Method Register
5281 operand inline_cache_RegP(iRegP reg)
5282 %{
5283 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5284 match(reg);
5285 match(iRegPNoSp);
5286 op_cost(0);
5287 format %{ %}
5288 interface(REG_INTER);
5289 %}
5290
5291 // Thread Register
5292 operand thread_RegP(iRegP reg)
5293 %{
5294 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5295 match(reg);
5296 op_cost(0);
5297 format %{ %}
5298 interface(REG_INTER);
5299 %}
5300
5301 //----------Memory Operands----------------------------------------------------
5302
5303 operand indirect(iRegP reg)
5304 %{
5305 constraint(ALLOC_IN_RC(ptr_reg));
5306 match(reg);
5307 op_cost(0);
5308 format %{ "[$reg]" %}
5309 interface(MEMORY_INTER) %{
5310 base($reg);
5311 index(0xffffffff);
5312 scale(0x0);
5313 disp(0x0);
5314 %}
5315 %}
5316
5317 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5318 %{
5319 constraint(ALLOC_IN_RC(ptr_reg));
5320 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5321 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5322 op_cost(0);
5323 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5324 interface(MEMORY_INTER) %{
5325 base($reg);
5326 index($ireg);
5327 scale($scale);
5328 disp(0x0);
5329 %}
5330 %}
5331
5332 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5333 %{
5334 constraint(ALLOC_IN_RC(ptr_reg));
5335 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5336 match(AddP reg (LShiftL lreg scale));
5337 op_cost(0);
5338 format %{ "$reg, $lreg lsl($scale)" %}
5339 interface(MEMORY_INTER) %{
5340 base($reg);
5341 index($lreg);
5342 scale($scale);
5343 disp(0x0);
5344 %}
5345 %}
5346
5347 operand indIndexI2L(iRegP reg, iRegI ireg)
5348 %{
5349 constraint(ALLOC_IN_RC(ptr_reg));
5350 match(AddP reg (ConvI2L ireg));
5351 op_cost(0);
5352 format %{ "$reg, $ireg, 0, I2L" %}
5353 interface(MEMORY_INTER) %{
5354 base($reg);
5355 index($ireg);
5356 scale(0x0);
5357 disp(0x0);
5358 %}
5359 %}
5360
5361 operand indIndex(iRegP reg, iRegL lreg)
5362 %{
5363 constraint(ALLOC_IN_RC(ptr_reg));
5364 match(AddP reg lreg);
5365 op_cost(0);
5366 format %{ "$reg, $lreg" %}
5367 interface(MEMORY_INTER) %{
5368 base($reg);
5369 index($lreg);
5370 scale(0x0);
5371 disp(0x0);
5372 %}
5373 %}
5374
5375 operand indOffI1(iRegP reg, immIOffset1 off)
5376 %{
5377 constraint(ALLOC_IN_RC(ptr_reg));
5378 match(AddP reg off);
5379 op_cost(0);
5380 format %{ "[$reg, $off]" %}
5381 interface(MEMORY_INTER) %{
5382 base($reg);
5383 index(0xffffffff);
5384 scale(0x0);
5385 disp($off);
5386 %}
5387 %}
5388
5389 operand indOffI2(iRegP reg, immIOffset2 off)
5390 %{
5391 constraint(ALLOC_IN_RC(ptr_reg));
5392 match(AddP reg off);
5393 op_cost(0);
5394 format %{ "[$reg, $off]" %}
5395 interface(MEMORY_INTER) %{
5396 base($reg);
5397 index(0xffffffff);
5398 scale(0x0);
5399 disp($off);
5400 %}
5401 %}
5402
5403 operand indOffI4(iRegP reg, immIOffset4 off)
5404 %{
5405 constraint(ALLOC_IN_RC(ptr_reg));
5406 match(AddP reg off);
5407 op_cost(0);
5408 format %{ "[$reg, $off]" %}
5409 interface(MEMORY_INTER) %{
5410 base($reg);
5411 index(0xffffffff);
5412 scale(0x0);
5413 disp($off);
5414 %}
5415 %}
5416
5417 operand indOffI8(iRegP reg, immIOffset8 off)
5418 %{
5419 constraint(ALLOC_IN_RC(ptr_reg));
5420 match(AddP reg off);
5421 op_cost(0);
5422 format %{ "[$reg, $off]" %}
5423 interface(MEMORY_INTER) %{
5424 base($reg);
5425 index(0xffffffff);
5426 scale(0x0);
5427 disp($off);
5428 %}
5429 %}
5430
5431 operand indOffI16(iRegP reg, immIOffset16 off)
5432 %{
5433 constraint(ALLOC_IN_RC(ptr_reg));
5434 match(AddP reg off);
5435 op_cost(0);
5436 format %{ "[$reg, $off]" %}
5437 interface(MEMORY_INTER) %{
5438 base($reg);
5439 index(0xffffffff);
5440 scale(0x0);
5441 disp($off);
5442 %}
5443 %}
5444
5445 operand indOffL1(iRegP reg, immLoffset1 off)
5446 %{
5447 constraint(ALLOC_IN_RC(ptr_reg));
5448 match(AddP reg off);
5449 op_cost(0);
5450 format %{ "[$reg, $off]" %}
5451 interface(MEMORY_INTER) %{
5452 base($reg);
5453 index(0xffffffff);
5454 scale(0x0);
5455 disp($off);
5456 %}
5457 %}
5458
5459 operand indOffL2(iRegP reg, immLoffset2 off)
5460 %{
5461 constraint(ALLOC_IN_RC(ptr_reg));
5462 match(AddP reg off);
5463 op_cost(0);
5464 format %{ "[$reg, $off]" %}
5465 interface(MEMORY_INTER) %{
5466 base($reg);
5467 index(0xffffffff);
5468 scale(0x0);
5469 disp($off);
5470 %}
5471 %}
5472
5473 operand indOffL4(iRegP reg, immLoffset4 off)
5474 %{
5475 constraint(ALLOC_IN_RC(ptr_reg));
5476 match(AddP reg off);
5477 op_cost(0);
5478 format %{ "[$reg, $off]" %}
5479 interface(MEMORY_INTER) %{
5480 base($reg);
5481 index(0xffffffff);
5482 scale(0x0);
5483 disp($off);
5484 %}
5485 %}
5486
5487 operand indOffL8(iRegP reg, immLoffset8 off)
5488 %{
5489 constraint(ALLOC_IN_RC(ptr_reg));
5490 match(AddP reg off);
5491 op_cost(0);
5492 format %{ "[$reg, $off]" %}
5493 interface(MEMORY_INTER) %{
5494 base($reg);
5495 index(0xffffffff);
5496 scale(0x0);
5497 disp($off);
5498 %}
5499 %}
5500
5501 operand indOffL16(iRegP reg, immLoffset16 off)
5502 %{
5503 constraint(ALLOC_IN_RC(ptr_reg));
5504 match(AddP reg off);
5505 op_cost(0);
5506 format %{ "[$reg, $off]" %}
5507 interface(MEMORY_INTER) %{
5508 base($reg);
5509 index(0xffffffff);
5510 scale(0x0);
5511 disp($off);
5512 %}
5513 %}
5514
5515 operand indirectX2P(iRegL reg)
5516 %{
5517 constraint(ALLOC_IN_RC(ptr_reg));
5518 match(CastX2P reg);
5519 op_cost(0);
5520 format %{ "[$reg]\t# long -> ptr" %}
5521 interface(MEMORY_INTER) %{
5522 base($reg);
5523 index(0xffffffff);
5524 scale(0x0);
5525 disp(0x0);
5526 %}
5527 %}
5528
5529 operand indOffX2P(iRegL reg, immLOffset off)
5530 %{
5531 constraint(ALLOC_IN_RC(ptr_reg));
5532 match(AddP (CastX2P reg) off);
5533 op_cost(0);
5534 format %{ "[$reg, $off]\t# long -> ptr" %}
5535 interface(MEMORY_INTER) %{
5536 base($reg);
5537 index(0xffffffff);
5538 scale(0x0);
5539 disp($off);
5540 %}
5541 %}
5542
5543 operand indirectN(iRegN reg)
5544 %{
5545 predicate(CompressedOops::shift() == 0);
5546 constraint(ALLOC_IN_RC(ptr_reg));
5547 match(DecodeN reg);
5548 op_cost(0);
5549 format %{ "[$reg]\t# narrow" %}
5550 interface(MEMORY_INTER) %{
5551 base($reg);
5552 index(0xffffffff);
5553 scale(0x0);
5554 disp(0x0);
5555 %}
5556 %}
5557
5558 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5559 %{
5560 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5561 constraint(ALLOC_IN_RC(ptr_reg));
5562 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5563 op_cost(0);
5564 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5565 interface(MEMORY_INTER) %{
5566 base($reg);
5567 index($ireg);
5568 scale($scale);
5569 disp(0x0);
5570 %}
5571 %}
5572
5573 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5574 %{
5575 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5576 constraint(ALLOC_IN_RC(ptr_reg));
5577 match(AddP (DecodeN reg) (LShiftL lreg scale));
5578 op_cost(0);
5579 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5580 interface(MEMORY_INTER) %{
5581 base($reg);
5582 index($lreg);
5583 scale($scale);
5584 disp(0x0);
5585 %}
5586 %}
5587
5588 operand indIndexI2LN(iRegN reg, iRegI ireg)
5589 %{
5590 predicate(CompressedOops::shift() == 0);
5591 constraint(ALLOC_IN_RC(ptr_reg));
5592 match(AddP (DecodeN reg) (ConvI2L ireg));
5593 op_cost(0);
5594 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5595 interface(MEMORY_INTER) %{
5596 base($reg);
5597 index($ireg);
5598 scale(0x0);
5599 disp(0x0);
5600 %}
5601 %}
5602
5603 operand indIndexN(iRegN reg, iRegL lreg)
5604 %{
5605 predicate(CompressedOops::shift() == 0);
5606 constraint(ALLOC_IN_RC(ptr_reg));
5607 match(AddP (DecodeN reg) lreg);
5608 op_cost(0);
5609 format %{ "$reg, $lreg\t# narrow" %}
5610 interface(MEMORY_INTER) %{
5611 base($reg);
5612 index($lreg);
5613 scale(0x0);
5614 disp(0x0);
5615 %}
5616 %}
5617
5618 operand indOffIN(iRegN reg, immIOffset off)
5619 %{
5620 predicate(CompressedOops::shift() == 0);
5621 constraint(ALLOC_IN_RC(ptr_reg));
5622 match(AddP (DecodeN reg) off);
5623 op_cost(0);
5624 format %{ "[$reg, $off]\t# narrow" %}
5625 interface(MEMORY_INTER) %{
5626 base($reg);
5627 index(0xffffffff);
5628 scale(0x0);
5629 disp($off);
5630 %}
5631 %}
5632
5633 operand indOffLN(iRegN reg, immLOffset off)
5634 %{
5635 predicate(CompressedOops::shift() == 0);
5636 constraint(ALLOC_IN_RC(ptr_reg));
5637 match(AddP (DecodeN reg) off);
5638 op_cost(0);
5639 format %{ "[$reg, $off]\t# narrow" %}
5640 interface(MEMORY_INTER) %{
5641 base($reg);
5642 index(0xffffffff);
5643 scale(0x0);
5644 disp($off);
5645 %}
5646 %}
5647
5648
5649 //----------Special Memory Operands--------------------------------------------
5650 // Stack Slot Operand - This operand is used for loading and storing temporary
5651 // values on the stack where a match requires a value to
5652 // flow through memory.
5653 operand stackSlotP(sRegP reg)
5654 %{
5655 constraint(ALLOC_IN_RC(stack_slots));
5656 op_cost(100);
5657 // No match rule because this operand is only generated in matching
5658 // match(RegP);
5659 format %{ "[$reg]" %}
5660 interface(MEMORY_INTER) %{
5661 base(0x1e); // RSP
5662 index(0x0); // No Index
5663 scale(0x0); // No Scale
5664 disp($reg); // Stack Offset
5665 %}
5666 %}
5667
5668 operand stackSlotI(sRegI reg)
5669 %{
5670 constraint(ALLOC_IN_RC(stack_slots));
5671 // No match rule because this operand is only generated in matching
5672 // match(RegI);
5673 format %{ "[$reg]" %}
5674 interface(MEMORY_INTER) %{
5675 base(0x1e); // RSP
5676 index(0x0); // No Index
5677 scale(0x0); // No Scale
5678 disp($reg); // Stack Offset
5679 %}
5680 %}
5681
5682 operand stackSlotF(sRegF reg)
5683 %{
5684 constraint(ALLOC_IN_RC(stack_slots));
5685 // No match rule because this operand is only generated in matching
5686 // match(RegF);
5687 format %{ "[$reg]" %}
5688 interface(MEMORY_INTER) %{
5689 base(0x1e); // RSP
5690 index(0x0); // No Index
5691 scale(0x0); // No Scale
5692 disp($reg); // Stack Offset
5693 %}
5694 %}
5695
5696 operand stackSlotD(sRegD reg)
5697 %{
5698 constraint(ALLOC_IN_RC(stack_slots));
5699 // No match rule because this operand is only generated in matching
5700 // match(RegD);
5701 format %{ "[$reg]" %}
5702 interface(MEMORY_INTER) %{
5703 base(0x1e); // RSP
5704 index(0x0); // No Index
5705 scale(0x0); // No Scale
5706 disp($reg); // Stack Offset
5707 %}
5708 %}
5709
5710 operand stackSlotL(sRegL reg)
5711 %{
5712 constraint(ALLOC_IN_RC(stack_slots));
5713 // No match rule because this operand is only generated in matching
5714 // match(RegL);
5715 format %{ "[$reg]" %}
5716 interface(MEMORY_INTER) %{
5717 base(0x1e); // RSP
5718 index(0x0); // No Index
5719 scale(0x0); // No Scale
5720 disp($reg); // Stack Offset
5721 %}
5722 %}
5723
5724 // Operands for expressing Control Flow
5725 // NOTE: Label is a predefined operand which should not be redefined in
5726 // the AD file. It is generically handled within the ADLC.
5727
5728 //----------Conditional Branch Operands----------------------------------------
5729 // Comparison Op - This is the operation of the comparison, and is limited to
5730 // the following set of codes:
5731 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5732 //
5733 // Other attributes of the comparison, such as unsignedness, are specified
5734 // by the comparison instruction that sets a condition code flags register.
5735 // That result is represented by a flags operand whose subtype is appropriate
5736 // to the unsignedness (etc.) of the comparison.
5737 //
5738 // Later, the instruction which matches both the Comparison Op (a Bool) and
5739 // the flags (produced by the Cmp) specifies the coding of the comparison op
5740 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5741
5742 // used for signed integral comparisons and fp comparisons
5743
5744 operand cmpOp()
5745 %{
5746 match(Bool);
5747
5748 format %{ "" %}
5749 interface(COND_INTER) %{
5750 equal(0x0, "eq");
5751 not_equal(0x1, "ne");
5752 less(0xb, "lt");
5753 greater_equal(0xa, "ge");
5754 less_equal(0xd, "le");
5755 greater(0xc, "gt");
5756 overflow(0x6, "vs");
5757 no_overflow(0x7, "vc");
5758 %}
5759 %}
5760
5761 // used for unsigned integral comparisons
5762
5763 operand cmpOpU()
5764 %{
5765 match(Bool);
5766
5767 format %{ "" %}
5768 interface(COND_INTER) %{
5769 equal(0x0, "eq");
5770 not_equal(0x1, "ne");
5771 less(0x3, "lo");
5772 greater_equal(0x2, "hs");
5773 less_equal(0x9, "ls");
5774 greater(0x8, "hi");
5775 overflow(0x6, "vs");
5776 no_overflow(0x7, "vc");
5777 %}
5778 %}
5779
5780 // used for certain integral comparisons which can be
5781 // converted to cbxx or tbxx instructions
5782
5783 operand cmpOpEqNe()
5784 %{
5785 match(Bool);
5786 op_cost(0);
5787 predicate(n->as_Bool()->_test._test == BoolTest::ne
5788 || n->as_Bool()->_test._test == BoolTest::eq);
5789
5790 format %{ "" %}
5791 interface(COND_INTER) %{
5792 equal(0x0, "eq");
5793 not_equal(0x1, "ne");
5794 less(0xb, "lt");
5795 greater_equal(0xa, "ge");
5796 less_equal(0xd, "le");
5797 greater(0xc, "gt");
5798 overflow(0x6, "vs");
5799 no_overflow(0x7, "vc");
5800 %}
5801 %}
5802
5803 // used for certain integral comparisons which can be
5804 // converted to cbxx or tbxx instructions
5805
5806 operand cmpOpLtGe()
5807 %{
5808 match(Bool);
5809 op_cost(0);
5810
5811 predicate(n->as_Bool()->_test._test == BoolTest::lt
5812 || n->as_Bool()->_test._test == BoolTest::ge);
5813
5814 format %{ "" %}
5815 interface(COND_INTER) %{
5816 equal(0x0, "eq");
5817 not_equal(0x1, "ne");
5818 less(0xb, "lt");
5819 greater_equal(0xa, "ge");
5820 less_equal(0xd, "le");
5821 greater(0xc, "gt");
5822 overflow(0x6, "vs");
5823 no_overflow(0x7, "vc");
5824 %}
5825 %}
5826
5827 // used for certain unsigned integral comparisons which can be
5828 // converted to cbxx or tbxx instructions
5829
5830 operand cmpOpUEqNeLeGt()
5831 %{
5832 match(Bool);
5833 op_cost(0);
5834
5835 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5836 n->as_Bool()->_test._test == BoolTest::ne ||
5837 n->as_Bool()->_test._test == BoolTest::le ||
5838 n->as_Bool()->_test._test == BoolTest::gt);
5839
5840 format %{ "" %}
5841 interface(COND_INTER) %{
5842 equal(0x0, "eq");
5843 not_equal(0x1, "ne");
5844 less(0x3, "lo");
5845 greater_equal(0x2, "hs");
5846 less_equal(0x9, "ls");
5847 greater(0x8, "hi");
5848 overflow(0x6, "vs");
5849 no_overflow(0x7, "vc");
5850 %}
5851 %}
5852
5853 // Special operand allowing long args to int ops to be truncated for free
5854
5855 operand iRegL2I(iRegL reg) %{
5856
5857 op_cost(0);
5858
5859 match(ConvL2I reg);
5860
5861 format %{ "l2i($reg)" %}
5862
5863 interface(REG_INTER)
5864 %}
5865
5866 operand iRegL2P(iRegL reg) %{
5867
5868 op_cost(0);
5869
5870 match(CastX2P reg);
5871
5872 format %{ "l2p($reg)" %}
5873
5874 interface(REG_INTER)
5875 %}
5876
5877 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5878 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5879 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5880 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5881
5882 //----------OPERAND CLASSES----------------------------------------------------
5883 // Operand Classes are groups of operands that are used as to simplify
5884 // instruction definitions by not requiring the AD writer to specify
5885 // separate instructions for every form of operand when the
5886 // instruction accepts multiple operand types with the same basic
5887 // encoding and format. The classic case of this is memory operands.
5888
5889 // memory is used to define read/write location for load/store
5890 // instruction defs. we can turn a memory op into an Address
5891
5892 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5893 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5894
5895 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5896 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5897
5898 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5899 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5900
5901 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5902 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5903
5904 // All of the memory operands. For the pipeline description.
5905 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5906 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5907 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5908
5909
5910 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5911 // operations. it allows the src to be either an iRegI or a (ConvL2I
5912 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5913 // can be elided because the 32-bit instruction will just employ the
5914 // lower 32 bits anyway.
5915 //
5916 // n.b. this does not elide all L2I conversions. if the truncated
5917 // value is consumed by more than one operation then the ConvL2I
5918 // cannot be bundled into the consuming nodes so an l2i gets planted
5919 // (actually a movw $dst $src) and the downstream instructions consume
5920 // the result of the l2i as an iRegI input. That's a shame since the
5921 // movw is actually redundant but its not too costly.
5922
5923 opclass iRegIorL2I(iRegI, iRegL2I);
5924 opclass iRegPorL2P(iRegP, iRegL2P);
5925
5926 //----------PIPELINE-----------------------------------------------------------
5927 // Rules which define the behavior of the target architectures pipeline.
5928
5929 // For specific pipelines, eg A53, define the stages of that pipeline
5930 //pipe_desc(ISS, EX1, EX2, WR);
5931 #define ISS S0
5932 #define EX1 S1
5933 #define EX2 S2
5934 #define WR S3
5935
5936 // Integer ALU reg operation
5937 pipeline %{
5938
5939 attributes %{
5940 // ARM instructions are of fixed length
5941 fixed_size_instructions; // Fixed size instructions TODO does
5942 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5943 // ARM instructions come in 32-bit word units
5944 instruction_unit_size = 4; // An instruction is 4 bytes long
5945 instruction_fetch_unit_size = 64; // The processor fetches one line
5946 instruction_fetch_units = 1; // of 64 bytes
5947 %}
5948
5949 // We don't use an actual pipeline model so don't care about resources
5950 // or description. we do use pipeline classes to introduce fixed
5951 // latencies
5952
5953 //----------RESOURCES----------------------------------------------------------
5954 // Resources are the functional units available to the machine
5955
5956 resources( INS0, INS1, INS01 = INS0 | INS1,
5957 ALU0, ALU1, ALU = ALU0 | ALU1,
5958 MAC,
5959 DIV,
5960 BRANCH,
5961 LDST,
5962 NEON_FP);
5963
5964 //----------PIPELINE DESCRIPTION-----------------------------------------------
5965 // Pipeline Description specifies the stages in the machine's pipeline
5966
5967 // Define the pipeline as a generic 6 stage pipeline
5968 pipe_desc(S0, S1, S2, S3, S4, S5);
5969
5970 //----------PIPELINE CLASSES---------------------------------------------------
5971 // Pipeline Classes describe the stages in which input and output are
5972 // referenced by the hardware pipeline.
5973
5974 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5975 %{
5976 single_instruction;
5977 src1 : S1(read);
5978 src2 : S2(read);
5979 dst : S5(write);
5980 INS01 : ISS;
5981 NEON_FP : S5;
5982 %}
5983
5984 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5985 %{
5986 single_instruction;
5987 src1 : S1(read);
5988 src2 : S2(read);
5989 dst : S5(write);
5990 INS01 : ISS;
5991 NEON_FP : S5;
5992 %}
5993
5994 pipe_class fp_uop_s(vRegF dst, vRegF src)
5995 %{
5996 single_instruction;
5997 src : S1(read);
5998 dst : S5(write);
5999 INS01 : ISS;
6000 NEON_FP : S5;
6001 %}
6002
6003 pipe_class fp_uop_d(vRegD dst, vRegD src)
6004 %{
6005 single_instruction;
6006 src : S1(read);
6007 dst : S5(write);
6008 INS01 : ISS;
6009 NEON_FP : S5;
6010 %}
6011
6012 pipe_class fp_d2f(vRegF dst, vRegD src)
6013 %{
6014 single_instruction;
6015 src : S1(read);
6016 dst : S5(write);
6017 INS01 : ISS;
6018 NEON_FP : S5;
6019 %}
6020
6021 pipe_class fp_f2d(vRegD dst, vRegF src)
6022 %{
6023 single_instruction;
6024 src : S1(read);
6025 dst : S5(write);
6026 INS01 : ISS;
6027 NEON_FP : S5;
6028 %}
6029
6030 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6031 %{
6032 single_instruction;
6033 src : S1(read);
6034 dst : S5(write);
6035 INS01 : ISS;
6036 NEON_FP : S5;
6037 %}
6038
6039 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6040 %{
6041 single_instruction;
6042 src : S1(read);
6043 dst : S5(write);
6044 INS01 : ISS;
6045 NEON_FP : S5;
6046 %}
6047
6048 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6049 %{
6050 single_instruction;
6051 src : S1(read);
6052 dst : S5(write);
6053 INS01 : ISS;
6054 NEON_FP : S5;
6055 %}
6056
6057 pipe_class fp_l2f(vRegF dst, iRegL src)
6058 %{
6059 single_instruction;
6060 src : S1(read);
6061 dst : S5(write);
6062 INS01 : ISS;
6063 NEON_FP : S5;
6064 %}
6065
6066 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6067 %{
6068 single_instruction;
6069 src : S1(read);
6070 dst : S5(write);
6071 INS01 : ISS;
6072 NEON_FP : S5;
6073 %}
6074
6075 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6076 %{
6077 single_instruction;
6078 src : S1(read);
6079 dst : S5(write);
6080 INS01 : ISS;
6081 NEON_FP : S5;
6082 %}
6083
6084 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6085 %{
6086 single_instruction;
6087 src : S1(read);
6088 dst : S5(write);
6089 INS01 : ISS;
6090 NEON_FP : S5;
6091 %}
6092
6093 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6094 %{
6095 single_instruction;
6096 src : S1(read);
6097 dst : S5(write);
6098 INS01 : ISS;
6099 NEON_FP : S5;
6100 %}
6101
6102 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6103 %{
6104 single_instruction;
6105 src1 : S1(read);
6106 src2 : S2(read);
6107 dst : S5(write);
6108 INS0 : ISS;
6109 NEON_FP : S5;
6110 %}
6111
6112 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6113 %{
6114 single_instruction;
6115 src1 : S1(read);
6116 src2 : S2(read);
6117 dst : S5(write);
6118 INS0 : ISS;
6119 NEON_FP : S5;
6120 %}
6121
6122 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6123 %{
6124 single_instruction;
6125 cr : S1(read);
6126 src1 : S1(read);
6127 src2 : S1(read);
6128 dst : S3(write);
6129 INS01 : ISS;
6130 NEON_FP : S3;
6131 %}
6132
6133 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6134 %{
6135 single_instruction;
6136 cr : S1(read);
6137 src1 : S1(read);
6138 src2 : S1(read);
6139 dst : S3(write);
6140 INS01 : ISS;
6141 NEON_FP : S3;
6142 %}
6143
6144 pipe_class fp_imm_s(vRegF dst)
6145 %{
6146 single_instruction;
6147 dst : S3(write);
6148 INS01 : ISS;
6149 NEON_FP : S3;
6150 %}
6151
6152 pipe_class fp_imm_d(vRegD dst)
6153 %{
6154 single_instruction;
6155 dst : S3(write);
6156 INS01 : ISS;
6157 NEON_FP : S3;
6158 %}
6159
6160 pipe_class fp_load_constant_s(vRegF dst)
6161 %{
6162 single_instruction;
6163 dst : S4(write);
6164 INS01 : ISS;
6165 NEON_FP : S4;
6166 %}
6167
6168 pipe_class fp_load_constant_d(vRegD dst)
6169 %{
6170 single_instruction;
6171 dst : S4(write);
6172 INS01 : ISS;
6173 NEON_FP : S4;
6174 %}
6175
6176 //------- Integer ALU operations --------------------------
6177
6178 // Integer ALU reg-reg operation
6179 // Operands needed in EX1, result generated in EX2
6180 // Eg. ADD x0, x1, x2
6181 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6182 %{
6183 single_instruction;
6184 dst : EX2(write);
6185 src1 : EX1(read);
6186 src2 : EX1(read);
6187 INS01 : ISS; // Dual issue as instruction 0 or 1
6188 ALU : EX2;
6189 %}
6190
6191 // Integer ALU reg-reg operation with constant shift
6192 // Shifted register must be available in LATE_ISS instead of EX1
6193 // Eg. ADD x0, x1, x2, LSL #2
6194 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6195 %{
6196 single_instruction;
6197 dst : EX2(write);
6198 src1 : EX1(read);
6199 src2 : ISS(read);
6200 INS01 : ISS;
6201 ALU : EX2;
6202 %}
6203
6204 // Integer ALU reg operation with constant shift
6205 // Eg. LSL x0, x1, #shift
6206 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6207 %{
6208 single_instruction;
6209 dst : EX2(write);
6210 src1 : ISS(read);
6211 INS01 : ISS;
6212 ALU : EX2;
6213 %}
6214
6215 // Integer ALU reg-reg operation with variable shift
6216 // Both operands must be available in LATE_ISS instead of EX1
6217 // Result is available in EX1 instead of EX2
6218 // Eg. LSLV x0, x1, x2
6219 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6220 %{
6221 single_instruction;
6222 dst : EX1(write);
6223 src1 : ISS(read);
6224 src2 : ISS(read);
6225 INS01 : ISS;
6226 ALU : EX1;
6227 %}
6228
6229 // Integer ALU reg-reg operation with extract
6230 // As for _vshift above, but result generated in EX2
6231 // Eg. EXTR x0, x1, x2, #N
6232 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6233 %{
6234 single_instruction;
6235 dst : EX2(write);
6236 src1 : ISS(read);
6237 src2 : ISS(read);
6238 INS1 : ISS; // Can only dual issue as Instruction 1
6239 ALU : EX1;
6240 %}
6241
6242 // Integer ALU reg operation
6243 // Eg. NEG x0, x1
6244 pipe_class ialu_reg(iRegI dst, iRegI src)
6245 %{
6246 single_instruction;
6247 dst : EX2(write);
6248 src : EX1(read);
6249 INS01 : ISS;
6250 ALU : EX2;
6251 %}
6252
6253 // Integer ALU reg mmediate operation
6254 // Eg. ADD x0, x1, #N
6255 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6256 %{
6257 single_instruction;
6258 dst : EX2(write);
6259 src1 : EX1(read);
6260 INS01 : ISS;
6261 ALU : EX2;
6262 %}
6263
6264 // Integer ALU immediate operation (no source operands)
6265 // Eg. MOV x0, #N
6266 pipe_class ialu_imm(iRegI dst)
6267 %{
6268 single_instruction;
6269 dst : EX1(write);
6270 INS01 : ISS;
6271 ALU : EX1;
6272 %}
6273
6274 //------- Compare operation -------------------------------
6275
6276 // Compare reg-reg
6277 // Eg. CMP x0, x1
6278 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6279 %{
6280 single_instruction;
6281 // fixed_latency(16);
6282 cr : EX2(write);
6283 op1 : EX1(read);
6284 op2 : EX1(read);
6285 INS01 : ISS;
6286 ALU : EX2;
6287 %}
6288
6289 // Compare reg-reg
6290 // Eg. CMP x0, #N
6291 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6292 %{
6293 single_instruction;
6294 // fixed_latency(16);
6295 cr : EX2(write);
6296 op1 : EX1(read);
6297 INS01 : ISS;
6298 ALU : EX2;
6299 %}
6300
6301 //------- Conditional instructions ------------------------
6302
6303 // Conditional no operands
6304 // Eg. CSINC x0, zr, zr, <cond>
6305 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6306 %{
6307 single_instruction;
6308 cr : EX1(read);
6309 dst : EX2(write);
6310 INS01 : ISS;
6311 ALU : EX2;
6312 %}
6313
6314 // Conditional 2 operand
6315 // EG. CSEL X0, X1, X2, <cond>
6316 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6317 %{
6318 single_instruction;
6319 cr : EX1(read);
6320 src1 : EX1(read);
6321 src2 : EX1(read);
6322 dst : EX2(write);
6323 INS01 : ISS;
6324 ALU : EX2;
6325 %}
6326
6327 // Conditional 2 operand
6328 // EG. CSEL X0, X1, X2, <cond>
6329 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6330 %{
6331 single_instruction;
6332 cr : EX1(read);
6333 src : EX1(read);
6334 dst : EX2(write);
6335 INS01 : ISS;
6336 ALU : EX2;
6337 %}
6338
6339 //------- Multiply pipeline operations --------------------
6340
6341 // Multiply reg-reg
6342 // Eg. MUL w0, w1, w2
6343 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6344 %{
6345 single_instruction;
6346 dst : WR(write);
6347 src1 : ISS(read);
6348 src2 : ISS(read);
6349 INS01 : ISS;
6350 MAC : WR;
6351 %}
6352
6353 // Multiply accumulate
6354 // Eg. MADD w0, w1, w2, w3
6355 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6356 %{
6357 single_instruction;
6358 dst : WR(write);
6359 src1 : ISS(read);
6360 src2 : ISS(read);
6361 src3 : ISS(read);
6362 INS01 : ISS;
6363 MAC : WR;
6364 %}
6365
6366 // Eg. MUL w0, w1, w2
6367 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6368 %{
6369 single_instruction;
6370 fixed_latency(3); // Maximum latency for 64 bit mul
6371 dst : WR(write);
6372 src1 : ISS(read);
6373 src2 : ISS(read);
6374 INS01 : ISS;
6375 MAC : WR;
6376 %}
6377
6378 // Multiply accumulate
6379 // Eg. MADD w0, w1, w2, w3
6380 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6381 %{
6382 single_instruction;
6383 fixed_latency(3); // Maximum latency for 64 bit mul
6384 dst : WR(write);
6385 src1 : ISS(read);
6386 src2 : ISS(read);
6387 src3 : ISS(read);
6388 INS01 : ISS;
6389 MAC : WR;
6390 %}
6391
6392 //------- Divide pipeline operations --------------------
6393
6394 // Eg. SDIV w0, w1, w2
6395 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6396 %{
6397 single_instruction;
6398 fixed_latency(8); // Maximum latency for 32 bit divide
6399 dst : WR(write);
6400 src1 : ISS(read);
6401 src2 : ISS(read);
6402 INS0 : ISS; // Can only dual issue as instruction 0
6403 DIV : WR;
6404 %}
6405
6406 // Eg. SDIV x0, x1, x2
6407 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6408 %{
6409 single_instruction;
6410 fixed_latency(16); // Maximum latency for 64 bit divide
6411 dst : WR(write);
6412 src1 : ISS(read);
6413 src2 : ISS(read);
6414 INS0 : ISS; // Can only dual issue as instruction 0
6415 DIV : WR;
6416 %}
6417
6418 //------- Load pipeline operations ------------------------
6419
6420 // Load - prefetch
6421 // Eg. PFRM <mem>
6422 pipe_class iload_prefetch(memory mem)
6423 %{
6424 single_instruction;
6425 mem : ISS(read);
6426 INS01 : ISS;
6427 LDST : WR;
6428 %}
6429
6430 // Load - reg, mem
6431 // Eg. LDR x0, <mem>
6432 pipe_class iload_reg_mem(iRegI dst, memory mem)
6433 %{
6434 single_instruction;
6435 dst : WR(write);
6436 mem : ISS(read);
6437 INS01 : ISS;
6438 LDST : WR;
6439 %}
6440
6441 // Load - reg, reg
6442 // Eg. LDR x0, [sp, x1]
6443 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6444 %{
6445 single_instruction;
6446 dst : WR(write);
6447 src : ISS(read);
6448 INS01 : ISS;
6449 LDST : WR;
6450 %}
6451
6452 //------- Store pipeline operations -----------------------
6453
6454 // Store - zr, mem
6455 // Eg. STR zr, <mem>
6456 pipe_class istore_mem(memory mem)
6457 %{
6458 single_instruction;
6459 mem : ISS(read);
6460 INS01 : ISS;
6461 LDST : WR;
6462 %}
6463
6464 // Store - reg, mem
6465 // Eg. STR x0, <mem>
6466 pipe_class istore_reg_mem(iRegI src, memory mem)
6467 %{
6468 single_instruction;
6469 mem : ISS(read);
6470 src : EX2(read);
6471 INS01 : ISS;
6472 LDST : WR;
6473 %}
6474
6475 // Store - reg, reg
6476 // Eg. STR x0, [sp, x1]
6477 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6478 %{
6479 single_instruction;
6480 dst : ISS(read);
6481 src : EX2(read);
6482 INS01 : ISS;
6483 LDST : WR;
6484 %}
6485
6486 //------- Store pipeline operations -----------------------
6487
6488 // Branch
6489 pipe_class pipe_branch()
6490 %{
6491 single_instruction;
6492 INS01 : ISS;
6493 BRANCH : EX1;
6494 %}
6495
6496 // Conditional branch
6497 pipe_class pipe_branch_cond(rFlagsReg cr)
6498 %{
6499 single_instruction;
6500 cr : EX1(read);
6501 INS01 : ISS;
6502 BRANCH : EX1;
6503 %}
6504
6505 // Compare & Branch
6506 // EG. CBZ/CBNZ
6507 pipe_class pipe_cmp_branch(iRegI op1)
6508 %{
6509 single_instruction;
6510 op1 : EX1(read);
6511 INS01 : ISS;
6512 BRANCH : EX1;
6513 %}
6514
6515 //------- Synchronisation operations ----------------------
6516
6517 // Any operation requiring serialization.
6518 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6519 pipe_class pipe_serial()
6520 %{
6521 single_instruction;
6522 force_serialization;
6523 fixed_latency(16);
6524 INS01 : ISS(2); // Cannot dual issue with any other instruction
6525 LDST : WR;
6526 %}
6527
6528 // Generic big/slow expanded idiom - also serialized
6529 pipe_class pipe_slow()
6530 %{
6531 instruction_count(10);
6532 multiple_bundles;
6533 force_serialization;
6534 fixed_latency(16);
6535 INS01 : ISS(2); // Cannot dual issue with any other instruction
6536 LDST : WR;
6537 %}
6538
6539 // Empty pipeline class
6540 pipe_class pipe_class_empty()
6541 %{
6542 single_instruction;
6543 fixed_latency(0);
6544 %}
6545
6546 // Default pipeline class.
6547 pipe_class pipe_class_default()
6548 %{
6549 single_instruction;
6550 fixed_latency(2);
6551 %}
6552
6553 // Pipeline class for compares.
6554 pipe_class pipe_class_compare()
6555 %{
6556 single_instruction;
6557 fixed_latency(16);
6558 %}
6559
6560 // Pipeline class for memory operations.
6561 pipe_class pipe_class_memory()
6562 %{
6563 single_instruction;
6564 fixed_latency(16);
6565 %}
6566
6567 // Pipeline class for call.
6568 pipe_class pipe_class_call()
6569 %{
6570 single_instruction;
6571 fixed_latency(100);
6572 %}
6573
6574 // Define the class for the Nop node.
6575 define %{
6576 MachNop = pipe_class_empty;
6577 %}
6578
6579 %}
6580 //----------INSTRUCTIONS-------------------------------------------------------
6581 //
6582 // match -- States which machine-independent subtree may be replaced
6583 // by this instruction.
6584 // ins_cost -- The estimated cost of this instruction is used by instruction
6585 // selection to identify a minimum cost tree of machine
6586 // instructions that matches a tree of machine-independent
6587 // instructions.
6588 // format -- A string providing the disassembly for this instruction.
6589 // The value of an instruction's operand may be inserted
6590 // by referring to it with a '$' prefix.
6591 // opcode -- Three instruction opcodes may be provided. These are referred
6592 // to within an encode class as $primary, $secondary, and $tertiary
6593 // rrspectively. The primary opcode is commonly used to
6594 // indicate the type of machine instruction, while secondary
6595 // and tertiary are often used for prefix options or addressing
6596 // modes.
6597 // ins_encode -- A list of encode classes with parameters. The encode class
6598 // name must have been defined in an 'enc_class' specification
6599 // in the encode section of the architecture description.
6600
6601 // ============================================================================
6602 // Memory (Load/Store) Instructions
6603
6604 // Load Instructions
6605
6606 // Load Byte (8 bit signed)
6607 instruct loadB(iRegINoSp dst, memory1 mem)
6608 %{
6609 match(Set dst (LoadB mem));
6610 predicate(!needs_acquiring_load(n));
6611
6612 ins_cost(4 * INSN_COST);
6613 format %{ "ldrsbw $dst, $mem\t# byte" %}
6614
6615 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6616
6617 ins_pipe(iload_reg_mem);
6618 %}
6619
6620 // Load Byte (8 bit signed) into long
6621 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6622 %{
6623 match(Set dst (ConvI2L (LoadB mem)));
6624 predicate(!needs_acquiring_load(n->in(1)));
6625
6626 ins_cost(4 * INSN_COST);
6627 format %{ "ldrsb $dst, $mem\t# byte" %}
6628
6629 ins_encode(aarch64_enc_ldrsb(dst, mem));
6630
6631 ins_pipe(iload_reg_mem);
6632 %}
6633
6634 // Load Byte (8 bit unsigned)
6635 instruct loadUB(iRegINoSp dst, memory1 mem)
6636 %{
6637 match(Set dst (LoadUB mem));
6638 predicate(!needs_acquiring_load(n));
6639
6640 ins_cost(4 * INSN_COST);
6641 format %{ "ldrbw $dst, $mem\t# byte" %}
6642
6643 ins_encode(aarch64_enc_ldrb(dst, mem));
6644
6645 ins_pipe(iload_reg_mem);
6646 %}
6647
6648 // Load Byte (8 bit unsigned) into long
6649 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6650 %{
6651 match(Set dst (ConvI2L (LoadUB mem)));
6652 predicate(!needs_acquiring_load(n->in(1)));
6653
6654 ins_cost(4 * INSN_COST);
6655 format %{ "ldrb $dst, $mem\t# byte" %}
6656
6657 ins_encode(aarch64_enc_ldrb(dst, mem));
6658
6659 ins_pipe(iload_reg_mem);
6660 %}
6661
6662 // Load Short (16 bit signed)
6663 instruct loadS(iRegINoSp dst, memory2 mem)
6664 %{
6665 match(Set dst (LoadS mem));
6666 predicate(!needs_acquiring_load(n));
6667
6668 ins_cost(4 * INSN_COST);
6669 format %{ "ldrshw $dst, $mem\t# short" %}
6670
6671 ins_encode(aarch64_enc_ldrshw(dst, mem));
6672
6673 ins_pipe(iload_reg_mem);
6674 %}
6675
6676 // Load Short (16 bit signed) into long
6677 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6678 %{
6679 match(Set dst (ConvI2L (LoadS mem)));
6680 predicate(!needs_acquiring_load(n->in(1)));
6681
6682 ins_cost(4 * INSN_COST);
6683 format %{ "ldrsh $dst, $mem\t# short" %}
6684
6685 ins_encode(aarch64_enc_ldrsh(dst, mem));
6686
6687 ins_pipe(iload_reg_mem);
6688 %}
6689
6690 // Load Char (16 bit unsigned)
6691 instruct loadUS(iRegINoSp dst, memory2 mem)
6692 %{
6693 match(Set dst (LoadUS mem));
6694 predicate(!needs_acquiring_load(n));
6695
6696 ins_cost(4 * INSN_COST);
6697 format %{ "ldrh $dst, $mem\t# short" %}
6698
6699 ins_encode(aarch64_enc_ldrh(dst, mem));
6700
6701 ins_pipe(iload_reg_mem);
6702 %}
6703
6704 // Load Short/Char (16 bit unsigned) into long
6705 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6706 %{
6707 match(Set dst (ConvI2L (LoadUS mem)));
6708 predicate(!needs_acquiring_load(n->in(1)));
6709
6710 ins_cost(4 * INSN_COST);
6711 format %{ "ldrh $dst, $mem\t# short" %}
6712
6713 ins_encode(aarch64_enc_ldrh(dst, mem));
6714
6715 ins_pipe(iload_reg_mem);
6716 %}
6717
6718 // Load Integer (32 bit signed)
6719 instruct loadI(iRegINoSp dst, memory4 mem)
6720 %{
6721 match(Set dst (LoadI mem));
6722 predicate(!needs_acquiring_load(n));
6723
6724 ins_cost(4 * INSN_COST);
6725 format %{ "ldrw $dst, $mem\t# int" %}
6726
6727 ins_encode(aarch64_enc_ldrw(dst, mem));
6728
6729 ins_pipe(iload_reg_mem);
6730 %}
6731
6732 // Load Integer (32 bit signed) into long
6733 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6734 %{
6735 match(Set dst (ConvI2L (LoadI mem)));
6736 predicate(!needs_acquiring_load(n->in(1)));
6737
6738 ins_cost(4 * INSN_COST);
6739 format %{ "ldrsw $dst, $mem\t# int" %}
6740
6741 ins_encode(aarch64_enc_ldrsw(dst, mem));
6742
6743 ins_pipe(iload_reg_mem);
6744 %}
6745
6746 // Load Integer (32 bit unsigned) into long
6747 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6748 %{
6749 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6750 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6751
6752 ins_cost(4 * INSN_COST);
6753 format %{ "ldrw $dst, $mem\t# int" %}
6754
6755 ins_encode(aarch64_enc_ldrw(dst, mem));
6756
6757 ins_pipe(iload_reg_mem);
6758 %}
6759
6760 // Load Long (64 bit signed)
6761 instruct loadL(iRegLNoSp dst, memory8 mem)
6762 %{
6763 match(Set dst (LoadL mem));
6764 predicate(!needs_acquiring_load(n));
6765
6766 ins_cost(4 * INSN_COST);
6767 format %{ "ldr $dst, $mem\t# int" %}
6768
6769 ins_encode(aarch64_enc_ldr(dst, mem));
6770
6771 ins_pipe(iload_reg_mem);
6772 %}
6773
6774 // Load Range
6775 instruct loadRange(iRegINoSp dst, memory4 mem)
6776 %{
6777 match(Set dst (LoadRange mem));
6778
6779 ins_cost(4 * INSN_COST);
6780 format %{ "ldrw $dst, $mem\t# range" %}
6781
6782 ins_encode(aarch64_enc_ldrw(dst, mem));
6783
6784 ins_pipe(iload_reg_mem);
6785 %}
6786
6787 // Load Pointer
6788 instruct loadP(iRegPNoSp dst, memory8 mem)
6789 %{
6790 match(Set dst (LoadP mem));
6791 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6792
6793 ins_cost(4 * INSN_COST);
6794 format %{ "ldr $dst, $mem\t# ptr" %}
6795
6796 ins_encode(aarch64_enc_ldr(dst, mem));
6797
6798 ins_pipe(iload_reg_mem);
6799 %}
6800
6801 // Load Compressed Pointer
6802 instruct loadN(iRegNNoSp dst, memory4 mem)
6803 %{
6804 match(Set dst (LoadN mem));
6805 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6806
6807 ins_cost(4 * INSN_COST);
6808 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6809
6810 ins_encode(aarch64_enc_ldrw(dst, mem));
6811
6812 ins_pipe(iload_reg_mem);
6813 %}
6814
6815 // Load Klass Pointer
6816 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6817 %{
6818 match(Set dst (LoadKlass mem));
6819 predicate(!needs_acquiring_load(n));
6820
6821 ins_cost(4 * INSN_COST);
6822 format %{ "ldr $dst, $mem\t# class" %}
6823
6824 ins_encode(aarch64_enc_ldr(dst, mem));
6825
6826 ins_pipe(iload_reg_mem);
6827 %}
6828
6829 // Load Narrow Klass Pointer
6830 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6831 %{
6832 match(Set dst (LoadNKlass mem));
6833 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6834
6835 ins_cost(4 * INSN_COST);
6836 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6837
6838 ins_encode(aarch64_enc_ldrw(dst, mem));
6839
6840 ins_pipe(iload_reg_mem);
6841 %}
6842
6843 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6844 %{
6845 match(Set dst (LoadNKlass mem));
6846 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6847
6848 ins_cost(4 * INSN_COST);
6849 format %{
6850 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6851 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6852 %}
6853 ins_encode %{
6854 // inlined aarch64_enc_ldrw
6855 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6856 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6857 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6858 %}
6859 ins_pipe(iload_reg_mem);
6860 %}
6861
6862 // Load Float
6863 instruct loadF(vRegF dst, memory4 mem)
6864 %{
6865 match(Set dst (LoadF mem));
6866 predicate(!needs_acquiring_load(n));
6867
6868 ins_cost(4 * INSN_COST);
6869 format %{ "ldrs $dst, $mem\t# float" %}
6870
6871 ins_encode( aarch64_enc_ldrs(dst, mem) );
6872
6873 ins_pipe(pipe_class_memory);
6874 %}
6875
6876 // Load Double
6877 instruct loadD(vRegD dst, memory8 mem)
6878 %{
6879 match(Set dst (LoadD mem));
6880 predicate(!needs_acquiring_load(n));
6881
6882 ins_cost(4 * INSN_COST);
6883 format %{ "ldrd $dst, $mem\t# double" %}
6884
6885 ins_encode( aarch64_enc_ldrd(dst, mem) );
6886
6887 ins_pipe(pipe_class_memory);
6888 %}
6889
6890
6891 // Load Int Constant
6892 instruct loadConI(iRegINoSp dst, immI src)
6893 %{
6894 match(Set dst src);
6895
6896 ins_cost(INSN_COST);
6897 format %{ "mov $dst, $src\t# int" %}
6898
6899 ins_encode( aarch64_enc_movw_imm(dst, src) );
6900
6901 ins_pipe(ialu_imm);
6902 %}
6903
6904 // Load Long Constant
6905 instruct loadConL(iRegLNoSp dst, immL src)
6906 %{
6907 match(Set dst src);
6908
6909 ins_cost(INSN_COST);
6910 format %{ "mov $dst, $src\t# long" %}
6911
6912 ins_encode( aarch64_enc_mov_imm(dst, src) );
6913
6914 ins_pipe(ialu_imm);
6915 %}
6916
6917 // Load Pointer Constant
6918
6919 instruct loadConP(iRegPNoSp dst, immP con)
6920 %{
6921 match(Set dst con);
6922
6923 ins_cost(INSN_COST * 4);
6924 format %{
6925 "mov $dst, $con\t# ptr\n\t"
6926 %}
6927
6928 ins_encode(aarch64_enc_mov_p(dst, con));
6929
6930 ins_pipe(ialu_imm);
6931 %}
6932
6933 // Load Null Pointer Constant
6934
6935 instruct loadConP0(iRegPNoSp dst, immP0 con)
6936 %{
6937 match(Set dst con);
6938
6939 ins_cost(INSN_COST);
6940 format %{ "mov $dst, $con\t# nullptr ptr" %}
6941
6942 ins_encode(aarch64_enc_mov_p0(dst, con));
6943
6944 ins_pipe(ialu_imm);
6945 %}
6946
6947 // Load Pointer Constant One
6948
6949 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6950 %{
6951 match(Set dst con);
6952
6953 ins_cost(INSN_COST);
6954 format %{ "mov $dst, $con\t# nullptr ptr" %}
6955
6956 ins_encode(aarch64_enc_mov_p1(dst, con));
6957
6958 ins_pipe(ialu_imm);
6959 %}
6960
6961 // Load Narrow Pointer Constant
6962
6963 instruct loadConN(iRegNNoSp dst, immN con)
6964 %{
6965 match(Set dst con);
6966
6967 ins_cost(INSN_COST * 4);
6968 format %{ "mov $dst, $con\t# compressed ptr" %}
6969
6970 ins_encode(aarch64_enc_mov_n(dst, con));
6971
6972 ins_pipe(ialu_imm);
6973 %}
6974
6975 // Load Narrow Null Pointer Constant
6976
6977 instruct loadConN0(iRegNNoSp dst, immN0 con)
6978 %{
6979 match(Set dst con);
6980
6981 ins_cost(INSN_COST);
6982 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6983
6984 ins_encode(aarch64_enc_mov_n0(dst, con));
6985
6986 ins_pipe(ialu_imm);
6987 %}
6988
6989 // Load Narrow Klass Constant
6990
6991 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6992 %{
6993 match(Set dst con);
6994
6995 ins_cost(INSN_COST);
6996 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6997
6998 ins_encode(aarch64_enc_mov_nk(dst, con));
6999
7000 ins_pipe(ialu_imm);
7001 %}
7002
7003 // Load Packed Float Constant
7004
7005 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7006 match(Set dst con);
7007 ins_cost(INSN_COST * 4);
7008 format %{ "fmovs $dst, $con"%}
7009 ins_encode %{
7010 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7011 %}
7012
7013 ins_pipe(fp_imm_s);
7014 %}
7015
7016 // Load Float Constant
7017
7018 instruct loadConF(vRegF dst, immF con) %{
7019 match(Set dst con);
7020
7021 ins_cost(INSN_COST * 4);
7022
7023 format %{
7024 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7025 %}
7026
7027 ins_encode %{
7028 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7029 %}
7030
7031 ins_pipe(fp_load_constant_s);
7032 %}
7033
7034 // Load Packed Double Constant
7035
7036 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7037 match(Set dst con);
7038 ins_cost(INSN_COST);
7039 format %{ "fmovd $dst, $con"%}
7040 ins_encode %{
7041 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7042 %}
7043
7044 ins_pipe(fp_imm_d);
7045 %}
7046
7047 // Load Double Constant
7048
7049 instruct loadConD(vRegD dst, immD con) %{
7050 match(Set dst con);
7051
7052 ins_cost(INSN_COST * 5);
7053 format %{
7054 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7055 %}
7056
7057 ins_encode %{
7058 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7059 %}
7060
7061 ins_pipe(fp_load_constant_d);
7062 %}
7063
7064 // Load Half Float Constant
7065 instruct loadConH(vRegF dst, immH con) %{
7066 match(Set dst con);
7067 format %{ "mov rscratch1, $con\n\t"
7068 "fmov $dst, rscratch1"
7069 %}
7070 ins_encode %{
7071 __ movw(rscratch1, (uint32_t)$con$$constant);
7072 __ fmovs($dst$$FloatRegister, rscratch1);
7073 %}
7074 ins_pipe(pipe_class_default);
7075 %}
7076
7077 // Store Instructions
7078
7079 // Store Byte
7080 instruct storeB(iRegIorL2I src, memory1 mem)
7081 %{
7082 match(Set mem (StoreB mem src));
7083 predicate(!needs_releasing_store(n));
7084
7085 ins_cost(INSN_COST);
7086 format %{ "strb $src, $mem\t# byte" %}
7087
7088 ins_encode(aarch64_enc_strb(src, mem));
7089
7090 ins_pipe(istore_reg_mem);
7091 %}
7092
7093
7094 instruct storeimmB0(immI0 zero, memory1 mem)
7095 %{
7096 match(Set mem (StoreB mem zero));
7097 predicate(!needs_releasing_store(n));
7098
7099 ins_cost(INSN_COST);
7100 format %{ "strb rscractch2, $mem\t# byte" %}
7101
7102 ins_encode(aarch64_enc_strb0(mem));
7103
7104 ins_pipe(istore_mem);
7105 %}
7106
7107 // Store Char/Short
7108 instruct storeC(iRegIorL2I src, memory2 mem)
7109 %{
7110 match(Set mem (StoreC mem src));
7111 predicate(!needs_releasing_store(n));
7112
7113 ins_cost(INSN_COST);
7114 format %{ "strh $src, $mem\t# short" %}
7115
7116 ins_encode(aarch64_enc_strh(src, mem));
7117
7118 ins_pipe(istore_reg_mem);
7119 %}
7120
7121 instruct storeimmC0(immI0 zero, memory2 mem)
7122 %{
7123 match(Set mem (StoreC mem zero));
7124 predicate(!needs_releasing_store(n));
7125
7126 ins_cost(INSN_COST);
7127 format %{ "strh zr, $mem\t# short" %}
7128
7129 ins_encode(aarch64_enc_strh0(mem));
7130
7131 ins_pipe(istore_mem);
7132 %}
7133
7134 // Store Integer
7135
7136 instruct storeI(iRegIorL2I src, memory4 mem)
7137 %{
7138 match(Set mem(StoreI mem src));
7139 predicate(!needs_releasing_store(n));
7140
7141 ins_cost(INSN_COST);
7142 format %{ "strw $src, $mem\t# int" %}
7143
7144 ins_encode(aarch64_enc_strw(src, mem));
7145
7146 ins_pipe(istore_reg_mem);
7147 %}
7148
7149 instruct storeimmI0(immI0 zero, memory4 mem)
7150 %{
7151 match(Set mem(StoreI mem zero));
7152 predicate(!needs_releasing_store(n));
7153
7154 ins_cost(INSN_COST);
7155 format %{ "strw zr, $mem\t# int" %}
7156
7157 ins_encode(aarch64_enc_strw0(mem));
7158
7159 ins_pipe(istore_mem);
7160 %}
7161
7162 // Store Long (64 bit signed)
7163 instruct storeL(iRegL src, memory8 mem)
7164 %{
7165 match(Set mem (StoreL mem src));
7166 predicate(!needs_releasing_store(n));
7167
7168 ins_cost(INSN_COST);
7169 format %{ "str $src, $mem\t# int" %}
7170
7171 ins_encode(aarch64_enc_str(src, mem));
7172
7173 ins_pipe(istore_reg_mem);
7174 %}
7175
7176 // Store Long (64 bit signed)
7177 instruct storeimmL0(immL0 zero, memory8 mem)
7178 %{
7179 match(Set mem (StoreL mem zero));
7180 predicate(!needs_releasing_store(n));
7181
7182 ins_cost(INSN_COST);
7183 format %{ "str zr, $mem\t# int" %}
7184
7185 ins_encode(aarch64_enc_str0(mem));
7186
7187 ins_pipe(istore_mem);
7188 %}
7189
7190 // Store Pointer
7191 instruct storeP(iRegP src, memory8 mem)
7192 %{
7193 match(Set mem (StoreP mem src));
7194 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7195
7196 ins_cost(INSN_COST);
7197 format %{ "str $src, $mem\t# ptr" %}
7198
7199 ins_encode(aarch64_enc_str(src, mem));
7200
7201 ins_pipe(istore_reg_mem);
7202 %}
7203
7204 // Store Pointer
7205 instruct storeimmP0(immP0 zero, memory8 mem)
7206 %{
7207 match(Set mem (StoreP mem zero));
7208 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7209
7210 ins_cost(INSN_COST);
7211 format %{ "str zr, $mem\t# ptr" %}
7212
7213 ins_encode(aarch64_enc_str0(mem));
7214
7215 ins_pipe(istore_mem);
7216 %}
7217
7218 // Store Compressed Pointer
7219 instruct storeN(iRegN src, memory4 mem)
7220 %{
7221 match(Set mem (StoreN mem src));
7222 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7223
7224 ins_cost(INSN_COST);
7225 format %{ "strw $src, $mem\t# compressed ptr" %}
7226
7227 ins_encode(aarch64_enc_strw(src, mem));
7228
7229 ins_pipe(istore_reg_mem);
7230 %}
7231
7232 instruct storeImmN0(immN0 zero, memory4 mem)
7233 %{
7234 match(Set mem (StoreN mem zero));
7235 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7236
7237 ins_cost(INSN_COST);
7238 format %{ "strw zr, $mem\t# compressed ptr" %}
7239
7240 ins_encode(aarch64_enc_strw0(mem));
7241
7242 ins_pipe(istore_mem);
7243 %}
7244
7245 // Store Float
7246 instruct storeF(vRegF src, memory4 mem)
7247 %{
7248 match(Set mem (StoreF mem src));
7249 predicate(!needs_releasing_store(n));
7250
7251 ins_cost(INSN_COST);
7252 format %{ "strs $src, $mem\t# float" %}
7253
7254 ins_encode( aarch64_enc_strs(src, mem) );
7255
7256 ins_pipe(pipe_class_memory);
7257 %}
7258
7259 // TODO
7260 // implement storeImmF0 and storeFImmPacked
7261
7262 // Store Double
7263 instruct storeD(vRegD src, memory8 mem)
7264 %{
7265 match(Set mem (StoreD mem src));
7266 predicate(!needs_releasing_store(n));
7267
7268 ins_cost(INSN_COST);
7269 format %{ "strd $src, $mem\t# double" %}
7270
7271 ins_encode( aarch64_enc_strd(src, mem) );
7272
7273 ins_pipe(pipe_class_memory);
7274 %}
7275
7276 // Store Compressed Klass Pointer
7277 instruct storeNKlass(iRegN src, memory4 mem)
7278 %{
7279 predicate(!needs_releasing_store(n));
7280 match(Set mem (StoreNKlass mem src));
7281
7282 ins_cost(INSN_COST);
7283 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7284
7285 ins_encode(aarch64_enc_strw(src, mem));
7286
7287 ins_pipe(istore_reg_mem);
7288 %}
7289
7290 // TODO
7291 // implement storeImmD0 and storeDImmPacked
7292
7293 // prefetch instructions
7294 // Must be safe to execute with invalid address (cannot fault).
7295
7296 instruct prefetchalloc( memory8 mem ) %{
7297 match(PrefetchAllocation mem);
7298
7299 ins_cost(INSN_COST);
7300 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7301
7302 ins_encode( aarch64_enc_prefetchw(mem) );
7303
7304 ins_pipe(iload_prefetch);
7305 %}
7306
7307 // ---------------- volatile loads and stores ----------------
7308
7309 // Load Byte (8 bit signed)
7310 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7311 %{
7312 match(Set dst (LoadB mem));
7313
7314 ins_cost(VOLATILE_REF_COST);
7315 format %{ "ldarsb $dst, $mem\t# byte" %}
7316
7317 ins_encode(aarch64_enc_ldarsb(dst, mem));
7318
7319 ins_pipe(pipe_serial);
7320 %}
7321
7322 // Load Byte (8 bit signed) into long
7323 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7324 %{
7325 match(Set dst (ConvI2L (LoadB mem)));
7326
7327 ins_cost(VOLATILE_REF_COST);
7328 format %{ "ldarsb $dst, $mem\t# byte" %}
7329
7330 ins_encode(aarch64_enc_ldarsb(dst, mem));
7331
7332 ins_pipe(pipe_serial);
7333 %}
7334
7335 // Load Byte (8 bit unsigned)
7336 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7337 %{
7338 match(Set dst (LoadUB mem));
7339
7340 ins_cost(VOLATILE_REF_COST);
7341 format %{ "ldarb $dst, $mem\t# byte" %}
7342
7343 ins_encode(aarch64_enc_ldarb(dst, mem));
7344
7345 ins_pipe(pipe_serial);
7346 %}
7347
7348 // Load Byte (8 bit unsigned) into long
7349 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7350 %{
7351 match(Set dst (ConvI2L (LoadUB mem)));
7352
7353 ins_cost(VOLATILE_REF_COST);
7354 format %{ "ldarb $dst, $mem\t# byte" %}
7355
7356 ins_encode(aarch64_enc_ldarb(dst, mem));
7357
7358 ins_pipe(pipe_serial);
7359 %}
7360
7361 // Load Short (16 bit signed)
7362 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7363 %{
7364 match(Set dst (LoadS mem));
7365
7366 ins_cost(VOLATILE_REF_COST);
7367 format %{ "ldarshw $dst, $mem\t# short" %}
7368
7369 ins_encode(aarch64_enc_ldarshw(dst, mem));
7370
7371 ins_pipe(pipe_serial);
7372 %}
7373
7374 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7375 %{
7376 match(Set dst (LoadUS mem));
7377
7378 ins_cost(VOLATILE_REF_COST);
7379 format %{ "ldarhw $dst, $mem\t# short" %}
7380
7381 ins_encode(aarch64_enc_ldarhw(dst, mem));
7382
7383 ins_pipe(pipe_serial);
7384 %}
7385
7386 // Load Short/Char (16 bit unsigned) into long
7387 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7388 %{
7389 match(Set dst (ConvI2L (LoadUS mem)));
7390
7391 ins_cost(VOLATILE_REF_COST);
7392 format %{ "ldarh $dst, $mem\t# short" %}
7393
7394 ins_encode(aarch64_enc_ldarh(dst, mem));
7395
7396 ins_pipe(pipe_serial);
7397 %}
7398
7399 // Load Short/Char (16 bit signed) into long
7400 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7401 %{
7402 match(Set dst (ConvI2L (LoadS mem)));
7403
7404 ins_cost(VOLATILE_REF_COST);
7405 format %{ "ldarh $dst, $mem\t# short" %}
7406
7407 ins_encode(aarch64_enc_ldarsh(dst, mem));
7408
7409 ins_pipe(pipe_serial);
7410 %}
7411
7412 // Load Integer (32 bit signed)
7413 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7414 %{
7415 match(Set dst (LoadI mem));
7416
7417 ins_cost(VOLATILE_REF_COST);
7418 format %{ "ldarw $dst, $mem\t# int" %}
7419
7420 ins_encode(aarch64_enc_ldarw(dst, mem));
7421
7422 ins_pipe(pipe_serial);
7423 %}
7424
7425 // Load Integer (32 bit unsigned) into long
7426 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7427 %{
7428 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7429
7430 ins_cost(VOLATILE_REF_COST);
7431 format %{ "ldarw $dst, $mem\t# int" %}
7432
7433 ins_encode(aarch64_enc_ldarw(dst, mem));
7434
7435 ins_pipe(pipe_serial);
7436 %}
7437
7438 // Load Long (64 bit signed)
7439 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7440 %{
7441 match(Set dst (LoadL mem));
7442
7443 ins_cost(VOLATILE_REF_COST);
7444 format %{ "ldar $dst, $mem\t# int" %}
7445
7446 ins_encode(aarch64_enc_ldar(dst, mem));
7447
7448 ins_pipe(pipe_serial);
7449 %}
7450
7451 // Load Pointer
7452 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7453 %{
7454 match(Set dst (LoadP mem));
7455 predicate(n->as_Load()->barrier_data() == 0);
7456
7457 ins_cost(VOLATILE_REF_COST);
7458 format %{ "ldar $dst, $mem\t# ptr" %}
7459
7460 ins_encode(aarch64_enc_ldar(dst, mem));
7461
7462 ins_pipe(pipe_serial);
7463 %}
7464
7465 // Load Compressed Pointer
7466 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7467 %{
7468 match(Set dst (LoadN mem));
7469 predicate(n->as_Load()->barrier_data() == 0);
7470
7471 ins_cost(VOLATILE_REF_COST);
7472 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7473
7474 ins_encode(aarch64_enc_ldarw(dst, mem));
7475
7476 ins_pipe(pipe_serial);
7477 %}
7478
7479 // Load Float
7480 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7481 %{
7482 match(Set dst (LoadF mem));
7483
7484 ins_cost(VOLATILE_REF_COST);
7485 format %{ "ldars $dst, $mem\t# float" %}
7486
7487 ins_encode( aarch64_enc_fldars(dst, mem) );
7488
7489 ins_pipe(pipe_serial);
7490 %}
7491
7492 // Load Double
7493 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7494 %{
7495 match(Set dst (LoadD mem));
7496
7497 ins_cost(VOLATILE_REF_COST);
7498 format %{ "ldard $dst, $mem\t# double" %}
7499
7500 ins_encode( aarch64_enc_fldard(dst, mem) );
7501
7502 ins_pipe(pipe_serial);
7503 %}
7504
7505 // Store Byte
7506 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7507 %{
7508 match(Set mem (StoreB mem src));
7509
7510 ins_cost(VOLATILE_REF_COST);
7511 format %{ "stlrb $src, $mem\t# byte" %}
7512
7513 ins_encode(aarch64_enc_stlrb(src, mem));
7514
7515 ins_pipe(pipe_class_memory);
7516 %}
7517
7518 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7519 %{
7520 match(Set mem (StoreB mem zero));
7521
7522 ins_cost(VOLATILE_REF_COST);
7523 format %{ "stlrb zr, $mem\t# byte" %}
7524
7525 ins_encode(aarch64_enc_stlrb0(mem));
7526
7527 ins_pipe(pipe_class_memory);
7528 %}
7529
7530 // Store Char/Short
7531 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7532 %{
7533 match(Set mem (StoreC mem src));
7534
7535 ins_cost(VOLATILE_REF_COST);
7536 format %{ "stlrh $src, $mem\t# short" %}
7537
7538 ins_encode(aarch64_enc_stlrh(src, mem));
7539
7540 ins_pipe(pipe_class_memory);
7541 %}
7542
7543 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7544 %{
7545 match(Set mem (StoreC mem zero));
7546
7547 ins_cost(VOLATILE_REF_COST);
7548 format %{ "stlrh zr, $mem\t# short" %}
7549
7550 ins_encode(aarch64_enc_stlrh0(mem));
7551
7552 ins_pipe(pipe_class_memory);
7553 %}
7554
7555 // Store Integer
7556
7557 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7558 %{
7559 match(Set mem(StoreI mem src));
7560
7561 ins_cost(VOLATILE_REF_COST);
7562 format %{ "stlrw $src, $mem\t# int" %}
7563
7564 ins_encode(aarch64_enc_stlrw(src, mem));
7565
7566 ins_pipe(pipe_class_memory);
7567 %}
7568
7569 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7570 %{
7571 match(Set mem(StoreI mem zero));
7572
7573 ins_cost(VOLATILE_REF_COST);
7574 format %{ "stlrw zr, $mem\t# int" %}
7575
7576 ins_encode(aarch64_enc_stlrw0(mem));
7577
7578 ins_pipe(pipe_class_memory);
7579 %}
7580
7581 // Store Long (64 bit signed)
7582 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7583 %{
7584 match(Set mem (StoreL mem src));
7585
7586 ins_cost(VOLATILE_REF_COST);
7587 format %{ "stlr $src, $mem\t# int" %}
7588
7589 ins_encode(aarch64_enc_stlr(src, mem));
7590
7591 ins_pipe(pipe_class_memory);
7592 %}
7593
7594 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7595 %{
7596 match(Set mem (StoreL mem zero));
7597
7598 ins_cost(VOLATILE_REF_COST);
7599 format %{ "stlr zr, $mem\t# int" %}
7600
7601 ins_encode(aarch64_enc_stlr0(mem));
7602
7603 ins_pipe(pipe_class_memory);
7604 %}
7605
7606 // Store Pointer
7607 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7608 %{
7609 match(Set mem (StoreP mem src));
7610 predicate(n->as_Store()->barrier_data() == 0);
7611
7612 ins_cost(VOLATILE_REF_COST);
7613 format %{ "stlr $src, $mem\t# ptr" %}
7614
7615 ins_encode(aarch64_enc_stlr(src, mem));
7616
7617 ins_pipe(pipe_class_memory);
7618 %}
7619
7620 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7621 %{
7622 match(Set mem (StoreP mem zero));
7623 predicate(n->as_Store()->barrier_data() == 0);
7624
7625 ins_cost(VOLATILE_REF_COST);
7626 format %{ "stlr zr, $mem\t# ptr" %}
7627
7628 ins_encode(aarch64_enc_stlr0(mem));
7629
7630 ins_pipe(pipe_class_memory);
7631 %}
7632
7633 // Store Compressed Pointer
7634 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7635 %{
7636 match(Set mem (StoreN mem src));
7637 predicate(n->as_Store()->barrier_data() == 0);
7638
7639 ins_cost(VOLATILE_REF_COST);
7640 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7641
7642 ins_encode(aarch64_enc_stlrw(src, mem));
7643
7644 ins_pipe(pipe_class_memory);
7645 %}
7646
7647 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7648 %{
7649 match(Set mem (StoreN mem zero));
7650 predicate(n->as_Store()->barrier_data() == 0);
7651
7652 ins_cost(VOLATILE_REF_COST);
7653 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7654
7655 ins_encode(aarch64_enc_stlrw0(mem));
7656
7657 ins_pipe(pipe_class_memory);
7658 %}
7659
7660 // Store Float
7661 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7662 %{
7663 match(Set mem (StoreF mem src));
7664
7665 ins_cost(VOLATILE_REF_COST);
7666 format %{ "stlrs $src, $mem\t# float" %}
7667
7668 ins_encode( aarch64_enc_fstlrs(src, mem) );
7669
7670 ins_pipe(pipe_class_memory);
7671 %}
7672
7673 // TODO
7674 // implement storeImmF0 and storeFImmPacked
7675
7676 // Store Double
7677 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7678 %{
7679 match(Set mem (StoreD mem src));
7680
7681 ins_cost(VOLATILE_REF_COST);
7682 format %{ "stlrd $src, $mem\t# double" %}
7683
7684 ins_encode( aarch64_enc_fstlrd(src, mem) );
7685
7686 ins_pipe(pipe_class_memory);
7687 %}
7688
7689 // ---------------- end of volatile loads and stores ----------------
7690
7691 instruct cacheWB(indirect addr)
7692 %{
7693 predicate(VM_Version::supports_data_cache_line_flush());
7694 match(CacheWB addr);
7695
7696 ins_cost(100);
7697 format %{"cache wb $addr" %}
7698 ins_encode %{
7699 assert($addr->index_position() < 0, "should be");
7700 assert($addr$$disp == 0, "should be");
7701 __ cache_wb(Address($addr$$base$$Register, 0));
7702 %}
7703 ins_pipe(pipe_slow); // XXX
7704 %}
7705
7706 instruct cacheWBPreSync()
7707 %{
7708 predicate(VM_Version::supports_data_cache_line_flush());
7709 match(CacheWBPreSync);
7710
7711 ins_cost(100);
7712 format %{"cache wb presync" %}
7713 ins_encode %{
7714 __ cache_wbsync(true);
7715 %}
7716 ins_pipe(pipe_slow); // XXX
7717 %}
7718
7719 instruct cacheWBPostSync()
7720 %{
7721 predicate(VM_Version::supports_data_cache_line_flush());
7722 match(CacheWBPostSync);
7723
7724 ins_cost(100);
7725 format %{"cache wb postsync" %}
7726 ins_encode %{
7727 __ cache_wbsync(false);
7728 %}
7729 ins_pipe(pipe_slow); // XXX
7730 %}
7731
7732 // ============================================================================
7733 // BSWAP Instructions
7734
7735 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7736 match(Set dst (ReverseBytesI src));
7737
7738 ins_cost(INSN_COST);
7739 format %{ "revw $dst, $src" %}
7740
7741 ins_encode %{
7742 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7743 %}
7744
7745 ins_pipe(ialu_reg);
7746 %}
7747
7748 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7749 match(Set dst (ReverseBytesL src));
7750
7751 ins_cost(INSN_COST);
7752 format %{ "rev $dst, $src" %}
7753
7754 ins_encode %{
7755 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7756 %}
7757
7758 ins_pipe(ialu_reg);
7759 %}
7760
7761 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7762 match(Set dst (ReverseBytesUS src));
7763
7764 ins_cost(INSN_COST);
7765 format %{ "rev16w $dst, $src" %}
7766
7767 ins_encode %{
7768 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7769 %}
7770
7771 ins_pipe(ialu_reg);
7772 %}
7773
7774 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7775 match(Set dst (ReverseBytesS src));
7776
7777 ins_cost(INSN_COST);
7778 format %{ "rev16w $dst, $src\n\t"
7779 "sbfmw $dst, $dst, #0, #15" %}
7780
7781 ins_encode %{
7782 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7783 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7784 %}
7785
7786 ins_pipe(ialu_reg);
7787 %}
7788
7789 // ============================================================================
7790 // Zero Count Instructions
7791
7792 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7793 match(Set dst (CountLeadingZerosI src));
7794
7795 ins_cost(INSN_COST);
7796 format %{ "clzw $dst, $src" %}
7797 ins_encode %{
7798 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7799 %}
7800
7801 ins_pipe(ialu_reg);
7802 %}
7803
7804 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7805 match(Set dst (CountLeadingZerosL src));
7806
7807 ins_cost(INSN_COST);
7808 format %{ "clz $dst, $src" %}
7809 ins_encode %{
7810 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7811 %}
7812
7813 ins_pipe(ialu_reg);
7814 %}
7815
7816 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7817 match(Set dst (CountTrailingZerosI src));
7818
7819 ins_cost(INSN_COST * 2);
7820 format %{ "rbitw $dst, $src\n\t"
7821 "clzw $dst, $dst" %}
7822 ins_encode %{
7823 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7824 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7825 %}
7826
7827 ins_pipe(ialu_reg);
7828 %}
7829
7830 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7831 match(Set dst (CountTrailingZerosL src));
7832
7833 ins_cost(INSN_COST * 2);
7834 format %{ "rbit $dst, $src\n\t"
7835 "clz $dst, $dst" %}
7836 ins_encode %{
7837 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7838 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7839 %}
7840
7841 ins_pipe(ialu_reg);
7842 %}
7843
7844 //---------- Population Count Instructions -------------------------------------
7845 //
7846
7847 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7848 match(Set dst (PopCountI src));
7849 effect(TEMP tmp);
7850 ins_cost(INSN_COST * 13);
7851
7852 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7853 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7854 "addv $tmp, $tmp\t# vector (8B)\n\t"
7855 "mov $dst, $tmp\t# vector (1D)" %}
7856 ins_encode %{
7857 __ fmovs($tmp$$FloatRegister, $src$$Register);
7858 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7859 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7860 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7861 %}
7862
7863 ins_pipe(pipe_class_default);
7864 %}
7865
7866 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7867 match(Set dst (PopCountI (LoadI mem)));
7868 effect(TEMP tmp);
7869 ins_cost(INSN_COST * 13);
7870
7871 format %{ "ldrs $tmp, $mem\n\t"
7872 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7873 "addv $tmp, $tmp\t# vector (8B)\n\t"
7874 "mov $dst, $tmp\t# vector (1D)" %}
7875 ins_encode %{
7876 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7877 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7878 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7879 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7880 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7881 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7882 %}
7883
7884 ins_pipe(pipe_class_default);
7885 %}
7886
7887 // Note: Long.bitCount(long) returns an int.
7888 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7889 match(Set dst (PopCountL src));
7890 effect(TEMP tmp);
7891 ins_cost(INSN_COST * 13);
7892
7893 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7894 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7895 "addv $tmp, $tmp\t# vector (8B)\n\t"
7896 "mov $dst, $tmp\t# vector (1D)" %}
7897 ins_encode %{
7898 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7899 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7900 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7901 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7902 %}
7903
7904 ins_pipe(pipe_class_default);
7905 %}
7906
7907 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7908 match(Set dst (PopCountL (LoadL mem)));
7909 effect(TEMP tmp);
7910 ins_cost(INSN_COST * 13);
7911
7912 format %{ "ldrd $tmp, $mem\n\t"
7913 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7914 "addv $tmp, $tmp\t# vector (8B)\n\t"
7915 "mov $dst, $tmp\t# vector (1D)" %}
7916 ins_encode %{
7917 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7918 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7919 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7920 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7921 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7922 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7923 %}
7924
7925 ins_pipe(pipe_class_default);
7926 %}
7927
7928 // ============================================================================
7929 // VerifyVectorAlignment Instruction
7930
7931 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7932 match(Set addr (VerifyVectorAlignment addr mask));
7933 effect(KILL cr);
7934 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7935 ins_encode %{
7936 Label Lskip;
7937 // check if masked bits of addr are zero
7938 __ tst($addr$$Register, $mask$$constant);
7939 __ br(Assembler::EQ, Lskip);
7940 __ stop("verify_vector_alignment found a misaligned vector memory access");
7941 __ bind(Lskip);
7942 %}
7943 ins_pipe(pipe_slow);
7944 %}
7945
7946 // ============================================================================
7947 // MemBar Instruction
7948
7949 instruct load_fence() %{
7950 match(LoadFence);
7951 ins_cost(VOLATILE_REF_COST);
7952
7953 format %{ "load_fence" %}
7954
7955 ins_encode %{
7956 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7957 %}
7958 ins_pipe(pipe_serial);
7959 %}
7960
7961 instruct unnecessary_membar_acquire() %{
7962 predicate(unnecessary_acquire(n));
7963 match(MemBarAcquire);
7964 ins_cost(0);
7965
7966 format %{ "membar_acquire (elided)" %}
7967
7968 ins_encode %{
7969 __ block_comment("membar_acquire (elided)");
7970 %}
7971
7972 ins_pipe(pipe_class_empty);
7973 %}
7974
7975 instruct membar_acquire() %{
7976 match(MemBarAcquire);
7977 ins_cost(VOLATILE_REF_COST);
7978
7979 format %{ "membar_acquire\n\t"
7980 "dmb ishld" %}
7981
7982 ins_encode %{
7983 __ block_comment("membar_acquire");
7984 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7985 %}
7986
7987 ins_pipe(pipe_serial);
7988 %}
7989
7990
7991 instruct membar_acquire_lock() %{
7992 match(MemBarAcquireLock);
7993 ins_cost(VOLATILE_REF_COST);
7994
7995 format %{ "membar_acquire_lock (elided)" %}
7996
7997 ins_encode %{
7998 __ block_comment("membar_acquire_lock (elided)");
7999 %}
8000
8001 ins_pipe(pipe_serial);
8002 %}
8003
8004 instruct store_fence() %{
8005 match(StoreFence);
8006 ins_cost(VOLATILE_REF_COST);
8007
8008 format %{ "store_fence" %}
8009
8010 ins_encode %{
8011 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8012 %}
8013 ins_pipe(pipe_serial);
8014 %}
8015
8016 instruct unnecessary_membar_release() %{
8017 predicate(unnecessary_release(n));
8018 match(MemBarRelease);
8019 ins_cost(0);
8020
8021 format %{ "membar_release (elided)" %}
8022
8023 ins_encode %{
8024 __ block_comment("membar_release (elided)");
8025 %}
8026 ins_pipe(pipe_serial);
8027 %}
8028
8029 instruct membar_release() %{
8030 match(MemBarRelease);
8031 ins_cost(VOLATILE_REF_COST);
8032
8033 format %{ "membar_release\n\t"
8034 "dmb ishst\n\tdmb ishld" %}
8035
8036 ins_encode %{
8037 __ block_comment("membar_release");
8038 // These will be merged if AlwaysMergeDMB is enabled.
8039 __ membar(Assembler::StoreStore);
8040 __ membar(Assembler::LoadStore);
8041 %}
8042 ins_pipe(pipe_serial);
8043 %}
8044
8045 instruct membar_storestore() %{
8046 match(MemBarStoreStore);
8047 match(StoreStoreFence);
8048 ins_cost(VOLATILE_REF_COST);
8049
8050 format %{ "MEMBAR-store-store" %}
8051
8052 ins_encode %{
8053 __ membar(Assembler::StoreStore);
8054 %}
8055 ins_pipe(pipe_serial);
8056 %}
8057
8058 instruct membar_release_lock() %{
8059 match(MemBarReleaseLock);
8060 ins_cost(VOLATILE_REF_COST);
8061
8062 format %{ "membar_release_lock (elided)" %}
8063
8064 ins_encode %{
8065 __ block_comment("membar_release_lock (elided)");
8066 %}
8067
8068 ins_pipe(pipe_serial);
8069 %}
8070
8071 instruct unnecessary_membar_volatile() %{
8072 predicate(unnecessary_volatile(n));
8073 match(MemBarVolatile);
8074 ins_cost(0);
8075
8076 format %{ "membar_volatile (elided)" %}
8077
8078 ins_encode %{
8079 __ block_comment("membar_volatile (elided)");
8080 %}
8081
8082 ins_pipe(pipe_serial);
8083 %}
8084
8085 instruct membar_volatile() %{
8086 match(MemBarVolatile);
8087 ins_cost(VOLATILE_REF_COST*100);
8088
8089 format %{ "membar_volatile\n\t"
8090 "dmb ish"%}
8091
8092 ins_encode %{
8093 __ block_comment("membar_volatile");
8094 __ membar(Assembler::StoreLoad);
8095 %}
8096
8097 ins_pipe(pipe_serial);
8098 %}
8099
8100 // ============================================================================
8101 // Cast/Convert Instructions
8102
8103 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8104 match(Set dst (CastX2P src));
8105
8106 ins_cost(INSN_COST);
8107 format %{ "mov $dst, $src\t# long -> ptr" %}
8108
8109 ins_encode %{
8110 if ($dst$$reg != $src$$reg) {
8111 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8112 }
8113 %}
8114
8115 ins_pipe(ialu_reg);
8116 %}
8117
8118 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8119 match(Set dst (CastP2X src));
8120
8121 ins_cost(INSN_COST);
8122 format %{ "mov $dst, $src\t# ptr -> long" %}
8123
8124 ins_encode %{
8125 if ($dst$$reg != $src$$reg) {
8126 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8127 }
8128 %}
8129
8130 ins_pipe(ialu_reg);
8131 %}
8132
8133 // Convert oop into int for vectors alignment masking
8134 instruct convP2I(iRegINoSp dst, iRegP src) %{
8135 match(Set dst (ConvL2I (CastP2X src)));
8136
8137 ins_cost(INSN_COST);
8138 format %{ "movw $dst, $src\t# ptr -> int" %}
8139 ins_encode %{
8140 __ movw($dst$$Register, $src$$Register);
8141 %}
8142
8143 ins_pipe(ialu_reg);
8144 %}
8145
8146 // Convert compressed oop into int for vectors alignment masking
8147 // in case of 32bit oops (heap < 4Gb).
8148 instruct convN2I(iRegINoSp dst, iRegN src)
8149 %{
8150 predicate(CompressedOops::shift() == 0);
8151 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8152
8153 ins_cost(INSN_COST);
8154 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8155 ins_encode %{
8156 __ movw($dst$$Register, $src$$Register);
8157 %}
8158
8159 ins_pipe(ialu_reg);
8160 %}
8161
8162
8163 // Convert oop pointer into compressed form
8164 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8165 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8166 match(Set dst (EncodeP src));
8167 effect(KILL cr);
8168 ins_cost(INSN_COST * 3);
8169 format %{ "encode_heap_oop $dst, $src" %}
8170 ins_encode %{
8171 Register s = $src$$Register;
8172 Register d = $dst$$Register;
8173 __ encode_heap_oop(d, s);
8174 %}
8175 ins_pipe(ialu_reg);
8176 %}
8177
8178 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8179 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8180 match(Set dst (EncodeP src));
8181 ins_cost(INSN_COST * 3);
8182 format %{ "encode_heap_oop_not_null $dst, $src" %}
8183 ins_encode %{
8184 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8185 %}
8186 ins_pipe(ialu_reg);
8187 %}
8188
8189 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8190 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8191 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8192 match(Set dst (DecodeN src));
8193 ins_cost(INSN_COST * 3);
8194 format %{ "decode_heap_oop $dst, $src" %}
8195 ins_encode %{
8196 Register s = $src$$Register;
8197 Register d = $dst$$Register;
8198 __ decode_heap_oop(d, s);
8199 %}
8200 ins_pipe(ialu_reg);
8201 %}
8202
8203 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8204 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8205 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8206 match(Set dst (DecodeN src));
8207 ins_cost(INSN_COST * 3);
8208 format %{ "decode_heap_oop_not_null $dst, $src" %}
8209 ins_encode %{
8210 Register s = $src$$Register;
8211 Register d = $dst$$Register;
8212 __ decode_heap_oop_not_null(d, s);
8213 %}
8214 ins_pipe(ialu_reg);
8215 %}
8216
8217 // n.b. AArch64 implementations of encode_klass_not_null and
8218 // decode_klass_not_null do not modify the flags register so, unlike
8219 // Intel, we don't kill CR as a side effect here
8220
8221 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8222 match(Set dst (EncodePKlass src));
8223
8224 ins_cost(INSN_COST * 3);
8225 format %{ "encode_klass_not_null $dst,$src" %}
8226
8227 ins_encode %{
8228 Register src_reg = as_Register($src$$reg);
8229 Register dst_reg = as_Register($dst$$reg);
8230 __ encode_klass_not_null(dst_reg, src_reg);
8231 %}
8232
8233 ins_pipe(ialu_reg);
8234 %}
8235
8236 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8237 match(Set dst (DecodeNKlass src));
8238
8239 ins_cost(INSN_COST * 3);
8240 format %{ "decode_klass_not_null $dst,$src" %}
8241
8242 ins_encode %{
8243 Register src_reg = as_Register($src$$reg);
8244 Register dst_reg = as_Register($dst$$reg);
8245 if (dst_reg != src_reg) {
8246 __ decode_klass_not_null(dst_reg, src_reg);
8247 } else {
8248 __ decode_klass_not_null(dst_reg);
8249 }
8250 %}
8251
8252 ins_pipe(ialu_reg);
8253 %}
8254
8255 instruct checkCastPP(iRegPNoSp dst)
8256 %{
8257 match(Set dst (CheckCastPP dst));
8258
8259 size(0);
8260 format %{ "# checkcastPP of $dst" %}
8261 ins_encode(/* empty encoding */);
8262 ins_pipe(pipe_class_empty);
8263 %}
8264
8265 instruct castPP(iRegPNoSp dst)
8266 %{
8267 match(Set dst (CastPP dst));
8268
8269 size(0);
8270 format %{ "# castPP of $dst" %}
8271 ins_encode(/* empty encoding */);
8272 ins_pipe(pipe_class_empty);
8273 %}
8274
8275 instruct castII(iRegI dst)
8276 %{
8277 predicate(VerifyConstraintCasts == 0);
8278 match(Set dst (CastII dst));
8279
8280 size(0);
8281 format %{ "# castII of $dst" %}
8282 ins_encode(/* empty encoding */);
8283 ins_cost(0);
8284 ins_pipe(pipe_class_empty);
8285 %}
8286
8287 instruct castII_checked(iRegI dst, rFlagsReg cr)
8288 %{
8289 predicate(VerifyConstraintCasts > 0);
8290 match(Set dst (CastII dst));
8291 effect(KILL cr);
8292
8293 format %{ "# castII_checked of $dst" %}
8294 ins_encode %{
8295 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8296 %}
8297 ins_pipe(pipe_slow);
8298 %}
8299
8300 instruct castLL(iRegL dst)
8301 %{
8302 predicate(VerifyConstraintCasts == 0);
8303 match(Set dst (CastLL dst));
8304
8305 size(0);
8306 format %{ "# castLL of $dst" %}
8307 ins_encode(/* empty encoding */);
8308 ins_cost(0);
8309 ins_pipe(pipe_class_empty);
8310 %}
8311
8312 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8313 %{
8314 predicate(VerifyConstraintCasts > 0);
8315 match(Set dst (CastLL dst));
8316 effect(KILL cr);
8317
8318 format %{ "# castLL_checked of $dst" %}
8319 ins_encode %{
8320 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8321 %}
8322 ins_pipe(pipe_slow);
8323 %}
8324
8325 instruct castHH(vRegF dst)
8326 %{
8327 match(Set dst (CastHH dst));
8328 size(0);
8329 format %{ "# castHH of $dst" %}
8330 ins_encode(/* empty encoding */);
8331 ins_cost(0);
8332 ins_pipe(pipe_class_empty);
8333 %}
8334
8335 instruct castFF(vRegF dst)
8336 %{
8337 match(Set dst (CastFF dst));
8338
8339 size(0);
8340 format %{ "# castFF of $dst" %}
8341 ins_encode(/* empty encoding */);
8342 ins_cost(0);
8343 ins_pipe(pipe_class_empty);
8344 %}
8345
8346 instruct castDD(vRegD dst)
8347 %{
8348 match(Set dst (CastDD dst));
8349
8350 size(0);
8351 format %{ "# castDD of $dst" %}
8352 ins_encode(/* empty encoding */);
8353 ins_cost(0);
8354 ins_pipe(pipe_class_empty);
8355 %}
8356
8357 instruct castVV(vReg dst)
8358 %{
8359 match(Set dst (CastVV dst));
8360
8361 size(0);
8362 format %{ "# castVV of $dst" %}
8363 ins_encode(/* empty encoding */);
8364 ins_cost(0);
8365 ins_pipe(pipe_class_empty);
8366 %}
8367
8368 instruct castVVMask(pRegGov dst)
8369 %{
8370 match(Set dst (CastVV dst));
8371
8372 size(0);
8373 format %{ "# castVV of $dst" %}
8374 ins_encode(/* empty encoding */);
8375 ins_cost(0);
8376 ins_pipe(pipe_class_empty);
8377 %}
8378
8379 // ============================================================================
8380 // Atomic operation instructions
8381 //
8382
8383 // standard CompareAndSwapX when we are using barriers
8384 // these have higher priority than the rules selected by a predicate
8385
8386 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8387 // can't match them
8388
8389 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8390
8391 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8392 ins_cost(2 * VOLATILE_REF_COST);
8393
8394 effect(KILL cr);
8395
8396 format %{
8397 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8398 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8399 %}
8400
8401 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8402 aarch64_enc_cset_eq(res));
8403
8404 ins_pipe(pipe_slow);
8405 %}
8406
8407 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8408
8409 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8410 ins_cost(2 * VOLATILE_REF_COST);
8411
8412 effect(KILL cr);
8413
8414 format %{
8415 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8416 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8417 %}
8418
8419 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8420 aarch64_enc_cset_eq(res));
8421
8422 ins_pipe(pipe_slow);
8423 %}
8424
8425 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8426
8427 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8428 ins_cost(2 * VOLATILE_REF_COST);
8429
8430 effect(KILL cr);
8431
8432 format %{
8433 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8434 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8435 %}
8436
8437 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8438 aarch64_enc_cset_eq(res));
8439
8440 ins_pipe(pipe_slow);
8441 %}
8442
8443 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8444
8445 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8446 ins_cost(2 * VOLATILE_REF_COST);
8447
8448 effect(KILL cr);
8449
8450 format %{
8451 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8452 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8453 %}
8454
8455 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8456 aarch64_enc_cset_eq(res));
8457
8458 ins_pipe(pipe_slow);
8459 %}
8460
8461 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8462
8463 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8464 predicate(n->as_LoadStore()->barrier_data() == 0);
8465 ins_cost(2 * VOLATILE_REF_COST);
8466
8467 effect(KILL cr);
8468
8469 format %{
8470 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8471 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8472 %}
8473
8474 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8475 aarch64_enc_cset_eq(res));
8476
8477 ins_pipe(pipe_slow);
8478 %}
8479
8480 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8481
8482 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8483 predicate(n->as_LoadStore()->barrier_data() == 0);
8484 ins_cost(2 * VOLATILE_REF_COST);
8485
8486 effect(KILL cr);
8487
8488 format %{
8489 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8490 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8491 %}
8492
8493 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8494 aarch64_enc_cset_eq(res));
8495
8496 ins_pipe(pipe_slow);
8497 %}
8498
8499 // alternative CompareAndSwapX when we are eliding barriers
8500
8501 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8502
8503 predicate(needs_acquiring_load_exclusive(n));
8504 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8505 ins_cost(VOLATILE_REF_COST);
8506
8507 effect(KILL cr);
8508
8509 format %{
8510 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8511 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8512 %}
8513
8514 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8515 aarch64_enc_cset_eq(res));
8516
8517 ins_pipe(pipe_slow);
8518 %}
8519
8520 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8521
8522 predicate(needs_acquiring_load_exclusive(n));
8523 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8524 ins_cost(VOLATILE_REF_COST);
8525
8526 effect(KILL cr);
8527
8528 format %{
8529 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8530 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8531 %}
8532
8533 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8534 aarch64_enc_cset_eq(res));
8535
8536 ins_pipe(pipe_slow);
8537 %}
8538
8539 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8540
8541 predicate(needs_acquiring_load_exclusive(n));
8542 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8543 ins_cost(VOLATILE_REF_COST);
8544
8545 effect(KILL cr);
8546
8547 format %{
8548 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8549 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8550 %}
8551
8552 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8553 aarch64_enc_cset_eq(res));
8554
8555 ins_pipe(pipe_slow);
8556 %}
8557
8558 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8559
8560 predicate(needs_acquiring_load_exclusive(n));
8561 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8562 ins_cost(VOLATILE_REF_COST);
8563
8564 effect(KILL cr);
8565
8566 format %{
8567 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8568 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8569 %}
8570
8571 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8572 aarch64_enc_cset_eq(res));
8573
8574 ins_pipe(pipe_slow);
8575 %}
8576
8577 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8578
8579 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8580 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8581 ins_cost(VOLATILE_REF_COST);
8582
8583 effect(KILL cr);
8584
8585 format %{
8586 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8587 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8588 %}
8589
8590 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8591 aarch64_enc_cset_eq(res));
8592
8593 ins_pipe(pipe_slow);
8594 %}
8595
8596 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8597
8598 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8599 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8600 ins_cost(VOLATILE_REF_COST);
8601
8602 effect(KILL cr);
8603
8604 format %{
8605 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8606 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8607 %}
8608
8609 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8610 aarch64_enc_cset_eq(res));
8611
8612 ins_pipe(pipe_slow);
8613 %}
8614
8615
8616 // ---------------------------------------------------------------------
8617
8618 // BEGIN This section of the file is automatically generated. Do not edit --------------
8619
8620 // Sundry CAS operations. Note that release is always true,
8621 // regardless of the memory ordering of the CAS. This is because we
8622 // need the volatile case to be sequentially consistent but there is
8623 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8624 // can't check the type of memory ordering here, so we always emit a
8625 // STLXR.
8626
8627 // This section is generated from cas.m4
8628
8629
8630 // This pattern is generated automatically from cas.m4.
8631 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8632 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8633 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8634 ins_cost(2 * VOLATILE_REF_COST);
8635 effect(TEMP_DEF res, KILL cr);
8636 format %{
8637 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8638 %}
8639 ins_encode %{
8640 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8641 Assembler::byte, /*acquire*/ false, /*release*/ true,
8642 /*weak*/ false, $res$$Register);
8643 __ sxtbw($res$$Register, $res$$Register);
8644 %}
8645 ins_pipe(pipe_slow);
8646 %}
8647
8648 // This pattern is generated automatically from cas.m4.
8649 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8650 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8651 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8652 ins_cost(2 * VOLATILE_REF_COST);
8653 effect(TEMP_DEF res, KILL cr);
8654 format %{
8655 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8656 %}
8657 ins_encode %{
8658 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8659 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8660 /*weak*/ false, $res$$Register);
8661 __ sxthw($res$$Register, $res$$Register);
8662 %}
8663 ins_pipe(pipe_slow);
8664 %}
8665
8666 // This pattern is generated automatically from cas.m4.
8667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8668 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8669 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8670 ins_cost(2 * VOLATILE_REF_COST);
8671 effect(TEMP_DEF res, KILL cr);
8672 format %{
8673 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8674 %}
8675 ins_encode %{
8676 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8677 Assembler::word, /*acquire*/ false, /*release*/ true,
8678 /*weak*/ false, $res$$Register);
8679 %}
8680 ins_pipe(pipe_slow);
8681 %}
8682
8683 // This pattern is generated automatically from cas.m4.
8684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8685 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8686 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8687 ins_cost(2 * VOLATILE_REF_COST);
8688 effect(TEMP_DEF res, KILL cr);
8689 format %{
8690 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8691 %}
8692 ins_encode %{
8693 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8694 Assembler::xword, /*acquire*/ false, /*release*/ true,
8695 /*weak*/ false, $res$$Register);
8696 %}
8697 ins_pipe(pipe_slow);
8698 %}
8699
8700 // This pattern is generated automatically from cas.m4.
8701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8702 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8703 predicate(n->as_LoadStore()->barrier_data() == 0);
8704 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8705 ins_cost(2 * VOLATILE_REF_COST);
8706 effect(TEMP_DEF res, KILL cr);
8707 format %{
8708 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8709 %}
8710 ins_encode %{
8711 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8712 Assembler::word, /*acquire*/ false, /*release*/ true,
8713 /*weak*/ false, $res$$Register);
8714 %}
8715 ins_pipe(pipe_slow);
8716 %}
8717
8718 // This pattern is generated automatically from cas.m4.
8719 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8720 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8721 predicate(n->as_LoadStore()->barrier_data() == 0);
8722 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8723 ins_cost(2 * VOLATILE_REF_COST);
8724 effect(TEMP_DEF res, KILL cr);
8725 format %{
8726 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8727 %}
8728 ins_encode %{
8729 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8730 Assembler::xword, /*acquire*/ false, /*release*/ true,
8731 /*weak*/ false, $res$$Register);
8732 %}
8733 ins_pipe(pipe_slow);
8734 %}
8735
8736 // This pattern is generated automatically from cas.m4.
8737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8738 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8739 predicate(needs_acquiring_load_exclusive(n));
8740 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8741 ins_cost(VOLATILE_REF_COST);
8742 effect(TEMP_DEF res, KILL cr);
8743 format %{
8744 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8745 %}
8746 ins_encode %{
8747 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8748 Assembler::byte, /*acquire*/ true, /*release*/ true,
8749 /*weak*/ false, $res$$Register);
8750 __ sxtbw($res$$Register, $res$$Register);
8751 %}
8752 ins_pipe(pipe_slow);
8753 %}
8754
8755 // This pattern is generated automatically from cas.m4.
8756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8757 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8758 predicate(needs_acquiring_load_exclusive(n));
8759 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8760 ins_cost(VOLATILE_REF_COST);
8761 effect(TEMP_DEF res, KILL cr);
8762 format %{
8763 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8764 %}
8765 ins_encode %{
8766 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8767 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8768 /*weak*/ false, $res$$Register);
8769 __ sxthw($res$$Register, $res$$Register);
8770 %}
8771 ins_pipe(pipe_slow);
8772 %}
8773
8774 // This pattern is generated automatically from cas.m4.
8775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8776 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8777 predicate(needs_acquiring_load_exclusive(n));
8778 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8779 ins_cost(VOLATILE_REF_COST);
8780 effect(TEMP_DEF res, KILL cr);
8781 format %{
8782 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8783 %}
8784 ins_encode %{
8785 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8786 Assembler::word, /*acquire*/ true, /*release*/ true,
8787 /*weak*/ false, $res$$Register);
8788 %}
8789 ins_pipe(pipe_slow);
8790 %}
8791
8792 // This pattern is generated automatically from cas.m4.
8793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8794 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8795 predicate(needs_acquiring_load_exclusive(n));
8796 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8797 ins_cost(VOLATILE_REF_COST);
8798 effect(TEMP_DEF res, KILL cr);
8799 format %{
8800 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8801 %}
8802 ins_encode %{
8803 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8804 Assembler::xword, /*acquire*/ true, /*release*/ true,
8805 /*weak*/ false, $res$$Register);
8806 %}
8807 ins_pipe(pipe_slow);
8808 %}
8809
8810 // This pattern is generated automatically from cas.m4.
8811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8812 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8813 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8814 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8815 ins_cost(VOLATILE_REF_COST);
8816 effect(TEMP_DEF res, KILL cr);
8817 format %{
8818 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8819 %}
8820 ins_encode %{
8821 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8822 Assembler::word, /*acquire*/ true, /*release*/ true,
8823 /*weak*/ false, $res$$Register);
8824 %}
8825 ins_pipe(pipe_slow);
8826 %}
8827
8828 // This pattern is generated automatically from cas.m4.
8829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8830 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8831 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8832 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8833 ins_cost(VOLATILE_REF_COST);
8834 effect(TEMP_DEF res, KILL cr);
8835 format %{
8836 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8837 %}
8838 ins_encode %{
8839 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8840 Assembler::xword, /*acquire*/ true, /*release*/ true,
8841 /*weak*/ false, $res$$Register);
8842 %}
8843 ins_pipe(pipe_slow);
8844 %}
8845
8846 // This pattern is generated automatically from cas.m4.
8847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8848 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8849 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8850 ins_cost(2 * VOLATILE_REF_COST);
8851 effect(KILL cr);
8852 format %{
8853 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8854 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8855 %}
8856 ins_encode %{
8857 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8858 Assembler::byte, /*acquire*/ false, /*release*/ true,
8859 /*weak*/ true, noreg);
8860 __ csetw($res$$Register, Assembler::EQ);
8861 %}
8862 ins_pipe(pipe_slow);
8863 %}
8864
8865 // This pattern is generated automatically from cas.m4.
8866 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8867 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8868 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8869 ins_cost(2 * VOLATILE_REF_COST);
8870 effect(KILL cr);
8871 format %{
8872 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8873 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8874 %}
8875 ins_encode %{
8876 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8877 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8878 /*weak*/ true, noreg);
8879 __ csetw($res$$Register, Assembler::EQ);
8880 %}
8881 ins_pipe(pipe_slow);
8882 %}
8883
8884 // This pattern is generated automatically from cas.m4.
8885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8886 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8887 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8888 ins_cost(2 * VOLATILE_REF_COST);
8889 effect(KILL cr);
8890 format %{
8891 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8892 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8893 %}
8894 ins_encode %{
8895 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8896 Assembler::word, /*acquire*/ false, /*release*/ true,
8897 /*weak*/ true, noreg);
8898 __ csetw($res$$Register, Assembler::EQ);
8899 %}
8900 ins_pipe(pipe_slow);
8901 %}
8902
8903 // This pattern is generated automatically from cas.m4.
8904 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8905 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8906 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8907 ins_cost(2 * VOLATILE_REF_COST);
8908 effect(KILL cr);
8909 format %{
8910 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8911 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8912 %}
8913 ins_encode %{
8914 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8915 Assembler::xword, /*acquire*/ false, /*release*/ true,
8916 /*weak*/ true, noreg);
8917 __ csetw($res$$Register, Assembler::EQ);
8918 %}
8919 ins_pipe(pipe_slow);
8920 %}
8921
8922 // This pattern is generated automatically from cas.m4.
8923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8924 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8925 predicate(n->as_LoadStore()->barrier_data() == 0);
8926 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8927 ins_cost(2 * VOLATILE_REF_COST);
8928 effect(KILL cr);
8929 format %{
8930 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8931 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8932 %}
8933 ins_encode %{
8934 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8935 Assembler::word, /*acquire*/ false, /*release*/ true,
8936 /*weak*/ true, noreg);
8937 __ csetw($res$$Register, Assembler::EQ);
8938 %}
8939 ins_pipe(pipe_slow);
8940 %}
8941
8942 // This pattern is generated automatically from cas.m4.
8943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8944 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8945 predicate(n->as_LoadStore()->barrier_data() == 0);
8946 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8947 ins_cost(2 * VOLATILE_REF_COST);
8948 effect(KILL cr);
8949 format %{
8950 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8951 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8952 %}
8953 ins_encode %{
8954 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8955 Assembler::xword, /*acquire*/ false, /*release*/ true,
8956 /*weak*/ true, noreg);
8957 __ csetw($res$$Register, Assembler::EQ);
8958 %}
8959 ins_pipe(pipe_slow);
8960 %}
8961
8962 // This pattern is generated automatically from cas.m4.
8963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8964 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8965 predicate(needs_acquiring_load_exclusive(n));
8966 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8967 ins_cost(VOLATILE_REF_COST);
8968 effect(KILL cr);
8969 format %{
8970 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8971 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8972 %}
8973 ins_encode %{
8974 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8975 Assembler::byte, /*acquire*/ true, /*release*/ true,
8976 /*weak*/ true, noreg);
8977 __ csetw($res$$Register, Assembler::EQ);
8978 %}
8979 ins_pipe(pipe_slow);
8980 %}
8981
8982 // This pattern is generated automatically from cas.m4.
8983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8984 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8985 predicate(needs_acquiring_load_exclusive(n));
8986 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8987 ins_cost(VOLATILE_REF_COST);
8988 effect(KILL cr);
8989 format %{
8990 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8991 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8992 %}
8993 ins_encode %{
8994 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8995 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8996 /*weak*/ true, noreg);
8997 __ csetw($res$$Register, Assembler::EQ);
8998 %}
8999 ins_pipe(pipe_slow);
9000 %}
9001
9002 // This pattern is generated automatically from cas.m4.
9003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9004 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9005 predicate(needs_acquiring_load_exclusive(n));
9006 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9007 ins_cost(VOLATILE_REF_COST);
9008 effect(KILL cr);
9009 format %{
9010 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9011 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9012 %}
9013 ins_encode %{
9014 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9015 Assembler::word, /*acquire*/ true, /*release*/ true,
9016 /*weak*/ true, noreg);
9017 __ csetw($res$$Register, Assembler::EQ);
9018 %}
9019 ins_pipe(pipe_slow);
9020 %}
9021
9022 // This pattern is generated automatically from cas.m4.
9023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9024 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9025 predicate(needs_acquiring_load_exclusive(n));
9026 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9027 ins_cost(VOLATILE_REF_COST);
9028 effect(KILL cr);
9029 format %{
9030 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9031 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9032 %}
9033 ins_encode %{
9034 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9035 Assembler::xword, /*acquire*/ true, /*release*/ true,
9036 /*weak*/ true, noreg);
9037 __ csetw($res$$Register, Assembler::EQ);
9038 %}
9039 ins_pipe(pipe_slow);
9040 %}
9041
9042 // This pattern is generated automatically from cas.m4.
9043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9044 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9045 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9046 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9047 ins_cost(VOLATILE_REF_COST);
9048 effect(KILL cr);
9049 format %{
9050 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9051 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9052 %}
9053 ins_encode %{
9054 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9055 Assembler::word, /*acquire*/ true, /*release*/ true,
9056 /*weak*/ true, noreg);
9057 __ csetw($res$$Register, Assembler::EQ);
9058 %}
9059 ins_pipe(pipe_slow);
9060 %}
9061
9062 // This pattern is generated automatically from cas.m4.
9063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9064 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9065 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9066 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9067 ins_cost(VOLATILE_REF_COST);
9068 effect(KILL cr);
9069 format %{
9070 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9071 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9072 %}
9073 ins_encode %{
9074 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9075 Assembler::xword, /*acquire*/ true, /*release*/ true,
9076 /*weak*/ true, noreg);
9077 __ csetw($res$$Register, Assembler::EQ);
9078 %}
9079 ins_pipe(pipe_slow);
9080 %}
9081
9082 // END This section of the file is automatically generated. Do not edit --------------
9083 // ---------------------------------------------------------------------
9084
9085 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9086 match(Set prev (GetAndSetI mem newv));
9087 ins_cost(2 * VOLATILE_REF_COST);
9088 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9089 ins_encode %{
9090 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9091 %}
9092 ins_pipe(pipe_serial);
9093 %}
9094
9095 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9096 match(Set prev (GetAndSetL mem newv));
9097 ins_cost(2 * VOLATILE_REF_COST);
9098 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9099 ins_encode %{
9100 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9101 %}
9102 ins_pipe(pipe_serial);
9103 %}
9104
9105 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9106 predicate(n->as_LoadStore()->barrier_data() == 0);
9107 match(Set prev (GetAndSetN mem newv));
9108 ins_cost(2 * VOLATILE_REF_COST);
9109 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9110 ins_encode %{
9111 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9112 %}
9113 ins_pipe(pipe_serial);
9114 %}
9115
9116 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9117 predicate(n->as_LoadStore()->barrier_data() == 0);
9118 match(Set prev (GetAndSetP mem newv));
9119 ins_cost(2 * VOLATILE_REF_COST);
9120 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9121 ins_encode %{
9122 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9123 %}
9124 ins_pipe(pipe_serial);
9125 %}
9126
9127 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9128 predicate(needs_acquiring_load_exclusive(n));
9129 match(Set prev (GetAndSetI mem newv));
9130 ins_cost(VOLATILE_REF_COST);
9131 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9132 ins_encode %{
9133 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9134 %}
9135 ins_pipe(pipe_serial);
9136 %}
9137
9138 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9139 predicate(needs_acquiring_load_exclusive(n));
9140 match(Set prev (GetAndSetL mem newv));
9141 ins_cost(VOLATILE_REF_COST);
9142 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9143 ins_encode %{
9144 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9145 %}
9146 ins_pipe(pipe_serial);
9147 %}
9148
9149 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9150 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9151 match(Set prev (GetAndSetN mem newv));
9152 ins_cost(VOLATILE_REF_COST);
9153 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9154 ins_encode %{
9155 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9156 %}
9157 ins_pipe(pipe_serial);
9158 %}
9159
9160 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9161 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9162 match(Set prev (GetAndSetP mem newv));
9163 ins_cost(VOLATILE_REF_COST);
9164 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9165 ins_encode %{
9166 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9167 %}
9168 ins_pipe(pipe_serial);
9169 %}
9170
9171
9172 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9173 match(Set newval (GetAndAddL mem incr));
9174 ins_cost(2 * VOLATILE_REF_COST + 1);
9175 format %{ "get_and_addL $newval, [$mem], $incr" %}
9176 ins_encode %{
9177 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9178 %}
9179 ins_pipe(pipe_serial);
9180 %}
9181
9182 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9183 predicate(n->as_LoadStore()->result_not_used());
9184 match(Set dummy (GetAndAddL mem incr));
9185 ins_cost(2 * VOLATILE_REF_COST);
9186 format %{ "get_and_addL [$mem], $incr" %}
9187 ins_encode %{
9188 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9189 %}
9190 ins_pipe(pipe_serial);
9191 %}
9192
9193 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9194 match(Set newval (GetAndAddL mem incr));
9195 ins_cost(2 * VOLATILE_REF_COST + 1);
9196 format %{ "get_and_addL $newval, [$mem], $incr" %}
9197 ins_encode %{
9198 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9199 %}
9200 ins_pipe(pipe_serial);
9201 %}
9202
9203 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9204 predicate(n->as_LoadStore()->result_not_used());
9205 match(Set dummy (GetAndAddL mem incr));
9206 ins_cost(2 * VOLATILE_REF_COST);
9207 format %{ "get_and_addL [$mem], $incr" %}
9208 ins_encode %{
9209 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9210 %}
9211 ins_pipe(pipe_serial);
9212 %}
9213
9214 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9215 match(Set newval (GetAndAddI mem incr));
9216 ins_cost(2 * VOLATILE_REF_COST + 1);
9217 format %{ "get_and_addI $newval, [$mem], $incr" %}
9218 ins_encode %{
9219 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9220 %}
9221 ins_pipe(pipe_serial);
9222 %}
9223
9224 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9225 predicate(n->as_LoadStore()->result_not_used());
9226 match(Set dummy (GetAndAddI mem incr));
9227 ins_cost(2 * VOLATILE_REF_COST);
9228 format %{ "get_and_addI [$mem], $incr" %}
9229 ins_encode %{
9230 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9231 %}
9232 ins_pipe(pipe_serial);
9233 %}
9234
9235 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9236 match(Set newval (GetAndAddI mem incr));
9237 ins_cost(2 * VOLATILE_REF_COST + 1);
9238 format %{ "get_and_addI $newval, [$mem], $incr" %}
9239 ins_encode %{
9240 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9241 %}
9242 ins_pipe(pipe_serial);
9243 %}
9244
9245 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9246 predicate(n->as_LoadStore()->result_not_used());
9247 match(Set dummy (GetAndAddI mem incr));
9248 ins_cost(2 * VOLATILE_REF_COST);
9249 format %{ "get_and_addI [$mem], $incr" %}
9250 ins_encode %{
9251 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9252 %}
9253 ins_pipe(pipe_serial);
9254 %}
9255
9256 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9257 predicate(needs_acquiring_load_exclusive(n));
9258 match(Set newval (GetAndAddL mem incr));
9259 ins_cost(VOLATILE_REF_COST + 1);
9260 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9261 ins_encode %{
9262 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9263 %}
9264 ins_pipe(pipe_serial);
9265 %}
9266
9267 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9268 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9269 match(Set dummy (GetAndAddL mem incr));
9270 ins_cost(VOLATILE_REF_COST);
9271 format %{ "get_and_addL_acq [$mem], $incr" %}
9272 ins_encode %{
9273 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9274 %}
9275 ins_pipe(pipe_serial);
9276 %}
9277
9278 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9279 predicate(needs_acquiring_load_exclusive(n));
9280 match(Set newval (GetAndAddL mem incr));
9281 ins_cost(VOLATILE_REF_COST + 1);
9282 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9283 ins_encode %{
9284 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9285 %}
9286 ins_pipe(pipe_serial);
9287 %}
9288
9289 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9290 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9291 match(Set dummy (GetAndAddL mem incr));
9292 ins_cost(VOLATILE_REF_COST);
9293 format %{ "get_and_addL_acq [$mem], $incr" %}
9294 ins_encode %{
9295 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9296 %}
9297 ins_pipe(pipe_serial);
9298 %}
9299
9300 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9301 predicate(needs_acquiring_load_exclusive(n));
9302 match(Set newval (GetAndAddI mem incr));
9303 ins_cost(VOLATILE_REF_COST + 1);
9304 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9305 ins_encode %{
9306 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9307 %}
9308 ins_pipe(pipe_serial);
9309 %}
9310
9311 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9312 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9313 match(Set dummy (GetAndAddI mem incr));
9314 ins_cost(VOLATILE_REF_COST);
9315 format %{ "get_and_addI_acq [$mem], $incr" %}
9316 ins_encode %{
9317 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9318 %}
9319 ins_pipe(pipe_serial);
9320 %}
9321
9322 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9323 predicate(needs_acquiring_load_exclusive(n));
9324 match(Set newval (GetAndAddI mem incr));
9325 ins_cost(VOLATILE_REF_COST + 1);
9326 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9327 ins_encode %{
9328 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9329 %}
9330 ins_pipe(pipe_serial);
9331 %}
9332
9333 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9334 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9335 match(Set dummy (GetAndAddI mem incr));
9336 ins_cost(VOLATILE_REF_COST);
9337 format %{ "get_and_addI_acq [$mem], $incr" %}
9338 ins_encode %{
9339 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9340 %}
9341 ins_pipe(pipe_serial);
9342 %}
9343
9344 // Manifest a CmpU result in an integer register.
9345 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9346 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9347 %{
9348 match(Set dst (CmpU3 src1 src2));
9349 effect(KILL flags);
9350
9351 ins_cost(INSN_COST * 3);
9352 format %{
9353 "cmpw $src1, $src2\n\t"
9354 "csetw $dst, ne\n\t"
9355 "cnegw $dst, lo\t# CmpU3(reg)"
9356 %}
9357 ins_encode %{
9358 __ cmpw($src1$$Register, $src2$$Register);
9359 __ csetw($dst$$Register, Assembler::NE);
9360 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9361 %}
9362
9363 ins_pipe(pipe_class_default);
9364 %}
9365
9366 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9367 %{
9368 match(Set dst (CmpU3 src1 src2));
9369 effect(KILL flags);
9370
9371 ins_cost(INSN_COST * 3);
9372 format %{
9373 "subsw zr, $src1, $src2\n\t"
9374 "csetw $dst, ne\n\t"
9375 "cnegw $dst, lo\t# CmpU3(imm)"
9376 %}
9377 ins_encode %{
9378 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9379 __ csetw($dst$$Register, Assembler::NE);
9380 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9381 %}
9382
9383 ins_pipe(pipe_class_default);
9384 %}
9385
9386 // Manifest a CmpUL result in an integer register.
9387 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9388 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9389 %{
9390 match(Set dst (CmpUL3 src1 src2));
9391 effect(KILL flags);
9392
9393 ins_cost(INSN_COST * 3);
9394 format %{
9395 "cmp $src1, $src2\n\t"
9396 "csetw $dst, ne\n\t"
9397 "cnegw $dst, lo\t# CmpUL3(reg)"
9398 %}
9399 ins_encode %{
9400 __ cmp($src1$$Register, $src2$$Register);
9401 __ csetw($dst$$Register, Assembler::NE);
9402 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9403 %}
9404
9405 ins_pipe(pipe_class_default);
9406 %}
9407
9408 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9409 %{
9410 match(Set dst (CmpUL3 src1 src2));
9411 effect(KILL flags);
9412
9413 ins_cost(INSN_COST * 3);
9414 format %{
9415 "subs zr, $src1, $src2\n\t"
9416 "csetw $dst, ne\n\t"
9417 "cnegw $dst, lo\t# CmpUL3(imm)"
9418 %}
9419 ins_encode %{
9420 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9421 __ csetw($dst$$Register, Assembler::NE);
9422 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9423 %}
9424
9425 ins_pipe(pipe_class_default);
9426 %}
9427
9428 // Manifest a CmpL result in an integer register.
9429 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9430 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9431 %{
9432 match(Set dst (CmpL3 src1 src2));
9433 effect(KILL flags);
9434
9435 ins_cost(INSN_COST * 3);
9436 format %{
9437 "cmp $src1, $src2\n\t"
9438 "csetw $dst, ne\n\t"
9439 "cnegw $dst, lt\t# CmpL3(reg)"
9440 %}
9441 ins_encode %{
9442 __ cmp($src1$$Register, $src2$$Register);
9443 __ csetw($dst$$Register, Assembler::NE);
9444 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9445 %}
9446
9447 ins_pipe(pipe_class_default);
9448 %}
9449
9450 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9451 %{
9452 match(Set dst (CmpL3 src1 src2));
9453 effect(KILL flags);
9454
9455 ins_cost(INSN_COST * 3);
9456 format %{
9457 "subs zr, $src1, $src2\n\t"
9458 "csetw $dst, ne\n\t"
9459 "cnegw $dst, lt\t# CmpL3(imm)"
9460 %}
9461 ins_encode %{
9462 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9463 __ csetw($dst$$Register, Assembler::NE);
9464 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9465 %}
9466
9467 ins_pipe(pipe_class_default);
9468 %}
9469
9470 // ============================================================================
9471 // Conditional Move Instructions
9472
9473 // n.b. we have identical rules for both a signed compare op (cmpOp)
9474 // and an unsigned compare op (cmpOpU). it would be nice if we could
9475 // define an op class which merged both inputs and use it to type the
9476 // argument to a single rule. unfortunatelyt his fails because the
9477 // opclass does not live up to the COND_INTER interface of its
9478 // component operands. When the generic code tries to negate the
9479 // operand it ends up running the generci Machoper::negate method
9480 // which throws a ShouldNotHappen. So, we have to provide two flavours
9481 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9482
9483 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9484 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9485
9486 ins_cost(INSN_COST * 2);
9487 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9488
9489 ins_encode %{
9490 __ cselw(as_Register($dst$$reg),
9491 as_Register($src2$$reg),
9492 as_Register($src1$$reg),
9493 (Assembler::Condition)$cmp$$cmpcode);
9494 %}
9495
9496 ins_pipe(icond_reg_reg);
9497 %}
9498
9499 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9500 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9501
9502 ins_cost(INSN_COST * 2);
9503 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9504
9505 ins_encode %{
9506 __ cselw(as_Register($dst$$reg),
9507 as_Register($src2$$reg),
9508 as_Register($src1$$reg),
9509 (Assembler::Condition)$cmp$$cmpcode);
9510 %}
9511
9512 ins_pipe(icond_reg_reg);
9513 %}
9514
9515 // special cases where one arg is zero
9516
9517 // n.b. this is selected in preference to the rule above because it
9518 // avoids loading constant 0 into a source register
9519
9520 // TODO
9521 // we ought only to be able to cull one of these variants as the ideal
9522 // transforms ought always to order the zero consistently (to left/right?)
9523
9524 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9525 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9526
9527 ins_cost(INSN_COST * 2);
9528 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9529
9530 ins_encode %{
9531 __ cselw(as_Register($dst$$reg),
9532 as_Register($src$$reg),
9533 zr,
9534 (Assembler::Condition)$cmp$$cmpcode);
9535 %}
9536
9537 ins_pipe(icond_reg);
9538 %}
9539
9540 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9541 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9542
9543 ins_cost(INSN_COST * 2);
9544 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9545
9546 ins_encode %{
9547 __ cselw(as_Register($dst$$reg),
9548 as_Register($src$$reg),
9549 zr,
9550 (Assembler::Condition)$cmp$$cmpcode);
9551 %}
9552
9553 ins_pipe(icond_reg);
9554 %}
9555
9556 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9557 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9558
9559 ins_cost(INSN_COST * 2);
9560 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9561
9562 ins_encode %{
9563 __ cselw(as_Register($dst$$reg),
9564 zr,
9565 as_Register($src$$reg),
9566 (Assembler::Condition)$cmp$$cmpcode);
9567 %}
9568
9569 ins_pipe(icond_reg);
9570 %}
9571
9572 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9573 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9574
9575 ins_cost(INSN_COST * 2);
9576 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9577
9578 ins_encode %{
9579 __ cselw(as_Register($dst$$reg),
9580 zr,
9581 as_Register($src$$reg),
9582 (Assembler::Condition)$cmp$$cmpcode);
9583 %}
9584
9585 ins_pipe(icond_reg);
9586 %}
9587
9588 // special case for creating a boolean 0 or 1
9589
9590 // n.b. this is selected in preference to the rule above because it
9591 // avoids loading constants 0 and 1 into a source register
9592
9593 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9594 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9595
9596 ins_cost(INSN_COST * 2);
9597 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9598
9599 ins_encode %{
9600 // equivalently
9601 // cset(as_Register($dst$$reg),
9602 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9603 __ csincw(as_Register($dst$$reg),
9604 zr,
9605 zr,
9606 (Assembler::Condition)$cmp$$cmpcode);
9607 %}
9608
9609 ins_pipe(icond_none);
9610 %}
9611
9612 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9613 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9614
9615 ins_cost(INSN_COST * 2);
9616 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9617
9618 ins_encode %{
9619 // equivalently
9620 // cset(as_Register($dst$$reg),
9621 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9622 __ csincw(as_Register($dst$$reg),
9623 zr,
9624 zr,
9625 (Assembler::Condition)$cmp$$cmpcode);
9626 %}
9627
9628 ins_pipe(icond_none);
9629 %}
9630
9631 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9632 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9633
9634 ins_cost(INSN_COST * 2);
9635 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9636
9637 ins_encode %{
9638 __ csel(as_Register($dst$$reg),
9639 as_Register($src2$$reg),
9640 as_Register($src1$$reg),
9641 (Assembler::Condition)$cmp$$cmpcode);
9642 %}
9643
9644 ins_pipe(icond_reg_reg);
9645 %}
9646
9647 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9648 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9649
9650 ins_cost(INSN_COST * 2);
9651 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9652
9653 ins_encode %{
9654 __ csel(as_Register($dst$$reg),
9655 as_Register($src2$$reg),
9656 as_Register($src1$$reg),
9657 (Assembler::Condition)$cmp$$cmpcode);
9658 %}
9659
9660 ins_pipe(icond_reg_reg);
9661 %}
9662
9663 // special cases where one arg is zero
9664
9665 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9666 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9667
9668 ins_cost(INSN_COST * 2);
9669 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9670
9671 ins_encode %{
9672 __ csel(as_Register($dst$$reg),
9673 zr,
9674 as_Register($src$$reg),
9675 (Assembler::Condition)$cmp$$cmpcode);
9676 %}
9677
9678 ins_pipe(icond_reg);
9679 %}
9680
9681 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9682 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9683
9684 ins_cost(INSN_COST * 2);
9685 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9686
9687 ins_encode %{
9688 __ csel(as_Register($dst$$reg),
9689 zr,
9690 as_Register($src$$reg),
9691 (Assembler::Condition)$cmp$$cmpcode);
9692 %}
9693
9694 ins_pipe(icond_reg);
9695 %}
9696
9697 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9698 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9699
9700 ins_cost(INSN_COST * 2);
9701 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9702
9703 ins_encode %{
9704 __ csel(as_Register($dst$$reg),
9705 as_Register($src$$reg),
9706 zr,
9707 (Assembler::Condition)$cmp$$cmpcode);
9708 %}
9709
9710 ins_pipe(icond_reg);
9711 %}
9712
9713 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9714 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9715
9716 ins_cost(INSN_COST * 2);
9717 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9718
9719 ins_encode %{
9720 __ csel(as_Register($dst$$reg),
9721 as_Register($src$$reg),
9722 zr,
9723 (Assembler::Condition)$cmp$$cmpcode);
9724 %}
9725
9726 ins_pipe(icond_reg);
9727 %}
9728
9729 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9730 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9731
9732 ins_cost(INSN_COST * 2);
9733 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9734
9735 ins_encode %{
9736 __ csel(as_Register($dst$$reg),
9737 as_Register($src2$$reg),
9738 as_Register($src1$$reg),
9739 (Assembler::Condition)$cmp$$cmpcode);
9740 %}
9741
9742 ins_pipe(icond_reg_reg);
9743 %}
9744
9745 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9746 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9747
9748 ins_cost(INSN_COST * 2);
9749 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9750
9751 ins_encode %{
9752 __ csel(as_Register($dst$$reg),
9753 as_Register($src2$$reg),
9754 as_Register($src1$$reg),
9755 (Assembler::Condition)$cmp$$cmpcode);
9756 %}
9757
9758 ins_pipe(icond_reg_reg);
9759 %}
9760
9761 // special cases where one arg is zero
9762
9763 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9764 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9765
9766 ins_cost(INSN_COST * 2);
9767 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9768
9769 ins_encode %{
9770 __ csel(as_Register($dst$$reg),
9771 zr,
9772 as_Register($src$$reg),
9773 (Assembler::Condition)$cmp$$cmpcode);
9774 %}
9775
9776 ins_pipe(icond_reg);
9777 %}
9778
9779 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9780 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9781
9782 ins_cost(INSN_COST * 2);
9783 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9784
9785 ins_encode %{
9786 __ csel(as_Register($dst$$reg),
9787 zr,
9788 as_Register($src$$reg),
9789 (Assembler::Condition)$cmp$$cmpcode);
9790 %}
9791
9792 ins_pipe(icond_reg);
9793 %}
9794
9795 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9796 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9797
9798 ins_cost(INSN_COST * 2);
9799 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9800
9801 ins_encode %{
9802 __ csel(as_Register($dst$$reg),
9803 as_Register($src$$reg),
9804 zr,
9805 (Assembler::Condition)$cmp$$cmpcode);
9806 %}
9807
9808 ins_pipe(icond_reg);
9809 %}
9810
9811 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9812 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9813
9814 ins_cost(INSN_COST * 2);
9815 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9816
9817 ins_encode %{
9818 __ csel(as_Register($dst$$reg),
9819 as_Register($src$$reg),
9820 zr,
9821 (Assembler::Condition)$cmp$$cmpcode);
9822 %}
9823
9824 ins_pipe(icond_reg);
9825 %}
9826
9827 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9828 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9829
9830 ins_cost(INSN_COST * 2);
9831 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9832
9833 ins_encode %{
9834 __ cselw(as_Register($dst$$reg),
9835 as_Register($src2$$reg),
9836 as_Register($src1$$reg),
9837 (Assembler::Condition)$cmp$$cmpcode);
9838 %}
9839
9840 ins_pipe(icond_reg_reg);
9841 %}
9842
9843 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9844 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9845
9846 ins_cost(INSN_COST * 2);
9847 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9848
9849 ins_encode %{
9850 __ cselw(as_Register($dst$$reg),
9851 as_Register($src2$$reg),
9852 as_Register($src1$$reg),
9853 (Assembler::Condition)$cmp$$cmpcode);
9854 %}
9855
9856 ins_pipe(icond_reg_reg);
9857 %}
9858
9859 // special cases where one arg is zero
9860
9861 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9862 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9863
9864 ins_cost(INSN_COST * 2);
9865 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9866
9867 ins_encode %{
9868 __ cselw(as_Register($dst$$reg),
9869 zr,
9870 as_Register($src$$reg),
9871 (Assembler::Condition)$cmp$$cmpcode);
9872 %}
9873
9874 ins_pipe(icond_reg);
9875 %}
9876
9877 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9878 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9879
9880 ins_cost(INSN_COST * 2);
9881 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9882
9883 ins_encode %{
9884 __ cselw(as_Register($dst$$reg),
9885 zr,
9886 as_Register($src$$reg),
9887 (Assembler::Condition)$cmp$$cmpcode);
9888 %}
9889
9890 ins_pipe(icond_reg);
9891 %}
9892
9893 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9894 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9895
9896 ins_cost(INSN_COST * 2);
9897 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9898
9899 ins_encode %{
9900 __ cselw(as_Register($dst$$reg),
9901 as_Register($src$$reg),
9902 zr,
9903 (Assembler::Condition)$cmp$$cmpcode);
9904 %}
9905
9906 ins_pipe(icond_reg);
9907 %}
9908
9909 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9910 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9911
9912 ins_cost(INSN_COST * 2);
9913 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9914
9915 ins_encode %{
9916 __ cselw(as_Register($dst$$reg),
9917 as_Register($src$$reg),
9918 zr,
9919 (Assembler::Condition)$cmp$$cmpcode);
9920 %}
9921
9922 ins_pipe(icond_reg);
9923 %}
9924
9925 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9926 %{
9927 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9928
9929 ins_cost(INSN_COST * 3);
9930
9931 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9932 ins_encode %{
9933 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9934 __ fcsels(as_FloatRegister($dst$$reg),
9935 as_FloatRegister($src2$$reg),
9936 as_FloatRegister($src1$$reg),
9937 cond);
9938 %}
9939
9940 ins_pipe(fp_cond_reg_reg_s);
9941 %}
9942
9943 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9944 %{
9945 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9946
9947 ins_cost(INSN_COST * 3);
9948
9949 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9950 ins_encode %{
9951 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9952 __ fcsels(as_FloatRegister($dst$$reg),
9953 as_FloatRegister($src2$$reg),
9954 as_FloatRegister($src1$$reg),
9955 cond);
9956 %}
9957
9958 ins_pipe(fp_cond_reg_reg_s);
9959 %}
9960
9961 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9962 %{
9963 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9964
9965 ins_cost(INSN_COST * 3);
9966
9967 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9968 ins_encode %{
9969 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9970 __ fcseld(as_FloatRegister($dst$$reg),
9971 as_FloatRegister($src2$$reg),
9972 as_FloatRegister($src1$$reg),
9973 cond);
9974 %}
9975
9976 ins_pipe(fp_cond_reg_reg_d);
9977 %}
9978
9979 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9980 %{
9981 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9982
9983 ins_cost(INSN_COST * 3);
9984
9985 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9986 ins_encode %{
9987 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9988 __ fcseld(as_FloatRegister($dst$$reg),
9989 as_FloatRegister($src2$$reg),
9990 as_FloatRegister($src1$$reg),
9991 cond);
9992 %}
9993
9994 ins_pipe(fp_cond_reg_reg_d);
9995 %}
9996
9997 // ============================================================================
9998 // Arithmetic Instructions
9999 //
10000
10001 // Integer Addition
10002
10003 // TODO
10004 // these currently employ operations which do not set CR and hence are
10005 // not flagged as killing CR but we would like to isolate the cases
10006 // where we want to set flags from those where we don't. need to work
10007 // out how to do that.
10008
10009 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10010 match(Set dst (AddI src1 src2));
10011
10012 ins_cost(INSN_COST);
10013 format %{ "addw $dst, $src1, $src2" %}
10014
10015 ins_encode %{
10016 __ addw(as_Register($dst$$reg),
10017 as_Register($src1$$reg),
10018 as_Register($src2$$reg));
10019 %}
10020
10021 ins_pipe(ialu_reg_reg);
10022 %}
10023
10024 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10025 match(Set dst (AddI src1 src2));
10026
10027 ins_cost(INSN_COST);
10028 format %{ "addw $dst, $src1, $src2" %}
10029
10030 // use opcode to indicate that this is an add not a sub
10031 opcode(0x0);
10032
10033 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10034
10035 ins_pipe(ialu_reg_imm);
10036 %}
10037
10038 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10039 match(Set dst (AddI (ConvL2I src1) src2));
10040
10041 ins_cost(INSN_COST);
10042 format %{ "addw $dst, $src1, $src2" %}
10043
10044 // use opcode to indicate that this is an add not a sub
10045 opcode(0x0);
10046
10047 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10048
10049 ins_pipe(ialu_reg_imm);
10050 %}
10051
10052 // Pointer Addition
10053 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10054 match(Set dst (AddP src1 src2));
10055
10056 ins_cost(INSN_COST);
10057 format %{ "add $dst, $src1, $src2\t# ptr" %}
10058
10059 ins_encode %{
10060 __ add(as_Register($dst$$reg),
10061 as_Register($src1$$reg),
10062 as_Register($src2$$reg));
10063 %}
10064
10065 ins_pipe(ialu_reg_reg);
10066 %}
10067
10068 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10069 match(Set dst (AddP src1 (ConvI2L src2)));
10070
10071 ins_cost(1.9 * INSN_COST);
10072 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10073
10074 ins_encode %{
10075 __ add(as_Register($dst$$reg),
10076 as_Register($src1$$reg),
10077 as_Register($src2$$reg), ext::sxtw);
10078 %}
10079
10080 ins_pipe(ialu_reg_reg);
10081 %}
10082
10083 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10084 match(Set dst (AddP src1 (LShiftL src2 scale)));
10085
10086 ins_cost(1.9 * INSN_COST);
10087 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10088
10089 ins_encode %{
10090 __ lea(as_Register($dst$$reg),
10091 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10092 Address::lsl($scale$$constant)));
10093 %}
10094
10095 ins_pipe(ialu_reg_reg_shift);
10096 %}
10097
10098 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10099 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10100
10101 ins_cost(1.9 * INSN_COST);
10102 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10103
10104 ins_encode %{
10105 __ lea(as_Register($dst$$reg),
10106 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10107 Address::sxtw($scale$$constant)));
10108 %}
10109
10110 ins_pipe(ialu_reg_reg_shift);
10111 %}
10112
10113 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10114 match(Set dst (LShiftL (ConvI2L src) scale));
10115
10116 ins_cost(INSN_COST);
10117 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10118
10119 ins_encode %{
10120 __ sbfiz(as_Register($dst$$reg),
10121 as_Register($src$$reg),
10122 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10123 %}
10124
10125 ins_pipe(ialu_reg_shift);
10126 %}
10127
10128 // Pointer Immediate Addition
10129 // n.b. this needs to be more expensive than using an indirect memory
10130 // operand
10131 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10132 match(Set dst (AddP src1 src2));
10133
10134 ins_cost(INSN_COST);
10135 format %{ "add $dst, $src1, $src2\t# ptr" %}
10136
10137 // use opcode to indicate that this is an add not a sub
10138 opcode(0x0);
10139
10140 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10141
10142 ins_pipe(ialu_reg_imm);
10143 %}
10144
10145 // Long Addition
10146 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10147
10148 match(Set dst (AddL src1 src2));
10149
10150 ins_cost(INSN_COST);
10151 format %{ "add $dst, $src1, $src2" %}
10152
10153 ins_encode %{
10154 __ add(as_Register($dst$$reg),
10155 as_Register($src1$$reg),
10156 as_Register($src2$$reg));
10157 %}
10158
10159 ins_pipe(ialu_reg_reg);
10160 %}
10161
10162 // No constant pool entries requiredLong Immediate Addition.
10163 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10164 match(Set dst (AddL src1 src2));
10165
10166 ins_cost(INSN_COST);
10167 format %{ "add $dst, $src1, $src2" %}
10168
10169 // use opcode to indicate that this is an add not a sub
10170 opcode(0x0);
10171
10172 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10173
10174 ins_pipe(ialu_reg_imm);
10175 %}
10176
10177 // Integer Subtraction
10178 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10179 match(Set dst (SubI src1 src2));
10180
10181 ins_cost(INSN_COST);
10182 format %{ "subw $dst, $src1, $src2" %}
10183
10184 ins_encode %{
10185 __ subw(as_Register($dst$$reg),
10186 as_Register($src1$$reg),
10187 as_Register($src2$$reg));
10188 %}
10189
10190 ins_pipe(ialu_reg_reg);
10191 %}
10192
10193 // Immediate Subtraction
10194 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10195 match(Set dst (SubI src1 src2));
10196
10197 ins_cost(INSN_COST);
10198 format %{ "subw $dst, $src1, $src2" %}
10199
10200 // use opcode to indicate that this is a sub not an add
10201 opcode(0x1);
10202
10203 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10204
10205 ins_pipe(ialu_reg_imm);
10206 %}
10207
10208 // Long Subtraction
10209 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10210
10211 match(Set dst (SubL src1 src2));
10212
10213 ins_cost(INSN_COST);
10214 format %{ "sub $dst, $src1, $src2" %}
10215
10216 ins_encode %{
10217 __ sub(as_Register($dst$$reg),
10218 as_Register($src1$$reg),
10219 as_Register($src2$$reg));
10220 %}
10221
10222 ins_pipe(ialu_reg_reg);
10223 %}
10224
10225 // No constant pool entries requiredLong Immediate Subtraction.
10226 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10227 match(Set dst (SubL src1 src2));
10228
10229 ins_cost(INSN_COST);
10230 format %{ "sub$dst, $src1, $src2" %}
10231
10232 // use opcode to indicate that this is a sub not an add
10233 opcode(0x1);
10234
10235 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10236
10237 ins_pipe(ialu_reg_imm);
10238 %}
10239
10240 // Integer Negation (special case for sub)
10241
10242 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10243 match(Set dst (SubI zero src));
10244
10245 ins_cost(INSN_COST);
10246 format %{ "negw $dst, $src\t# int" %}
10247
10248 ins_encode %{
10249 __ negw(as_Register($dst$$reg),
10250 as_Register($src$$reg));
10251 %}
10252
10253 ins_pipe(ialu_reg);
10254 %}
10255
10256 // Long Negation
10257
10258 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10259 match(Set dst (SubL zero src));
10260
10261 ins_cost(INSN_COST);
10262 format %{ "neg $dst, $src\t# long" %}
10263
10264 ins_encode %{
10265 __ neg(as_Register($dst$$reg),
10266 as_Register($src$$reg));
10267 %}
10268
10269 ins_pipe(ialu_reg);
10270 %}
10271
10272 // Integer Multiply
10273
10274 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10275 match(Set dst (MulI src1 src2));
10276
10277 ins_cost(INSN_COST * 3);
10278 format %{ "mulw $dst, $src1, $src2" %}
10279
10280 ins_encode %{
10281 __ mulw(as_Register($dst$$reg),
10282 as_Register($src1$$reg),
10283 as_Register($src2$$reg));
10284 %}
10285
10286 ins_pipe(imul_reg_reg);
10287 %}
10288
10289 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10290 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10291
10292 ins_cost(INSN_COST * 3);
10293 format %{ "smull $dst, $src1, $src2" %}
10294
10295 ins_encode %{
10296 __ smull(as_Register($dst$$reg),
10297 as_Register($src1$$reg),
10298 as_Register($src2$$reg));
10299 %}
10300
10301 ins_pipe(imul_reg_reg);
10302 %}
10303
10304 // Long Multiply
10305
10306 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10307 match(Set dst (MulL src1 src2));
10308
10309 ins_cost(INSN_COST * 5);
10310 format %{ "mul $dst, $src1, $src2" %}
10311
10312 ins_encode %{
10313 __ mul(as_Register($dst$$reg),
10314 as_Register($src1$$reg),
10315 as_Register($src2$$reg));
10316 %}
10317
10318 ins_pipe(lmul_reg_reg);
10319 %}
10320
10321 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10322 %{
10323 match(Set dst (MulHiL src1 src2));
10324
10325 ins_cost(INSN_COST * 7);
10326 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10327
10328 ins_encode %{
10329 __ smulh(as_Register($dst$$reg),
10330 as_Register($src1$$reg),
10331 as_Register($src2$$reg));
10332 %}
10333
10334 ins_pipe(lmul_reg_reg);
10335 %}
10336
10337 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10338 %{
10339 match(Set dst (UMulHiL src1 src2));
10340
10341 ins_cost(INSN_COST * 7);
10342 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10343
10344 ins_encode %{
10345 __ umulh(as_Register($dst$$reg),
10346 as_Register($src1$$reg),
10347 as_Register($src2$$reg));
10348 %}
10349
10350 ins_pipe(lmul_reg_reg);
10351 %}
10352
10353 // Combined Integer Multiply & Add/Sub
10354
10355 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10356 match(Set dst (AddI src3 (MulI src1 src2)));
10357
10358 ins_cost(INSN_COST * 3);
10359 format %{ "madd $dst, $src1, $src2, $src3" %}
10360
10361 ins_encode %{
10362 __ maddw(as_Register($dst$$reg),
10363 as_Register($src1$$reg),
10364 as_Register($src2$$reg),
10365 as_Register($src3$$reg));
10366 %}
10367
10368 ins_pipe(imac_reg_reg);
10369 %}
10370
10371 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10372 match(Set dst (SubI src3 (MulI src1 src2)));
10373
10374 ins_cost(INSN_COST * 3);
10375 format %{ "msub $dst, $src1, $src2, $src3" %}
10376
10377 ins_encode %{
10378 __ msubw(as_Register($dst$$reg),
10379 as_Register($src1$$reg),
10380 as_Register($src2$$reg),
10381 as_Register($src3$$reg));
10382 %}
10383
10384 ins_pipe(imac_reg_reg);
10385 %}
10386
10387 // Combined Integer Multiply & Neg
10388
10389 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10390 match(Set dst (MulI (SubI zero src1) src2));
10391
10392 ins_cost(INSN_COST * 3);
10393 format %{ "mneg $dst, $src1, $src2" %}
10394
10395 ins_encode %{
10396 __ mnegw(as_Register($dst$$reg),
10397 as_Register($src1$$reg),
10398 as_Register($src2$$reg));
10399 %}
10400
10401 ins_pipe(imac_reg_reg);
10402 %}
10403
10404 // Combined Long Multiply & Add/Sub
10405
10406 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10407 match(Set dst (AddL src3 (MulL src1 src2)));
10408
10409 ins_cost(INSN_COST * 5);
10410 format %{ "madd $dst, $src1, $src2, $src3" %}
10411
10412 ins_encode %{
10413 __ madd(as_Register($dst$$reg),
10414 as_Register($src1$$reg),
10415 as_Register($src2$$reg),
10416 as_Register($src3$$reg));
10417 %}
10418
10419 ins_pipe(lmac_reg_reg);
10420 %}
10421
10422 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10423 match(Set dst (SubL src3 (MulL src1 src2)));
10424
10425 ins_cost(INSN_COST * 5);
10426 format %{ "msub $dst, $src1, $src2, $src3" %}
10427
10428 ins_encode %{
10429 __ msub(as_Register($dst$$reg),
10430 as_Register($src1$$reg),
10431 as_Register($src2$$reg),
10432 as_Register($src3$$reg));
10433 %}
10434
10435 ins_pipe(lmac_reg_reg);
10436 %}
10437
10438 // Combined Long Multiply & Neg
10439
10440 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10441 match(Set dst (MulL (SubL zero src1) src2));
10442
10443 ins_cost(INSN_COST * 5);
10444 format %{ "mneg $dst, $src1, $src2" %}
10445
10446 ins_encode %{
10447 __ mneg(as_Register($dst$$reg),
10448 as_Register($src1$$reg),
10449 as_Register($src2$$reg));
10450 %}
10451
10452 ins_pipe(lmac_reg_reg);
10453 %}
10454
10455 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10456
10457 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10458 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10459
10460 ins_cost(INSN_COST * 3);
10461 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10462
10463 ins_encode %{
10464 __ smaddl(as_Register($dst$$reg),
10465 as_Register($src1$$reg),
10466 as_Register($src2$$reg),
10467 as_Register($src3$$reg));
10468 %}
10469
10470 ins_pipe(imac_reg_reg);
10471 %}
10472
10473 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10474 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10475
10476 ins_cost(INSN_COST * 3);
10477 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10478
10479 ins_encode %{
10480 __ smsubl(as_Register($dst$$reg),
10481 as_Register($src1$$reg),
10482 as_Register($src2$$reg),
10483 as_Register($src3$$reg));
10484 %}
10485
10486 ins_pipe(imac_reg_reg);
10487 %}
10488
10489 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10490 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10491
10492 ins_cost(INSN_COST * 3);
10493 format %{ "smnegl $dst, $src1, $src2" %}
10494
10495 ins_encode %{
10496 __ smnegl(as_Register($dst$$reg),
10497 as_Register($src1$$reg),
10498 as_Register($src2$$reg));
10499 %}
10500
10501 ins_pipe(imac_reg_reg);
10502 %}
10503
10504 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10505
10506 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10507 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10508
10509 ins_cost(INSN_COST * 5);
10510 format %{ "mulw rscratch1, $src1, $src2\n\t"
10511 "maddw $dst, $src3, $src4, rscratch1" %}
10512
10513 ins_encode %{
10514 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10515 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10516
10517 ins_pipe(imac_reg_reg);
10518 %}
10519
10520 // Integer Divide
10521
10522 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10523 match(Set dst (DivI src1 src2));
10524
10525 ins_cost(INSN_COST * 19);
10526 format %{ "sdivw $dst, $src1, $src2" %}
10527
10528 ins_encode(aarch64_enc_divw(dst, src1, src2));
10529 ins_pipe(idiv_reg_reg);
10530 %}
10531
10532 // Long Divide
10533
10534 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10535 match(Set dst (DivL src1 src2));
10536
10537 ins_cost(INSN_COST * 35);
10538 format %{ "sdiv $dst, $src1, $src2" %}
10539
10540 ins_encode(aarch64_enc_div(dst, src1, src2));
10541 ins_pipe(ldiv_reg_reg);
10542 %}
10543
10544 // Integer Remainder
10545
10546 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10547 match(Set dst (ModI src1 src2));
10548
10549 ins_cost(INSN_COST * 22);
10550 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10551 "msubw $dst, rscratch1, $src2, $src1" %}
10552
10553 ins_encode(aarch64_enc_modw(dst, src1, src2));
10554 ins_pipe(idiv_reg_reg);
10555 %}
10556
10557 // Long Remainder
10558
10559 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10560 match(Set dst (ModL src1 src2));
10561
10562 ins_cost(INSN_COST * 38);
10563 format %{ "sdiv rscratch1, $src1, $src2\n"
10564 "msub $dst, rscratch1, $src2, $src1" %}
10565
10566 ins_encode(aarch64_enc_mod(dst, src1, src2));
10567 ins_pipe(ldiv_reg_reg);
10568 %}
10569
10570 // Unsigned Integer Divide
10571
10572 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10573 match(Set dst (UDivI src1 src2));
10574
10575 ins_cost(INSN_COST * 19);
10576 format %{ "udivw $dst, $src1, $src2" %}
10577
10578 ins_encode %{
10579 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10580 %}
10581
10582 ins_pipe(idiv_reg_reg);
10583 %}
10584
10585 // Unsigned Long Divide
10586
10587 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10588 match(Set dst (UDivL src1 src2));
10589
10590 ins_cost(INSN_COST * 35);
10591 format %{ "udiv $dst, $src1, $src2" %}
10592
10593 ins_encode %{
10594 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10595 %}
10596
10597 ins_pipe(ldiv_reg_reg);
10598 %}
10599
10600 // Unsigned Integer Remainder
10601
10602 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10603 match(Set dst (UModI src1 src2));
10604
10605 ins_cost(INSN_COST * 22);
10606 format %{ "udivw rscratch1, $src1, $src2\n\t"
10607 "msubw $dst, rscratch1, $src2, $src1" %}
10608
10609 ins_encode %{
10610 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10611 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10612 %}
10613
10614 ins_pipe(idiv_reg_reg);
10615 %}
10616
10617 // Unsigned Long Remainder
10618
10619 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10620 match(Set dst (UModL src1 src2));
10621
10622 ins_cost(INSN_COST * 38);
10623 format %{ "udiv rscratch1, $src1, $src2\n"
10624 "msub $dst, rscratch1, $src2, $src1" %}
10625
10626 ins_encode %{
10627 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10628 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10629 %}
10630
10631 ins_pipe(ldiv_reg_reg);
10632 %}
10633
10634 // Integer Shifts
10635
10636 // Shift Left Register
10637 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10638 match(Set dst (LShiftI src1 src2));
10639
10640 ins_cost(INSN_COST * 2);
10641 format %{ "lslvw $dst, $src1, $src2" %}
10642
10643 ins_encode %{
10644 __ lslvw(as_Register($dst$$reg),
10645 as_Register($src1$$reg),
10646 as_Register($src2$$reg));
10647 %}
10648
10649 ins_pipe(ialu_reg_reg_vshift);
10650 %}
10651
10652 // Shift Left Immediate
10653 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10654 match(Set dst (LShiftI src1 src2));
10655
10656 ins_cost(INSN_COST);
10657 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10658
10659 ins_encode %{
10660 __ lslw(as_Register($dst$$reg),
10661 as_Register($src1$$reg),
10662 $src2$$constant & 0x1f);
10663 %}
10664
10665 ins_pipe(ialu_reg_shift);
10666 %}
10667
10668 // Shift Right Logical Register
10669 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10670 match(Set dst (URShiftI src1 src2));
10671
10672 ins_cost(INSN_COST * 2);
10673 format %{ "lsrvw $dst, $src1, $src2" %}
10674
10675 ins_encode %{
10676 __ lsrvw(as_Register($dst$$reg),
10677 as_Register($src1$$reg),
10678 as_Register($src2$$reg));
10679 %}
10680
10681 ins_pipe(ialu_reg_reg_vshift);
10682 %}
10683
10684 // Shift Right Logical Immediate
10685 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10686 match(Set dst (URShiftI src1 src2));
10687
10688 ins_cost(INSN_COST);
10689 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10690
10691 ins_encode %{
10692 __ lsrw(as_Register($dst$$reg),
10693 as_Register($src1$$reg),
10694 $src2$$constant & 0x1f);
10695 %}
10696
10697 ins_pipe(ialu_reg_shift);
10698 %}
10699
10700 // Shift Right Arithmetic Register
10701 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10702 match(Set dst (RShiftI src1 src2));
10703
10704 ins_cost(INSN_COST * 2);
10705 format %{ "asrvw $dst, $src1, $src2" %}
10706
10707 ins_encode %{
10708 __ asrvw(as_Register($dst$$reg),
10709 as_Register($src1$$reg),
10710 as_Register($src2$$reg));
10711 %}
10712
10713 ins_pipe(ialu_reg_reg_vshift);
10714 %}
10715
10716 // Shift Right Arithmetic Immediate
10717 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10718 match(Set dst (RShiftI src1 src2));
10719
10720 ins_cost(INSN_COST);
10721 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10722
10723 ins_encode %{
10724 __ asrw(as_Register($dst$$reg),
10725 as_Register($src1$$reg),
10726 $src2$$constant & 0x1f);
10727 %}
10728
10729 ins_pipe(ialu_reg_shift);
10730 %}
10731
10732 // Combined Int Mask and Right Shift (using UBFM)
10733 // TODO
10734
10735 // Long Shifts
10736
10737 // Shift Left Register
10738 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10739 match(Set dst (LShiftL src1 src2));
10740
10741 ins_cost(INSN_COST * 2);
10742 format %{ "lslv $dst, $src1, $src2" %}
10743
10744 ins_encode %{
10745 __ lslv(as_Register($dst$$reg),
10746 as_Register($src1$$reg),
10747 as_Register($src2$$reg));
10748 %}
10749
10750 ins_pipe(ialu_reg_reg_vshift);
10751 %}
10752
10753 // Shift Left Immediate
10754 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10755 match(Set dst (LShiftL src1 src2));
10756
10757 ins_cost(INSN_COST);
10758 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10759
10760 ins_encode %{
10761 __ lsl(as_Register($dst$$reg),
10762 as_Register($src1$$reg),
10763 $src2$$constant & 0x3f);
10764 %}
10765
10766 ins_pipe(ialu_reg_shift);
10767 %}
10768
10769 // Shift Right Logical Register
10770 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10771 match(Set dst (URShiftL src1 src2));
10772
10773 ins_cost(INSN_COST * 2);
10774 format %{ "lsrv $dst, $src1, $src2" %}
10775
10776 ins_encode %{
10777 __ lsrv(as_Register($dst$$reg),
10778 as_Register($src1$$reg),
10779 as_Register($src2$$reg));
10780 %}
10781
10782 ins_pipe(ialu_reg_reg_vshift);
10783 %}
10784
10785 // Shift Right Logical Immediate
10786 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10787 match(Set dst (URShiftL src1 src2));
10788
10789 ins_cost(INSN_COST);
10790 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10791
10792 ins_encode %{
10793 __ lsr(as_Register($dst$$reg),
10794 as_Register($src1$$reg),
10795 $src2$$constant & 0x3f);
10796 %}
10797
10798 ins_pipe(ialu_reg_shift);
10799 %}
10800
10801 // A special-case pattern for card table stores.
10802 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10803 match(Set dst (URShiftL (CastP2X src1) src2));
10804
10805 ins_cost(INSN_COST);
10806 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10807
10808 ins_encode %{
10809 __ lsr(as_Register($dst$$reg),
10810 as_Register($src1$$reg),
10811 $src2$$constant & 0x3f);
10812 %}
10813
10814 ins_pipe(ialu_reg_shift);
10815 %}
10816
10817 // Shift Right Arithmetic Register
10818 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10819 match(Set dst (RShiftL src1 src2));
10820
10821 ins_cost(INSN_COST * 2);
10822 format %{ "asrv $dst, $src1, $src2" %}
10823
10824 ins_encode %{
10825 __ asrv(as_Register($dst$$reg),
10826 as_Register($src1$$reg),
10827 as_Register($src2$$reg));
10828 %}
10829
10830 ins_pipe(ialu_reg_reg_vshift);
10831 %}
10832
10833 // Shift Right Arithmetic Immediate
10834 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10835 match(Set dst (RShiftL src1 src2));
10836
10837 ins_cost(INSN_COST);
10838 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10839
10840 ins_encode %{
10841 __ asr(as_Register($dst$$reg),
10842 as_Register($src1$$reg),
10843 $src2$$constant & 0x3f);
10844 %}
10845
10846 ins_pipe(ialu_reg_shift);
10847 %}
10848
10849 // BEGIN This section of the file is automatically generated. Do not edit --------------
10850 // This section is generated from aarch64_ad.m4
10851
10852 // This pattern is automatically generated from aarch64_ad.m4
10853 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10854 instruct regL_not_reg(iRegLNoSp dst,
10855 iRegL src1, immL_M1 m1,
10856 rFlagsReg cr) %{
10857 match(Set dst (XorL src1 m1));
10858 ins_cost(INSN_COST);
10859 format %{ "eon $dst, $src1, zr" %}
10860
10861 ins_encode %{
10862 __ eon(as_Register($dst$$reg),
10863 as_Register($src1$$reg),
10864 zr,
10865 Assembler::LSL, 0);
10866 %}
10867
10868 ins_pipe(ialu_reg);
10869 %}
10870
10871 // This pattern is automatically generated from aarch64_ad.m4
10872 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10873 instruct regI_not_reg(iRegINoSp dst,
10874 iRegIorL2I src1, immI_M1 m1,
10875 rFlagsReg cr) %{
10876 match(Set dst (XorI src1 m1));
10877 ins_cost(INSN_COST);
10878 format %{ "eonw $dst, $src1, zr" %}
10879
10880 ins_encode %{
10881 __ eonw(as_Register($dst$$reg),
10882 as_Register($src1$$reg),
10883 zr,
10884 Assembler::LSL, 0);
10885 %}
10886
10887 ins_pipe(ialu_reg);
10888 %}
10889
10890 // This pattern is automatically generated from aarch64_ad.m4
10891 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10892 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10893 immI0 zero, iRegIorL2I src1, immI src2) %{
10894 match(Set dst (SubI zero (URShiftI src1 src2)));
10895
10896 ins_cost(1.9 * INSN_COST);
10897 format %{ "negw $dst, $src1, LSR $src2" %}
10898
10899 ins_encode %{
10900 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10901 Assembler::LSR, $src2$$constant & 0x1f);
10902 %}
10903
10904 ins_pipe(ialu_reg_shift);
10905 %}
10906
10907 // This pattern is automatically generated from aarch64_ad.m4
10908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10909 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10910 immI0 zero, iRegIorL2I src1, immI src2) %{
10911 match(Set dst (SubI zero (RShiftI src1 src2)));
10912
10913 ins_cost(1.9 * INSN_COST);
10914 format %{ "negw $dst, $src1, ASR $src2" %}
10915
10916 ins_encode %{
10917 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10918 Assembler::ASR, $src2$$constant & 0x1f);
10919 %}
10920
10921 ins_pipe(ialu_reg_shift);
10922 %}
10923
10924 // This pattern is automatically generated from aarch64_ad.m4
10925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10926 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10927 immI0 zero, iRegIorL2I src1, immI src2) %{
10928 match(Set dst (SubI zero (LShiftI src1 src2)));
10929
10930 ins_cost(1.9 * INSN_COST);
10931 format %{ "negw $dst, $src1, LSL $src2" %}
10932
10933 ins_encode %{
10934 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10935 Assembler::LSL, $src2$$constant & 0x1f);
10936 %}
10937
10938 ins_pipe(ialu_reg_shift);
10939 %}
10940
10941 // This pattern is automatically generated from aarch64_ad.m4
10942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10943 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10944 immL0 zero, iRegL src1, immI src2) %{
10945 match(Set dst (SubL zero (URShiftL src1 src2)));
10946
10947 ins_cost(1.9 * INSN_COST);
10948 format %{ "neg $dst, $src1, LSR $src2" %}
10949
10950 ins_encode %{
10951 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10952 Assembler::LSR, $src2$$constant & 0x3f);
10953 %}
10954
10955 ins_pipe(ialu_reg_shift);
10956 %}
10957
10958 // This pattern is automatically generated from aarch64_ad.m4
10959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10960 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10961 immL0 zero, iRegL src1, immI src2) %{
10962 match(Set dst (SubL zero (RShiftL src1 src2)));
10963
10964 ins_cost(1.9 * INSN_COST);
10965 format %{ "neg $dst, $src1, ASR $src2" %}
10966
10967 ins_encode %{
10968 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10969 Assembler::ASR, $src2$$constant & 0x3f);
10970 %}
10971
10972 ins_pipe(ialu_reg_shift);
10973 %}
10974
10975 // This pattern is automatically generated from aarch64_ad.m4
10976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10977 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10978 immL0 zero, iRegL src1, immI src2) %{
10979 match(Set dst (SubL zero (LShiftL src1 src2)));
10980
10981 ins_cost(1.9 * INSN_COST);
10982 format %{ "neg $dst, $src1, LSL $src2" %}
10983
10984 ins_encode %{
10985 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10986 Assembler::LSL, $src2$$constant & 0x3f);
10987 %}
10988
10989 ins_pipe(ialu_reg_shift);
10990 %}
10991
10992 // This pattern is automatically generated from aarch64_ad.m4
10993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10994 instruct AndI_reg_not_reg(iRegINoSp dst,
10995 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10996 match(Set dst (AndI src1 (XorI src2 m1)));
10997 ins_cost(INSN_COST);
10998 format %{ "bicw $dst, $src1, $src2" %}
10999
11000 ins_encode %{
11001 __ bicw(as_Register($dst$$reg),
11002 as_Register($src1$$reg),
11003 as_Register($src2$$reg),
11004 Assembler::LSL, 0);
11005 %}
11006
11007 ins_pipe(ialu_reg_reg);
11008 %}
11009
11010 // This pattern is automatically generated from aarch64_ad.m4
11011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11012 instruct AndL_reg_not_reg(iRegLNoSp dst,
11013 iRegL src1, iRegL src2, immL_M1 m1) %{
11014 match(Set dst (AndL src1 (XorL src2 m1)));
11015 ins_cost(INSN_COST);
11016 format %{ "bic $dst, $src1, $src2" %}
11017
11018 ins_encode %{
11019 __ bic(as_Register($dst$$reg),
11020 as_Register($src1$$reg),
11021 as_Register($src2$$reg),
11022 Assembler::LSL, 0);
11023 %}
11024
11025 ins_pipe(ialu_reg_reg);
11026 %}
11027
11028 // This pattern is automatically generated from aarch64_ad.m4
11029 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11030 instruct OrI_reg_not_reg(iRegINoSp dst,
11031 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11032 match(Set dst (OrI src1 (XorI src2 m1)));
11033 ins_cost(INSN_COST);
11034 format %{ "ornw $dst, $src1, $src2" %}
11035
11036 ins_encode %{
11037 __ ornw(as_Register($dst$$reg),
11038 as_Register($src1$$reg),
11039 as_Register($src2$$reg),
11040 Assembler::LSL, 0);
11041 %}
11042
11043 ins_pipe(ialu_reg_reg);
11044 %}
11045
11046 // This pattern is automatically generated from aarch64_ad.m4
11047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11048 instruct OrL_reg_not_reg(iRegLNoSp dst,
11049 iRegL src1, iRegL src2, immL_M1 m1) %{
11050 match(Set dst (OrL src1 (XorL src2 m1)));
11051 ins_cost(INSN_COST);
11052 format %{ "orn $dst, $src1, $src2" %}
11053
11054 ins_encode %{
11055 __ orn(as_Register($dst$$reg),
11056 as_Register($src1$$reg),
11057 as_Register($src2$$reg),
11058 Assembler::LSL, 0);
11059 %}
11060
11061 ins_pipe(ialu_reg_reg);
11062 %}
11063
11064 // This pattern is automatically generated from aarch64_ad.m4
11065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11066 instruct XorI_reg_not_reg(iRegINoSp dst,
11067 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11068 match(Set dst (XorI m1 (XorI src2 src1)));
11069 ins_cost(INSN_COST);
11070 format %{ "eonw $dst, $src1, $src2" %}
11071
11072 ins_encode %{
11073 __ eonw(as_Register($dst$$reg),
11074 as_Register($src1$$reg),
11075 as_Register($src2$$reg),
11076 Assembler::LSL, 0);
11077 %}
11078
11079 ins_pipe(ialu_reg_reg);
11080 %}
11081
11082 // This pattern is automatically generated from aarch64_ad.m4
11083 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11084 instruct XorL_reg_not_reg(iRegLNoSp dst,
11085 iRegL src1, iRegL src2, immL_M1 m1) %{
11086 match(Set dst (XorL m1 (XorL src2 src1)));
11087 ins_cost(INSN_COST);
11088 format %{ "eon $dst, $src1, $src2" %}
11089
11090 ins_encode %{
11091 __ eon(as_Register($dst$$reg),
11092 as_Register($src1$$reg),
11093 as_Register($src2$$reg),
11094 Assembler::LSL, 0);
11095 %}
11096
11097 ins_pipe(ialu_reg_reg);
11098 %}
11099
11100 // This pattern is automatically generated from aarch64_ad.m4
11101 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11102 // val & (-1 ^ (val >>> shift)) ==> bicw
11103 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11104 iRegIorL2I src1, iRegIorL2I src2,
11105 immI src3, immI_M1 src4) %{
11106 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11107 ins_cost(1.9 * INSN_COST);
11108 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11109
11110 ins_encode %{
11111 __ bicw(as_Register($dst$$reg),
11112 as_Register($src1$$reg),
11113 as_Register($src2$$reg),
11114 Assembler::LSR,
11115 $src3$$constant & 0x1f);
11116 %}
11117
11118 ins_pipe(ialu_reg_reg_shift);
11119 %}
11120
11121 // This pattern is automatically generated from aarch64_ad.m4
11122 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11123 // val & (-1 ^ (val >>> shift)) ==> bic
11124 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11125 iRegL src1, iRegL src2,
11126 immI src3, immL_M1 src4) %{
11127 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11128 ins_cost(1.9 * INSN_COST);
11129 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11130
11131 ins_encode %{
11132 __ bic(as_Register($dst$$reg),
11133 as_Register($src1$$reg),
11134 as_Register($src2$$reg),
11135 Assembler::LSR,
11136 $src3$$constant & 0x3f);
11137 %}
11138
11139 ins_pipe(ialu_reg_reg_shift);
11140 %}
11141
11142 // This pattern is automatically generated from aarch64_ad.m4
11143 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11144 // val & (-1 ^ (val >> shift)) ==> bicw
11145 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11146 iRegIorL2I src1, iRegIorL2I src2,
11147 immI src3, immI_M1 src4) %{
11148 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11149 ins_cost(1.9 * INSN_COST);
11150 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11151
11152 ins_encode %{
11153 __ bicw(as_Register($dst$$reg),
11154 as_Register($src1$$reg),
11155 as_Register($src2$$reg),
11156 Assembler::ASR,
11157 $src3$$constant & 0x1f);
11158 %}
11159
11160 ins_pipe(ialu_reg_reg_shift);
11161 %}
11162
11163 // This pattern is automatically generated from aarch64_ad.m4
11164 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11165 // val & (-1 ^ (val >> shift)) ==> bic
11166 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11167 iRegL src1, iRegL src2,
11168 immI src3, immL_M1 src4) %{
11169 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11170 ins_cost(1.9 * INSN_COST);
11171 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11172
11173 ins_encode %{
11174 __ bic(as_Register($dst$$reg),
11175 as_Register($src1$$reg),
11176 as_Register($src2$$reg),
11177 Assembler::ASR,
11178 $src3$$constant & 0x3f);
11179 %}
11180
11181 ins_pipe(ialu_reg_reg_shift);
11182 %}
11183
11184 // This pattern is automatically generated from aarch64_ad.m4
11185 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11186 // val & (-1 ^ (val ror shift)) ==> bicw
11187 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11188 iRegIorL2I src1, iRegIorL2I src2,
11189 immI src3, immI_M1 src4) %{
11190 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11191 ins_cost(1.9 * INSN_COST);
11192 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11193
11194 ins_encode %{
11195 __ bicw(as_Register($dst$$reg),
11196 as_Register($src1$$reg),
11197 as_Register($src2$$reg),
11198 Assembler::ROR,
11199 $src3$$constant & 0x1f);
11200 %}
11201
11202 ins_pipe(ialu_reg_reg_shift);
11203 %}
11204
11205 // This pattern is automatically generated from aarch64_ad.m4
11206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11207 // val & (-1 ^ (val ror shift)) ==> bic
11208 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11209 iRegL src1, iRegL src2,
11210 immI src3, immL_M1 src4) %{
11211 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11212 ins_cost(1.9 * INSN_COST);
11213 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11214
11215 ins_encode %{
11216 __ bic(as_Register($dst$$reg),
11217 as_Register($src1$$reg),
11218 as_Register($src2$$reg),
11219 Assembler::ROR,
11220 $src3$$constant & 0x3f);
11221 %}
11222
11223 ins_pipe(ialu_reg_reg_shift);
11224 %}
11225
11226 // This pattern is automatically generated from aarch64_ad.m4
11227 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11228 // val & (-1 ^ (val << shift)) ==> bicw
11229 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11230 iRegIorL2I src1, iRegIorL2I src2,
11231 immI src3, immI_M1 src4) %{
11232 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11233 ins_cost(1.9 * INSN_COST);
11234 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11235
11236 ins_encode %{
11237 __ bicw(as_Register($dst$$reg),
11238 as_Register($src1$$reg),
11239 as_Register($src2$$reg),
11240 Assembler::LSL,
11241 $src3$$constant & 0x1f);
11242 %}
11243
11244 ins_pipe(ialu_reg_reg_shift);
11245 %}
11246
11247 // This pattern is automatically generated from aarch64_ad.m4
11248 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11249 // val & (-1 ^ (val << shift)) ==> bic
11250 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11251 iRegL src1, iRegL src2,
11252 immI src3, immL_M1 src4) %{
11253 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11254 ins_cost(1.9 * INSN_COST);
11255 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11256
11257 ins_encode %{
11258 __ bic(as_Register($dst$$reg),
11259 as_Register($src1$$reg),
11260 as_Register($src2$$reg),
11261 Assembler::LSL,
11262 $src3$$constant & 0x3f);
11263 %}
11264
11265 ins_pipe(ialu_reg_reg_shift);
11266 %}
11267
11268 // This pattern is automatically generated from aarch64_ad.m4
11269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11270 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11271 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11272 iRegIorL2I src1, iRegIorL2I src2,
11273 immI src3, immI_M1 src4) %{
11274 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11275 ins_cost(1.9 * INSN_COST);
11276 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11277
11278 ins_encode %{
11279 __ eonw(as_Register($dst$$reg),
11280 as_Register($src1$$reg),
11281 as_Register($src2$$reg),
11282 Assembler::LSR,
11283 $src3$$constant & 0x1f);
11284 %}
11285
11286 ins_pipe(ialu_reg_reg_shift);
11287 %}
11288
11289 // This pattern is automatically generated from aarch64_ad.m4
11290 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11291 // val ^ (-1 ^ (val >>> shift)) ==> eon
11292 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11293 iRegL src1, iRegL src2,
11294 immI src3, immL_M1 src4) %{
11295 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11296 ins_cost(1.9 * INSN_COST);
11297 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11298
11299 ins_encode %{
11300 __ eon(as_Register($dst$$reg),
11301 as_Register($src1$$reg),
11302 as_Register($src2$$reg),
11303 Assembler::LSR,
11304 $src3$$constant & 0x3f);
11305 %}
11306
11307 ins_pipe(ialu_reg_reg_shift);
11308 %}
11309
11310 // This pattern is automatically generated from aarch64_ad.m4
11311 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11312 // val ^ (-1 ^ (val >> shift)) ==> eonw
11313 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11314 iRegIorL2I src1, iRegIorL2I src2,
11315 immI src3, immI_M1 src4) %{
11316 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11317 ins_cost(1.9 * INSN_COST);
11318 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11319
11320 ins_encode %{
11321 __ eonw(as_Register($dst$$reg),
11322 as_Register($src1$$reg),
11323 as_Register($src2$$reg),
11324 Assembler::ASR,
11325 $src3$$constant & 0x1f);
11326 %}
11327
11328 ins_pipe(ialu_reg_reg_shift);
11329 %}
11330
11331 // This pattern is automatically generated from aarch64_ad.m4
11332 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11333 // val ^ (-1 ^ (val >> shift)) ==> eon
11334 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11335 iRegL src1, iRegL src2,
11336 immI src3, immL_M1 src4) %{
11337 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11338 ins_cost(1.9 * INSN_COST);
11339 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11340
11341 ins_encode %{
11342 __ eon(as_Register($dst$$reg),
11343 as_Register($src1$$reg),
11344 as_Register($src2$$reg),
11345 Assembler::ASR,
11346 $src3$$constant & 0x3f);
11347 %}
11348
11349 ins_pipe(ialu_reg_reg_shift);
11350 %}
11351
11352 // This pattern is automatically generated from aarch64_ad.m4
11353 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11354 // val ^ (-1 ^ (val ror shift)) ==> eonw
11355 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11356 iRegIorL2I src1, iRegIorL2I src2,
11357 immI src3, immI_M1 src4) %{
11358 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11359 ins_cost(1.9 * INSN_COST);
11360 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11361
11362 ins_encode %{
11363 __ eonw(as_Register($dst$$reg),
11364 as_Register($src1$$reg),
11365 as_Register($src2$$reg),
11366 Assembler::ROR,
11367 $src3$$constant & 0x1f);
11368 %}
11369
11370 ins_pipe(ialu_reg_reg_shift);
11371 %}
11372
11373 // This pattern is automatically generated from aarch64_ad.m4
11374 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11375 // val ^ (-1 ^ (val ror shift)) ==> eon
11376 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11377 iRegL src1, iRegL src2,
11378 immI src3, immL_M1 src4) %{
11379 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11380 ins_cost(1.9 * INSN_COST);
11381 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11382
11383 ins_encode %{
11384 __ eon(as_Register($dst$$reg),
11385 as_Register($src1$$reg),
11386 as_Register($src2$$reg),
11387 Assembler::ROR,
11388 $src3$$constant & 0x3f);
11389 %}
11390
11391 ins_pipe(ialu_reg_reg_shift);
11392 %}
11393
11394 // This pattern is automatically generated from aarch64_ad.m4
11395 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11396 // val ^ (-1 ^ (val << shift)) ==> eonw
11397 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11398 iRegIorL2I src1, iRegIorL2I src2,
11399 immI src3, immI_M1 src4) %{
11400 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11401 ins_cost(1.9 * INSN_COST);
11402 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11403
11404 ins_encode %{
11405 __ eonw(as_Register($dst$$reg),
11406 as_Register($src1$$reg),
11407 as_Register($src2$$reg),
11408 Assembler::LSL,
11409 $src3$$constant & 0x1f);
11410 %}
11411
11412 ins_pipe(ialu_reg_reg_shift);
11413 %}
11414
11415 // This pattern is automatically generated from aarch64_ad.m4
11416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11417 // val ^ (-1 ^ (val << shift)) ==> eon
11418 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11419 iRegL src1, iRegL src2,
11420 immI src3, immL_M1 src4) %{
11421 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11422 ins_cost(1.9 * INSN_COST);
11423 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11424
11425 ins_encode %{
11426 __ eon(as_Register($dst$$reg),
11427 as_Register($src1$$reg),
11428 as_Register($src2$$reg),
11429 Assembler::LSL,
11430 $src3$$constant & 0x3f);
11431 %}
11432
11433 ins_pipe(ialu_reg_reg_shift);
11434 %}
11435
11436 // This pattern is automatically generated from aarch64_ad.m4
11437 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11438 // val | (-1 ^ (val >>> shift)) ==> ornw
11439 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11440 iRegIorL2I src1, iRegIorL2I src2,
11441 immI src3, immI_M1 src4) %{
11442 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11443 ins_cost(1.9 * INSN_COST);
11444 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11445
11446 ins_encode %{
11447 __ ornw(as_Register($dst$$reg),
11448 as_Register($src1$$reg),
11449 as_Register($src2$$reg),
11450 Assembler::LSR,
11451 $src3$$constant & 0x1f);
11452 %}
11453
11454 ins_pipe(ialu_reg_reg_shift);
11455 %}
11456
11457 // This pattern is automatically generated from aarch64_ad.m4
11458 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11459 // val | (-1 ^ (val >>> shift)) ==> orn
11460 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11461 iRegL src1, iRegL src2,
11462 immI src3, immL_M1 src4) %{
11463 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11464 ins_cost(1.9 * INSN_COST);
11465 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11466
11467 ins_encode %{
11468 __ orn(as_Register($dst$$reg),
11469 as_Register($src1$$reg),
11470 as_Register($src2$$reg),
11471 Assembler::LSR,
11472 $src3$$constant & 0x3f);
11473 %}
11474
11475 ins_pipe(ialu_reg_reg_shift);
11476 %}
11477
11478 // This pattern is automatically generated from aarch64_ad.m4
11479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11480 // val | (-1 ^ (val >> shift)) ==> ornw
11481 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11482 iRegIorL2I src1, iRegIorL2I src2,
11483 immI src3, immI_M1 src4) %{
11484 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11485 ins_cost(1.9 * INSN_COST);
11486 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11487
11488 ins_encode %{
11489 __ ornw(as_Register($dst$$reg),
11490 as_Register($src1$$reg),
11491 as_Register($src2$$reg),
11492 Assembler::ASR,
11493 $src3$$constant & 0x1f);
11494 %}
11495
11496 ins_pipe(ialu_reg_reg_shift);
11497 %}
11498
11499 // This pattern is automatically generated from aarch64_ad.m4
11500 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11501 // val | (-1 ^ (val >> shift)) ==> orn
11502 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11503 iRegL src1, iRegL src2,
11504 immI src3, immL_M1 src4) %{
11505 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11506 ins_cost(1.9 * INSN_COST);
11507 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11508
11509 ins_encode %{
11510 __ orn(as_Register($dst$$reg),
11511 as_Register($src1$$reg),
11512 as_Register($src2$$reg),
11513 Assembler::ASR,
11514 $src3$$constant & 0x3f);
11515 %}
11516
11517 ins_pipe(ialu_reg_reg_shift);
11518 %}
11519
11520 // This pattern is automatically generated from aarch64_ad.m4
11521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11522 // val | (-1 ^ (val ror shift)) ==> ornw
11523 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11524 iRegIorL2I src1, iRegIorL2I src2,
11525 immI src3, immI_M1 src4) %{
11526 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11527 ins_cost(1.9 * INSN_COST);
11528 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11529
11530 ins_encode %{
11531 __ ornw(as_Register($dst$$reg),
11532 as_Register($src1$$reg),
11533 as_Register($src2$$reg),
11534 Assembler::ROR,
11535 $src3$$constant & 0x1f);
11536 %}
11537
11538 ins_pipe(ialu_reg_reg_shift);
11539 %}
11540
11541 // This pattern is automatically generated from aarch64_ad.m4
11542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11543 // val | (-1 ^ (val ror shift)) ==> orn
11544 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11545 iRegL src1, iRegL src2,
11546 immI src3, immL_M1 src4) %{
11547 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11548 ins_cost(1.9 * INSN_COST);
11549 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11550
11551 ins_encode %{
11552 __ orn(as_Register($dst$$reg),
11553 as_Register($src1$$reg),
11554 as_Register($src2$$reg),
11555 Assembler::ROR,
11556 $src3$$constant & 0x3f);
11557 %}
11558
11559 ins_pipe(ialu_reg_reg_shift);
11560 %}
11561
11562 // This pattern is automatically generated from aarch64_ad.m4
11563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11564 // val | (-1 ^ (val << shift)) ==> ornw
11565 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11566 iRegIorL2I src1, iRegIorL2I src2,
11567 immI src3, immI_M1 src4) %{
11568 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11569 ins_cost(1.9 * INSN_COST);
11570 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11571
11572 ins_encode %{
11573 __ ornw(as_Register($dst$$reg),
11574 as_Register($src1$$reg),
11575 as_Register($src2$$reg),
11576 Assembler::LSL,
11577 $src3$$constant & 0x1f);
11578 %}
11579
11580 ins_pipe(ialu_reg_reg_shift);
11581 %}
11582
11583 // This pattern is automatically generated from aarch64_ad.m4
11584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11585 // val | (-1 ^ (val << shift)) ==> orn
11586 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11587 iRegL src1, iRegL src2,
11588 immI src3, immL_M1 src4) %{
11589 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11590 ins_cost(1.9 * INSN_COST);
11591 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11592
11593 ins_encode %{
11594 __ orn(as_Register($dst$$reg),
11595 as_Register($src1$$reg),
11596 as_Register($src2$$reg),
11597 Assembler::LSL,
11598 $src3$$constant & 0x3f);
11599 %}
11600
11601 ins_pipe(ialu_reg_reg_shift);
11602 %}
11603
11604 // This pattern is automatically generated from aarch64_ad.m4
11605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11606 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11607 iRegIorL2I src1, iRegIorL2I src2,
11608 immI src3) %{
11609 match(Set dst (AndI src1 (URShiftI src2 src3)));
11610
11611 ins_cost(1.9 * INSN_COST);
11612 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11613
11614 ins_encode %{
11615 __ andw(as_Register($dst$$reg),
11616 as_Register($src1$$reg),
11617 as_Register($src2$$reg),
11618 Assembler::LSR,
11619 $src3$$constant & 0x1f);
11620 %}
11621
11622 ins_pipe(ialu_reg_reg_shift);
11623 %}
11624
11625 // This pattern is automatically generated from aarch64_ad.m4
11626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11627 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11628 iRegL src1, iRegL src2,
11629 immI src3) %{
11630 match(Set dst (AndL src1 (URShiftL src2 src3)));
11631
11632 ins_cost(1.9 * INSN_COST);
11633 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11634
11635 ins_encode %{
11636 __ andr(as_Register($dst$$reg),
11637 as_Register($src1$$reg),
11638 as_Register($src2$$reg),
11639 Assembler::LSR,
11640 $src3$$constant & 0x3f);
11641 %}
11642
11643 ins_pipe(ialu_reg_reg_shift);
11644 %}
11645
11646 // This pattern is automatically generated from aarch64_ad.m4
11647 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11648 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11649 iRegIorL2I src1, iRegIorL2I src2,
11650 immI src3) %{
11651 match(Set dst (AndI src1 (RShiftI src2 src3)));
11652
11653 ins_cost(1.9 * INSN_COST);
11654 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11655
11656 ins_encode %{
11657 __ andw(as_Register($dst$$reg),
11658 as_Register($src1$$reg),
11659 as_Register($src2$$reg),
11660 Assembler::ASR,
11661 $src3$$constant & 0x1f);
11662 %}
11663
11664 ins_pipe(ialu_reg_reg_shift);
11665 %}
11666
11667 // This pattern is automatically generated from aarch64_ad.m4
11668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11669 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11670 iRegL src1, iRegL src2,
11671 immI src3) %{
11672 match(Set dst (AndL src1 (RShiftL src2 src3)));
11673
11674 ins_cost(1.9 * INSN_COST);
11675 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11676
11677 ins_encode %{
11678 __ andr(as_Register($dst$$reg),
11679 as_Register($src1$$reg),
11680 as_Register($src2$$reg),
11681 Assembler::ASR,
11682 $src3$$constant & 0x3f);
11683 %}
11684
11685 ins_pipe(ialu_reg_reg_shift);
11686 %}
11687
11688 // This pattern is automatically generated from aarch64_ad.m4
11689 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11690 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11691 iRegIorL2I src1, iRegIorL2I src2,
11692 immI src3) %{
11693 match(Set dst (AndI src1 (LShiftI src2 src3)));
11694
11695 ins_cost(1.9 * INSN_COST);
11696 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11697
11698 ins_encode %{
11699 __ andw(as_Register($dst$$reg),
11700 as_Register($src1$$reg),
11701 as_Register($src2$$reg),
11702 Assembler::LSL,
11703 $src3$$constant & 0x1f);
11704 %}
11705
11706 ins_pipe(ialu_reg_reg_shift);
11707 %}
11708
11709 // This pattern is automatically generated from aarch64_ad.m4
11710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11711 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11712 iRegL src1, iRegL src2,
11713 immI src3) %{
11714 match(Set dst (AndL src1 (LShiftL src2 src3)));
11715
11716 ins_cost(1.9 * INSN_COST);
11717 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11718
11719 ins_encode %{
11720 __ andr(as_Register($dst$$reg),
11721 as_Register($src1$$reg),
11722 as_Register($src2$$reg),
11723 Assembler::LSL,
11724 $src3$$constant & 0x3f);
11725 %}
11726
11727 ins_pipe(ialu_reg_reg_shift);
11728 %}
11729
11730 // This pattern is automatically generated from aarch64_ad.m4
11731 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11732 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11733 iRegIorL2I src1, iRegIorL2I src2,
11734 immI src3) %{
11735 match(Set dst (AndI src1 (RotateRight src2 src3)));
11736
11737 ins_cost(1.9 * INSN_COST);
11738 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11739
11740 ins_encode %{
11741 __ andw(as_Register($dst$$reg),
11742 as_Register($src1$$reg),
11743 as_Register($src2$$reg),
11744 Assembler::ROR,
11745 $src3$$constant & 0x1f);
11746 %}
11747
11748 ins_pipe(ialu_reg_reg_shift);
11749 %}
11750
11751 // This pattern is automatically generated from aarch64_ad.m4
11752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11753 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11754 iRegL src1, iRegL src2,
11755 immI src3) %{
11756 match(Set dst (AndL src1 (RotateRight src2 src3)));
11757
11758 ins_cost(1.9 * INSN_COST);
11759 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11760
11761 ins_encode %{
11762 __ andr(as_Register($dst$$reg),
11763 as_Register($src1$$reg),
11764 as_Register($src2$$reg),
11765 Assembler::ROR,
11766 $src3$$constant & 0x3f);
11767 %}
11768
11769 ins_pipe(ialu_reg_reg_shift);
11770 %}
11771
11772 // This pattern is automatically generated from aarch64_ad.m4
11773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11774 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11775 iRegIorL2I src1, iRegIorL2I src2,
11776 immI src3) %{
11777 match(Set dst (XorI src1 (URShiftI src2 src3)));
11778
11779 ins_cost(1.9 * INSN_COST);
11780 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11781
11782 ins_encode %{
11783 __ eorw(as_Register($dst$$reg),
11784 as_Register($src1$$reg),
11785 as_Register($src2$$reg),
11786 Assembler::LSR,
11787 $src3$$constant & 0x1f);
11788 %}
11789
11790 ins_pipe(ialu_reg_reg_shift);
11791 %}
11792
11793 // This pattern is automatically generated from aarch64_ad.m4
11794 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11795 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11796 iRegL src1, iRegL src2,
11797 immI src3) %{
11798 match(Set dst (XorL src1 (URShiftL src2 src3)));
11799
11800 ins_cost(1.9 * INSN_COST);
11801 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11802
11803 ins_encode %{
11804 __ eor(as_Register($dst$$reg),
11805 as_Register($src1$$reg),
11806 as_Register($src2$$reg),
11807 Assembler::LSR,
11808 $src3$$constant & 0x3f);
11809 %}
11810
11811 ins_pipe(ialu_reg_reg_shift);
11812 %}
11813
11814 // This pattern is automatically generated from aarch64_ad.m4
11815 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11816 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11817 iRegIorL2I src1, iRegIorL2I src2,
11818 immI src3) %{
11819 match(Set dst (XorI src1 (RShiftI src2 src3)));
11820
11821 ins_cost(1.9 * INSN_COST);
11822 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11823
11824 ins_encode %{
11825 __ eorw(as_Register($dst$$reg),
11826 as_Register($src1$$reg),
11827 as_Register($src2$$reg),
11828 Assembler::ASR,
11829 $src3$$constant & 0x1f);
11830 %}
11831
11832 ins_pipe(ialu_reg_reg_shift);
11833 %}
11834
11835 // This pattern is automatically generated from aarch64_ad.m4
11836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11837 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11838 iRegL src1, iRegL src2,
11839 immI src3) %{
11840 match(Set dst (XorL src1 (RShiftL src2 src3)));
11841
11842 ins_cost(1.9 * INSN_COST);
11843 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11844
11845 ins_encode %{
11846 __ eor(as_Register($dst$$reg),
11847 as_Register($src1$$reg),
11848 as_Register($src2$$reg),
11849 Assembler::ASR,
11850 $src3$$constant & 0x3f);
11851 %}
11852
11853 ins_pipe(ialu_reg_reg_shift);
11854 %}
11855
11856 // This pattern is automatically generated from aarch64_ad.m4
11857 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11858 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11859 iRegIorL2I src1, iRegIorL2I src2,
11860 immI src3) %{
11861 match(Set dst (XorI src1 (LShiftI src2 src3)));
11862
11863 ins_cost(1.9 * INSN_COST);
11864 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11865
11866 ins_encode %{
11867 __ eorw(as_Register($dst$$reg),
11868 as_Register($src1$$reg),
11869 as_Register($src2$$reg),
11870 Assembler::LSL,
11871 $src3$$constant & 0x1f);
11872 %}
11873
11874 ins_pipe(ialu_reg_reg_shift);
11875 %}
11876
11877 // This pattern is automatically generated from aarch64_ad.m4
11878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11879 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11880 iRegL src1, iRegL src2,
11881 immI src3) %{
11882 match(Set dst (XorL src1 (LShiftL src2 src3)));
11883
11884 ins_cost(1.9 * INSN_COST);
11885 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11886
11887 ins_encode %{
11888 __ eor(as_Register($dst$$reg),
11889 as_Register($src1$$reg),
11890 as_Register($src2$$reg),
11891 Assembler::LSL,
11892 $src3$$constant & 0x3f);
11893 %}
11894
11895 ins_pipe(ialu_reg_reg_shift);
11896 %}
11897
11898 // This pattern is automatically generated from aarch64_ad.m4
11899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11900 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11901 iRegIorL2I src1, iRegIorL2I src2,
11902 immI src3) %{
11903 match(Set dst (XorI src1 (RotateRight src2 src3)));
11904
11905 ins_cost(1.9 * INSN_COST);
11906 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11907
11908 ins_encode %{
11909 __ eorw(as_Register($dst$$reg),
11910 as_Register($src1$$reg),
11911 as_Register($src2$$reg),
11912 Assembler::ROR,
11913 $src3$$constant & 0x1f);
11914 %}
11915
11916 ins_pipe(ialu_reg_reg_shift);
11917 %}
11918
11919 // This pattern is automatically generated from aarch64_ad.m4
11920 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11921 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11922 iRegL src1, iRegL src2,
11923 immI src3) %{
11924 match(Set dst (XorL src1 (RotateRight src2 src3)));
11925
11926 ins_cost(1.9 * INSN_COST);
11927 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11928
11929 ins_encode %{
11930 __ eor(as_Register($dst$$reg),
11931 as_Register($src1$$reg),
11932 as_Register($src2$$reg),
11933 Assembler::ROR,
11934 $src3$$constant & 0x3f);
11935 %}
11936
11937 ins_pipe(ialu_reg_reg_shift);
11938 %}
11939
11940 // This pattern is automatically generated from aarch64_ad.m4
11941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11942 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11943 iRegIorL2I src1, iRegIorL2I src2,
11944 immI src3) %{
11945 match(Set dst (OrI src1 (URShiftI src2 src3)));
11946
11947 ins_cost(1.9 * INSN_COST);
11948 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11949
11950 ins_encode %{
11951 __ orrw(as_Register($dst$$reg),
11952 as_Register($src1$$reg),
11953 as_Register($src2$$reg),
11954 Assembler::LSR,
11955 $src3$$constant & 0x1f);
11956 %}
11957
11958 ins_pipe(ialu_reg_reg_shift);
11959 %}
11960
11961 // This pattern is automatically generated from aarch64_ad.m4
11962 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11963 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11964 iRegL src1, iRegL src2,
11965 immI src3) %{
11966 match(Set dst (OrL src1 (URShiftL src2 src3)));
11967
11968 ins_cost(1.9 * INSN_COST);
11969 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11970
11971 ins_encode %{
11972 __ orr(as_Register($dst$$reg),
11973 as_Register($src1$$reg),
11974 as_Register($src2$$reg),
11975 Assembler::LSR,
11976 $src3$$constant & 0x3f);
11977 %}
11978
11979 ins_pipe(ialu_reg_reg_shift);
11980 %}
11981
11982 // This pattern is automatically generated from aarch64_ad.m4
11983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11984 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11985 iRegIorL2I src1, iRegIorL2I src2,
11986 immI src3) %{
11987 match(Set dst (OrI src1 (RShiftI src2 src3)));
11988
11989 ins_cost(1.9 * INSN_COST);
11990 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11991
11992 ins_encode %{
11993 __ orrw(as_Register($dst$$reg),
11994 as_Register($src1$$reg),
11995 as_Register($src2$$reg),
11996 Assembler::ASR,
11997 $src3$$constant & 0x1f);
11998 %}
11999
12000 ins_pipe(ialu_reg_reg_shift);
12001 %}
12002
12003 // This pattern is automatically generated from aarch64_ad.m4
12004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12005 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12006 iRegL src1, iRegL src2,
12007 immI src3) %{
12008 match(Set dst (OrL src1 (RShiftL src2 src3)));
12009
12010 ins_cost(1.9 * INSN_COST);
12011 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12012
12013 ins_encode %{
12014 __ orr(as_Register($dst$$reg),
12015 as_Register($src1$$reg),
12016 as_Register($src2$$reg),
12017 Assembler::ASR,
12018 $src3$$constant & 0x3f);
12019 %}
12020
12021 ins_pipe(ialu_reg_reg_shift);
12022 %}
12023
12024 // This pattern is automatically generated from aarch64_ad.m4
12025 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12026 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12027 iRegIorL2I src1, iRegIorL2I src2,
12028 immI src3) %{
12029 match(Set dst (OrI src1 (LShiftI src2 src3)));
12030
12031 ins_cost(1.9 * INSN_COST);
12032 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12033
12034 ins_encode %{
12035 __ orrw(as_Register($dst$$reg),
12036 as_Register($src1$$reg),
12037 as_Register($src2$$reg),
12038 Assembler::LSL,
12039 $src3$$constant & 0x1f);
12040 %}
12041
12042 ins_pipe(ialu_reg_reg_shift);
12043 %}
12044
12045 // This pattern is automatically generated from aarch64_ad.m4
12046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12047 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12048 iRegL src1, iRegL src2,
12049 immI src3) %{
12050 match(Set dst (OrL src1 (LShiftL src2 src3)));
12051
12052 ins_cost(1.9 * INSN_COST);
12053 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12054
12055 ins_encode %{
12056 __ orr(as_Register($dst$$reg),
12057 as_Register($src1$$reg),
12058 as_Register($src2$$reg),
12059 Assembler::LSL,
12060 $src3$$constant & 0x3f);
12061 %}
12062
12063 ins_pipe(ialu_reg_reg_shift);
12064 %}
12065
12066 // This pattern is automatically generated from aarch64_ad.m4
12067 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12068 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12069 iRegIorL2I src1, iRegIorL2I src2,
12070 immI src3) %{
12071 match(Set dst (OrI src1 (RotateRight src2 src3)));
12072
12073 ins_cost(1.9 * INSN_COST);
12074 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12075
12076 ins_encode %{
12077 __ orrw(as_Register($dst$$reg),
12078 as_Register($src1$$reg),
12079 as_Register($src2$$reg),
12080 Assembler::ROR,
12081 $src3$$constant & 0x1f);
12082 %}
12083
12084 ins_pipe(ialu_reg_reg_shift);
12085 %}
12086
12087 // This pattern is automatically generated from aarch64_ad.m4
12088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12089 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12090 iRegL src1, iRegL src2,
12091 immI src3) %{
12092 match(Set dst (OrL src1 (RotateRight src2 src3)));
12093
12094 ins_cost(1.9 * INSN_COST);
12095 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12096
12097 ins_encode %{
12098 __ orr(as_Register($dst$$reg),
12099 as_Register($src1$$reg),
12100 as_Register($src2$$reg),
12101 Assembler::ROR,
12102 $src3$$constant & 0x3f);
12103 %}
12104
12105 ins_pipe(ialu_reg_reg_shift);
12106 %}
12107
12108 // This pattern is automatically generated from aarch64_ad.m4
12109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12110 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12111 iRegIorL2I src1, iRegIorL2I src2,
12112 immI src3) %{
12113 match(Set dst (AddI src1 (URShiftI src2 src3)));
12114
12115 ins_cost(1.9 * INSN_COST);
12116 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12117
12118 ins_encode %{
12119 __ addw(as_Register($dst$$reg),
12120 as_Register($src1$$reg),
12121 as_Register($src2$$reg),
12122 Assembler::LSR,
12123 $src3$$constant & 0x1f);
12124 %}
12125
12126 ins_pipe(ialu_reg_reg_shift);
12127 %}
12128
12129 // This pattern is automatically generated from aarch64_ad.m4
12130 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12131 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12132 iRegL src1, iRegL src2,
12133 immI src3) %{
12134 match(Set dst (AddL src1 (URShiftL src2 src3)));
12135
12136 ins_cost(1.9 * INSN_COST);
12137 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12138
12139 ins_encode %{
12140 __ add(as_Register($dst$$reg),
12141 as_Register($src1$$reg),
12142 as_Register($src2$$reg),
12143 Assembler::LSR,
12144 $src3$$constant & 0x3f);
12145 %}
12146
12147 ins_pipe(ialu_reg_reg_shift);
12148 %}
12149
12150 // This pattern is automatically generated from aarch64_ad.m4
12151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12152 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12153 iRegIorL2I src1, iRegIorL2I src2,
12154 immI src3) %{
12155 match(Set dst (AddI src1 (RShiftI src2 src3)));
12156
12157 ins_cost(1.9 * INSN_COST);
12158 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12159
12160 ins_encode %{
12161 __ addw(as_Register($dst$$reg),
12162 as_Register($src1$$reg),
12163 as_Register($src2$$reg),
12164 Assembler::ASR,
12165 $src3$$constant & 0x1f);
12166 %}
12167
12168 ins_pipe(ialu_reg_reg_shift);
12169 %}
12170
12171 // This pattern is automatically generated from aarch64_ad.m4
12172 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12173 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12174 iRegL src1, iRegL src2,
12175 immI src3) %{
12176 match(Set dst (AddL src1 (RShiftL src2 src3)));
12177
12178 ins_cost(1.9 * INSN_COST);
12179 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12180
12181 ins_encode %{
12182 __ add(as_Register($dst$$reg),
12183 as_Register($src1$$reg),
12184 as_Register($src2$$reg),
12185 Assembler::ASR,
12186 $src3$$constant & 0x3f);
12187 %}
12188
12189 ins_pipe(ialu_reg_reg_shift);
12190 %}
12191
12192 // This pattern is automatically generated from aarch64_ad.m4
12193 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12194 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12195 iRegIorL2I src1, iRegIorL2I src2,
12196 immI src3) %{
12197 match(Set dst (AddI src1 (LShiftI src2 src3)));
12198
12199 ins_cost(1.9 * INSN_COST);
12200 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12201
12202 ins_encode %{
12203 __ addw(as_Register($dst$$reg),
12204 as_Register($src1$$reg),
12205 as_Register($src2$$reg),
12206 Assembler::LSL,
12207 $src3$$constant & 0x1f);
12208 %}
12209
12210 ins_pipe(ialu_reg_reg_shift);
12211 %}
12212
12213 // This pattern is automatically generated from aarch64_ad.m4
12214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12215 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12216 iRegL src1, iRegL src2,
12217 immI src3) %{
12218 match(Set dst (AddL src1 (LShiftL src2 src3)));
12219
12220 ins_cost(1.9 * INSN_COST);
12221 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12222
12223 ins_encode %{
12224 __ add(as_Register($dst$$reg),
12225 as_Register($src1$$reg),
12226 as_Register($src2$$reg),
12227 Assembler::LSL,
12228 $src3$$constant & 0x3f);
12229 %}
12230
12231 ins_pipe(ialu_reg_reg_shift);
12232 %}
12233
12234 // This pattern is automatically generated from aarch64_ad.m4
12235 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12236 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12237 iRegIorL2I src1, iRegIorL2I src2,
12238 immI src3) %{
12239 match(Set dst (SubI src1 (URShiftI src2 src3)));
12240
12241 ins_cost(1.9 * INSN_COST);
12242 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12243
12244 ins_encode %{
12245 __ subw(as_Register($dst$$reg),
12246 as_Register($src1$$reg),
12247 as_Register($src2$$reg),
12248 Assembler::LSR,
12249 $src3$$constant & 0x1f);
12250 %}
12251
12252 ins_pipe(ialu_reg_reg_shift);
12253 %}
12254
12255 // This pattern is automatically generated from aarch64_ad.m4
12256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12257 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12258 iRegL src1, iRegL src2,
12259 immI src3) %{
12260 match(Set dst (SubL src1 (URShiftL src2 src3)));
12261
12262 ins_cost(1.9 * INSN_COST);
12263 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12264
12265 ins_encode %{
12266 __ sub(as_Register($dst$$reg),
12267 as_Register($src1$$reg),
12268 as_Register($src2$$reg),
12269 Assembler::LSR,
12270 $src3$$constant & 0x3f);
12271 %}
12272
12273 ins_pipe(ialu_reg_reg_shift);
12274 %}
12275
12276 // This pattern is automatically generated from aarch64_ad.m4
12277 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12278 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12279 iRegIorL2I src1, iRegIorL2I src2,
12280 immI src3) %{
12281 match(Set dst (SubI src1 (RShiftI src2 src3)));
12282
12283 ins_cost(1.9 * INSN_COST);
12284 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12285
12286 ins_encode %{
12287 __ subw(as_Register($dst$$reg),
12288 as_Register($src1$$reg),
12289 as_Register($src2$$reg),
12290 Assembler::ASR,
12291 $src3$$constant & 0x1f);
12292 %}
12293
12294 ins_pipe(ialu_reg_reg_shift);
12295 %}
12296
12297 // This pattern is automatically generated from aarch64_ad.m4
12298 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12299 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12300 iRegL src1, iRegL src2,
12301 immI src3) %{
12302 match(Set dst (SubL src1 (RShiftL src2 src3)));
12303
12304 ins_cost(1.9 * INSN_COST);
12305 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12306
12307 ins_encode %{
12308 __ sub(as_Register($dst$$reg),
12309 as_Register($src1$$reg),
12310 as_Register($src2$$reg),
12311 Assembler::ASR,
12312 $src3$$constant & 0x3f);
12313 %}
12314
12315 ins_pipe(ialu_reg_reg_shift);
12316 %}
12317
12318 // This pattern is automatically generated from aarch64_ad.m4
12319 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12320 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12321 iRegIorL2I src1, iRegIorL2I src2,
12322 immI src3) %{
12323 match(Set dst (SubI src1 (LShiftI src2 src3)));
12324
12325 ins_cost(1.9 * INSN_COST);
12326 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12327
12328 ins_encode %{
12329 __ subw(as_Register($dst$$reg),
12330 as_Register($src1$$reg),
12331 as_Register($src2$$reg),
12332 Assembler::LSL,
12333 $src3$$constant & 0x1f);
12334 %}
12335
12336 ins_pipe(ialu_reg_reg_shift);
12337 %}
12338
12339 // This pattern is automatically generated from aarch64_ad.m4
12340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12341 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12342 iRegL src1, iRegL src2,
12343 immI src3) %{
12344 match(Set dst (SubL src1 (LShiftL src2 src3)));
12345
12346 ins_cost(1.9 * INSN_COST);
12347 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12348
12349 ins_encode %{
12350 __ sub(as_Register($dst$$reg),
12351 as_Register($src1$$reg),
12352 as_Register($src2$$reg),
12353 Assembler::LSL,
12354 $src3$$constant & 0x3f);
12355 %}
12356
12357 ins_pipe(ialu_reg_reg_shift);
12358 %}
12359
12360 // This pattern is automatically generated from aarch64_ad.m4
12361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12362
12363 // Shift Left followed by Shift Right.
12364 // This idiom is used by the compiler for the i2b bytecode etc.
12365 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12366 %{
12367 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12368 ins_cost(INSN_COST * 2);
12369 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12370 ins_encode %{
12371 int lshift = $lshift_count$$constant & 63;
12372 int rshift = $rshift_count$$constant & 63;
12373 int s = 63 - lshift;
12374 int r = (rshift - lshift) & 63;
12375 __ sbfm(as_Register($dst$$reg),
12376 as_Register($src$$reg),
12377 r, s);
12378 %}
12379
12380 ins_pipe(ialu_reg_shift);
12381 %}
12382
12383 // This pattern is automatically generated from aarch64_ad.m4
12384 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12385
12386 // Shift Left followed by Shift Right.
12387 // This idiom is used by the compiler for the i2b bytecode etc.
12388 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12389 %{
12390 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12391 ins_cost(INSN_COST * 2);
12392 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12393 ins_encode %{
12394 int lshift = $lshift_count$$constant & 31;
12395 int rshift = $rshift_count$$constant & 31;
12396 int s = 31 - lshift;
12397 int r = (rshift - lshift) & 31;
12398 __ sbfmw(as_Register($dst$$reg),
12399 as_Register($src$$reg),
12400 r, s);
12401 %}
12402
12403 ins_pipe(ialu_reg_shift);
12404 %}
12405
12406 // This pattern is automatically generated from aarch64_ad.m4
12407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12408
12409 // Shift Left followed by Shift Right.
12410 // This idiom is used by the compiler for the i2b bytecode etc.
12411 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12412 %{
12413 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12414 ins_cost(INSN_COST * 2);
12415 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12416 ins_encode %{
12417 int lshift = $lshift_count$$constant & 63;
12418 int rshift = $rshift_count$$constant & 63;
12419 int s = 63 - lshift;
12420 int r = (rshift - lshift) & 63;
12421 __ ubfm(as_Register($dst$$reg),
12422 as_Register($src$$reg),
12423 r, s);
12424 %}
12425
12426 ins_pipe(ialu_reg_shift);
12427 %}
12428
12429 // This pattern is automatically generated from aarch64_ad.m4
12430 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12431
12432 // Shift Left followed by Shift Right.
12433 // This idiom is used by the compiler for the i2b bytecode etc.
12434 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12435 %{
12436 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12437 ins_cost(INSN_COST * 2);
12438 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12439 ins_encode %{
12440 int lshift = $lshift_count$$constant & 31;
12441 int rshift = $rshift_count$$constant & 31;
12442 int s = 31 - lshift;
12443 int r = (rshift - lshift) & 31;
12444 __ ubfmw(as_Register($dst$$reg),
12445 as_Register($src$$reg),
12446 r, s);
12447 %}
12448
12449 ins_pipe(ialu_reg_shift);
12450 %}
12451
12452 // Bitfield extract with shift & mask
12453
12454 // This pattern is automatically generated from aarch64_ad.m4
12455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12456 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12457 %{
12458 match(Set dst (AndI (URShiftI src rshift) mask));
12459 // Make sure we are not going to exceed what ubfxw can do.
12460 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12461
12462 ins_cost(INSN_COST);
12463 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12464 ins_encode %{
12465 int rshift = $rshift$$constant & 31;
12466 intptr_t mask = $mask$$constant;
12467 int width = exact_log2(mask+1);
12468 __ ubfxw(as_Register($dst$$reg),
12469 as_Register($src$$reg), rshift, width);
12470 %}
12471 ins_pipe(ialu_reg_shift);
12472 %}
12473
12474 // This pattern is automatically generated from aarch64_ad.m4
12475 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12476 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12477 %{
12478 match(Set dst (AndL (URShiftL src rshift) mask));
12479 // Make sure we are not going to exceed what ubfx can do.
12480 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12481
12482 ins_cost(INSN_COST);
12483 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12484 ins_encode %{
12485 int rshift = $rshift$$constant & 63;
12486 intptr_t mask = $mask$$constant;
12487 int width = exact_log2_long(mask+1);
12488 __ ubfx(as_Register($dst$$reg),
12489 as_Register($src$$reg), rshift, width);
12490 %}
12491 ins_pipe(ialu_reg_shift);
12492 %}
12493
12494
12495 // This pattern is automatically generated from aarch64_ad.m4
12496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12497
12498 // We can use ubfx when extending an And with a mask when we know mask
12499 // is positive. We know that because immI_bitmask guarantees it.
12500 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12501 %{
12502 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12503 // Make sure we are not going to exceed what ubfxw can do.
12504 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12505
12506 ins_cost(INSN_COST * 2);
12507 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12508 ins_encode %{
12509 int rshift = $rshift$$constant & 31;
12510 intptr_t mask = $mask$$constant;
12511 int width = exact_log2(mask+1);
12512 __ ubfx(as_Register($dst$$reg),
12513 as_Register($src$$reg), rshift, width);
12514 %}
12515 ins_pipe(ialu_reg_shift);
12516 %}
12517
12518
12519 // This pattern is automatically generated from aarch64_ad.m4
12520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12521
12522 // We can use ubfiz when masking by a positive number and then left shifting the result.
12523 // We know that the mask is positive because immI_bitmask guarantees it.
12524 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12525 %{
12526 match(Set dst (LShiftI (AndI src mask) lshift));
12527 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12528
12529 ins_cost(INSN_COST);
12530 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12531 ins_encode %{
12532 int lshift = $lshift$$constant & 31;
12533 intptr_t mask = $mask$$constant;
12534 int width = exact_log2(mask+1);
12535 __ ubfizw(as_Register($dst$$reg),
12536 as_Register($src$$reg), lshift, width);
12537 %}
12538 ins_pipe(ialu_reg_shift);
12539 %}
12540
12541 // This pattern is automatically generated from aarch64_ad.m4
12542 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12543
12544 // We can use ubfiz when masking by a positive number and then left shifting the result.
12545 // We know that the mask is positive because immL_bitmask guarantees it.
12546 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12547 %{
12548 match(Set dst (LShiftL (AndL src mask) lshift));
12549 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12550
12551 ins_cost(INSN_COST);
12552 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12553 ins_encode %{
12554 int lshift = $lshift$$constant & 63;
12555 intptr_t mask = $mask$$constant;
12556 int width = exact_log2_long(mask+1);
12557 __ ubfiz(as_Register($dst$$reg),
12558 as_Register($src$$reg), lshift, width);
12559 %}
12560 ins_pipe(ialu_reg_shift);
12561 %}
12562
12563 // This pattern is automatically generated from aarch64_ad.m4
12564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12565
12566 // We can use ubfiz when masking by a positive number and then left shifting the result.
12567 // We know that the mask is positive because immI_bitmask guarantees it.
12568 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12569 %{
12570 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12571 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12572
12573 ins_cost(INSN_COST);
12574 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12575 ins_encode %{
12576 int lshift = $lshift$$constant & 31;
12577 intptr_t mask = $mask$$constant;
12578 int width = exact_log2(mask+1);
12579 __ ubfizw(as_Register($dst$$reg),
12580 as_Register($src$$reg), lshift, width);
12581 %}
12582 ins_pipe(ialu_reg_shift);
12583 %}
12584
12585 // This pattern is automatically generated from aarch64_ad.m4
12586 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12587
12588 // We can use ubfiz when masking by a positive number and then left shifting the result.
12589 // We know that the mask is positive because immL_bitmask guarantees it.
12590 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12591 %{
12592 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12593 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12594
12595 ins_cost(INSN_COST);
12596 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12597 ins_encode %{
12598 int lshift = $lshift$$constant & 63;
12599 intptr_t mask = $mask$$constant;
12600 int width = exact_log2_long(mask+1);
12601 __ ubfiz(as_Register($dst$$reg),
12602 as_Register($src$$reg), lshift, width);
12603 %}
12604 ins_pipe(ialu_reg_shift);
12605 %}
12606
12607
12608 // This pattern is automatically generated from aarch64_ad.m4
12609 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12610
12611 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12612 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12613 %{
12614 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12615 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12616
12617 ins_cost(INSN_COST);
12618 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12619 ins_encode %{
12620 int lshift = $lshift$$constant & 63;
12621 intptr_t mask = $mask$$constant;
12622 int width = exact_log2(mask+1);
12623 __ ubfiz(as_Register($dst$$reg),
12624 as_Register($src$$reg), lshift, width);
12625 %}
12626 ins_pipe(ialu_reg_shift);
12627 %}
12628
12629 // This pattern is automatically generated from aarch64_ad.m4
12630 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12631
12632 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12633 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12634 %{
12635 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12636 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12637
12638 ins_cost(INSN_COST);
12639 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12640 ins_encode %{
12641 int lshift = $lshift$$constant & 31;
12642 intptr_t mask = $mask$$constant;
12643 int width = exact_log2(mask+1);
12644 __ ubfiz(as_Register($dst$$reg),
12645 as_Register($src$$reg), lshift, width);
12646 %}
12647 ins_pipe(ialu_reg_shift);
12648 %}
12649
12650 // This pattern is automatically generated from aarch64_ad.m4
12651 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12652
12653 // Can skip int2long conversions after AND with small bitmask
12654 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12655 %{
12656 match(Set dst (ConvI2L (AndI src msk)));
12657 ins_cost(INSN_COST);
12658 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12659 ins_encode %{
12660 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12661 %}
12662 ins_pipe(ialu_reg_shift);
12663 %}
12664
12665
12666 // Rotations
12667
12668 // This pattern is automatically generated from aarch64_ad.m4
12669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12670 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12671 %{
12672 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12673 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12674
12675 ins_cost(INSN_COST);
12676 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12677
12678 ins_encode %{
12679 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12680 $rshift$$constant & 63);
12681 %}
12682 ins_pipe(ialu_reg_reg_extr);
12683 %}
12684
12685
12686 // This pattern is automatically generated from aarch64_ad.m4
12687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12688 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12689 %{
12690 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12691 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12692
12693 ins_cost(INSN_COST);
12694 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12695
12696 ins_encode %{
12697 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12698 $rshift$$constant & 31);
12699 %}
12700 ins_pipe(ialu_reg_reg_extr);
12701 %}
12702
12703
12704 // This pattern is automatically generated from aarch64_ad.m4
12705 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12706 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12707 %{
12708 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12709 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12710
12711 ins_cost(INSN_COST);
12712 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12713
12714 ins_encode %{
12715 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12716 $rshift$$constant & 63);
12717 %}
12718 ins_pipe(ialu_reg_reg_extr);
12719 %}
12720
12721
12722 // This pattern is automatically generated from aarch64_ad.m4
12723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12724 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12725 %{
12726 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12727 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12728
12729 ins_cost(INSN_COST);
12730 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12731
12732 ins_encode %{
12733 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12734 $rshift$$constant & 31);
12735 %}
12736 ins_pipe(ialu_reg_reg_extr);
12737 %}
12738
12739 // This pattern is automatically generated from aarch64_ad.m4
12740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12741 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12742 %{
12743 match(Set dst (RotateRight src shift));
12744
12745 ins_cost(INSN_COST);
12746 format %{ "ror $dst, $src, $shift" %}
12747
12748 ins_encode %{
12749 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12750 $shift$$constant & 0x1f);
12751 %}
12752 ins_pipe(ialu_reg_reg_vshift);
12753 %}
12754
12755 // This pattern is automatically generated from aarch64_ad.m4
12756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12757 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12758 %{
12759 match(Set dst (RotateRight src shift));
12760
12761 ins_cost(INSN_COST);
12762 format %{ "ror $dst, $src, $shift" %}
12763
12764 ins_encode %{
12765 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12766 $shift$$constant & 0x3f);
12767 %}
12768 ins_pipe(ialu_reg_reg_vshift);
12769 %}
12770
12771 // This pattern is automatically generated from aarch64_ad.m4
12772 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12773 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12774 %{
12775 match(Set dst (RotateRight src shift));
12776
12777 ins_cost(INSN_COST);
12778 format %{ "ror $dst, $src, $shift" %}
12779
12780 ins_encode %{
12781 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12782 %}
12783 ins_pipe(ialu_reg_reg_vshift);
12784 %}
12785
12786 // This pattern is automatically generated from aarch64_ad.m4
12787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12788 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12789 %{
12790 match(Set dst (RotateRight src shift));
12791
12792 ins_cost(INSN_COST);
12793 format %{ "ror $dst, $src, $shift" %}
12794
12795 ins_encode %{
12796 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12797 %}
12798 ins_pipe(ialu_reg_reg_vshift);
12799 %}
12800
12801 // This pattern is automatically generated from aarch64_ad.m4
12802 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12803 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12804 %{
12805 match(Set dst (RotateLeft src shift));
12806
12807 ins_cost(INSN_COST);
12808 format %{ "rol $dst, $src, $shift" %}
12809
12810 ins_encode %{
12811 __ subw(rscratch1, zr, as_Register($shift$$reg));
12812 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12813 %}
12814 ins_pipe(ialu_reg_reg_vshift);
12815 %}
12816
12817 // This pattern is automatically generated from aarch64_ad.m4
12818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12819 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12820 %{
12821 match(Set dst (RotateLeft src shift));
12822
12823 ins_cost(INSN_COST);
12824 format %{ "rol $dst, $src, $shift" %}
12825
12826 ins_encode %{
12827 __ subw(rscratch1, zr, as_Register($shift$$reg));
12828 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12829 %}
12830 ins_pipe(ialu_reg_reg_vshift);
12831 %}
12832
12833
12834 // Add/subtract (extended)
12835
12836 // This pattern is automatically generated from aarch64_ad.m4
12837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12838 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12839 %{
12840 match(Set dst (AddL src1 (ConvI2L src2)));
12841 ins_cost(INSN_COST);
12842 format %{ "add $dst, $src1, $src2, sxtw" %}
12843
12844 ins_encode %{
12845 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12846 as_Register($src2$$reg), ext::sxtw);
12847 %}
12848 ins_pipe(ialu_reg_reg);
12849 %}
12850
12851 // This pattern is automatically generated from aarch64_ad.m4
12852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12853 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12854 %{
12855 match(Set dst (SubL src1 (ConvI2L src2)));
12856 ins_cost(INSN_COST);
12857 format %{ "sub $dst, $src1, $src2, sxtw" %}
12858
12859 ins_encode %{
12860 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12861 as_Register($src2$$reg), ext::sxtw);
12862 %}
12863 ins_pipe(ialu_reg_reg);
12864 %}
12865
12866 // This pattern is automatically generated from aarch64_ad.m4
12867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12868 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12869 %{
12870 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12871 ins_cost(INSN_COST);
12872 format %{ "add $dst, $src1, $src2, sxth" %}
12873
12874 ins_encode %{
12875 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12876 as_Register($src2$$reg), ext::sxth);
12877 %}
12878 ins_pipe(ialu_reg_reg);
12879 %}
12880
12881 // This pattern is automatically generated from aarch64_ad.m4
12882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12883 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12884 %{
12885 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12886 ins_cost(INSN_COST);
12887 format %{ "add $dst, $src1, $src2, sxtb" %}
12888
12889 ins_encode %{
12890 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12891 as_Register($src2$$reg), ext::sxtb);
12892 %}
12893 ins_pipe(ialu_reg_reg);
12894 %}
12895
12896 // This pattern is automatically generated from aarch64_ad.m4
12897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12898 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12899 %{
12900 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12901 ins_cost(INSN_COST);
12902 format %{ "add $dst, $src1, $src2, uxtb" %}
12903
12904 ins_encode %{
12905 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12906 as_Register($src2$$reg), ext::uxtb);
12907 %}
12908 ins_pipe(ialu_reg_reg);
12909 %}
12910
12911 // This pattern is automatically generated from aarch64_ad.m4
12912 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12913 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12914 %{
12915 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12916 ins_cost(INSN_COST);
12917 format %{ "add $dst, $src1, $src2, sxth" %}
12918
12919 ins_encode %{
12920 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12921 as_Register($src2$$reg), ext::sxth);
12922 %}
12923 ins_pipe(ialu_reg_reg);
12924 %}
12925
12926 // This pattern is automatically generated from aarch64_ad.m4
12927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12928 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12929 %{
12930 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12931 ins_cost(INSN_COST);
12932 format %{ "add $dst, $src1, $src2, sxtw" %}
12933
12934 ins_encode %{
12935 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12936 as_Register($src2$$reg), ext::sxtw);
12937 %}
12938 ins_pipe(ialu_reg_reg);
12939 %}
12940
12941 // This pattern is automatically generated from aarch64_ad.m4
12942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12943 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12944 %{
12945 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12946 ins_cost(INSN_COST);
12947 format %{ "add $dst, $src1, $src2, sxtb" %}
12948
12949 ins_encode %{
12950 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12951 as_Register($src2$$reg), ext::sxtb);
12952 %}
12953 ins_pipe(ialu_reg_reg);
12954 %}
12955
12956 // This pattern is automatically generated from aarch64_ad.m4
12957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12958 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12959 %{
12960 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12961 ins_cost(INSN_COST);
12962 format %{ "add $dst, $src1, $src2, uxtb" %}
12963
12964 ins_encode %{
12965 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12966 as_Register($src2$$reg), ext::uxtb);
12967 %}
12968 ins_pipe(ialu_reg_reg);
12969 %}
12970
12971 // This pattern is automatically generated from aarch64_ad.m4
12972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12973 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12974 %{
12975 match(Set dst (AddI src1 (AndI src2 mask)));
12976 ins_cost(INSN_COST);
12977 format %{ "addw $dst, $src1, $src2, uxtb" %}
12978
12979 ins_encode %{
12980 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12981 as_Register($src2$$reg), ext::uxtb);
12982 %}
12983 ins_pipe(ialu_reg_reg);
12984 %}
12985
12986 // This pattern is automatically generated from aarch64_ad.m4
12987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12988 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12989 %{
12990 match(Set dst (AddI src1 (AndI src2 mask)));
12991 ins_cost(INSN_COST);
12992 format %{ "addw $dst, $src1, $src2, uxth" %}
12993
12994 ins_encode %{
12995 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12996 as_Register($src2$$reg), ext::uxth);
12997 %}
12998 ins_pipe(ialu_reg_reg);
12999 %}
13000
13001 // This pattern is automatically generated from aarch64_ad.m4
13002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13003 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13004 %{
13005 match(Set dst (AddL src1 (AndL src2 mask)));
13006 ins_cost(INSN_COST);
13007 format %{ "add $dst, $src1, $src2, uxtb" %}
13008
13009 ins_encode %{
13010 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13011 as_Register($src2$$reg), ext::uxtb);
13012 %}
13013 ins_pipe(ialu_reg_reg);
13014 %}
13015
13016 // This pattern is automatically generated from aarch64_ad.m4
13017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13018 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13019 %{
13020 match(Set dst (AddL src1 (AndL src2 mask)));
13021 ins_cost(INSN_COST);
13022 format %{ "add $dst, $src1, $src2, uxth" %}
13023
13024 ins_encode %{
13025 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13026 as_Register($src2$$reg), ext::uxth);
13027 %}
13028 ins_pipe(ialu_reg_reg);
13029 %}
13030
13031 // This pattern is automatically generated from aarch64_ad.m4
13032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13033 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13034 %{
13035 match(Set dst (AddL src1 (AndL src2 mask)));
13036 ins_cost(INSN_COST);
13037 format %{ "add $dst, $src1, $src2, uxtw" %}
13038
13039 ins_encode %{
13040 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13041 as_Register($src2$$reg), ext::uxtw);
13042 %}
13043 ins_pipe(ialu_reg_reg);
13044 %}
13045
13046 // This pattern is automatically generated from aarch64_ad.m4
13047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13048 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13049 %{
13050 match(Set dst (SubI src1 (AndI src2 mask)));
13051 ins_cost(INSN_COST);
13052 format %{ "subw $dst, $src1, $src2, uxtb" %}
13053
13054 ins_encode %{
13055 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13056 as_Register($src2$$reg), ext::uxtb);
13057 %}
13058 ins_pipe(ialu_reg_reg);
13059 %}
13060
13061 // This pattern is automatically generated from aarch64_ad.m4
13062 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13063 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13064 %{
13065 match(Set dst (SubI src1 (AndI src2 mask)));
13066 ins_cost(INSN_COST);
13067 format %{ "subw $dst, $src1, $src2, uxth" %}
13068
13069 ins_encode %{
13070 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13071 as_Register($src2$$reg), ext::uxth);
13072 %}
13073 ins_pipe(ialu_reg_reg);
13074 %}
13075
13076 // This pattern is automatically generated from aarch64_ad.m4
13077 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13078 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13079 %{
13080 match(Set dst (SubL src1 (AndL src2 mask)));
13081 ins_cost(INSN_COST);
13082 format %{ "sub $dst, $src1, $src2, uxtb" %}
13083
13084 ins_encode %{
13085 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13086 as_Register($src2$$reg), ext::uxtb);
13087 %}
13088 ins_pipe(ialu_reg_reg);
13089 %}
13090
13091 // This pattern is automatically generated from aarch64_ad.m4
13092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13093 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13094 %{
13095 match(Set dst (SubL src1 (AndL src2 mask)));
13096 ins_cost(INSN_COST);
13097 format %{ "sub $dst, $src1, $src2, uxth" %}
13098
13099 ins_encode %{
13100 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13101 as_Register($src2$$reg), ext::uxth);
13102 %}
13103 ins_pipe(ialu_reg_reg);
13104 %}
13105
13106 // This pattern is automatically generated from aarch64_ad.m4
13107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13108 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13109 %{
13110 match(Set dst (SubL src1 (AndL src2 mask)));
13111 ins_cost(INSN_COST);
13112 format %{ "sub $dst, $src1, $src2, uxtw" %}
13113
13114 ins_encode %{
13115 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13116 as_Register($src2$$reg), ext::uxtw);
13117 %}
13118 ins_pipe(ialu_reg_reg);
13119 %}
13120
13121
13122 // This pattern is automatically generated from aarch64_ad.m4
13123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13124 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13125 %{
13126 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13127 ins_cost(1.9 * INSN_COST);
13128 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13129
13130 ins_encode %{
13131 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13132 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13133 %}
13134 ins_pipe(ialu_reg_reg_shift);
13135 %}
13136
13137 // This pattern is automatically generated from aarch64_ad.m4
13138 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13139 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13140 %{
13141 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13142 ins_cost(1.9 * INSN_COST);
13143 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13144
13145 ins_encode %{
13146 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13147 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13148 %}
13149 ins_pipe(ialu_reg_reg_shift);
13150 %}
13151
13152 // This pattern is automatically generated from aarch64_ad.m4
13153 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13154 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13155 %{
13156 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13157 ins_cost(1.9 * INSN_COST);
13158 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13159
13160 ins_encode %{
13161 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13162 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13163 %}
13164 ins_pipe(ialu_reg_reg_shift);
13165 %}
13166
13167 // This pattern is automatically generated from aarch64_ad.m4
13168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13169 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13170 %{
13171 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13172 ins_cost(1.9 * INSN_COST);
13173 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13174
13175 ins_encode %{
13176 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13177 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13178 %}
13179 ins_pipe(ialu_reg_reg_shift);
13180 %}
13181
13182 // This pattern is automatically generated from aarch64_ad.m4
13183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13184 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13185 %{
13186 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13187 ins_cost(1.9 * INSN_COST);
13188 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13189
13190 ins_encode %{
13191 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13192 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13193 %}
13194 ins_pipe(ialu_reg_reg_shift);
13195 %}
13196
13197 // This pattern is automatically generated from aarch64_ad.m4
13198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13199 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13200 %{
13201 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13202 ins_cost(1.9 * INSN_COST);
13203 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13204
13205 ins_encode %{
13206 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13207 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13208 %}
13209 ins_pipe(ialu_reg_reg_shift);
13210 %}
13211
13212 // This pattern is automatically generated from aarch64_ad.m4
13213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13214 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13215 %{
13216 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13217 ins_cost(1.9 * INSN_COST);
13218 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13219
13220 ins_encode %{
13221 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13222 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13223 %}
13224 ins_pipe(ialu_reg_reg_shift);
13225 %}
13226
13227 // This pattern is automatically generated from aarch64_ad.m4
13228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13229 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13230 %{
13231 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13232 ins_cost(1.9 * INSN_COST);
13233 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13234
13235 ins_encode %{
13236 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13237 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13238 %}
13239 ins_pipe(ialu_reg_reg_shift);
13240 %}
13241
13242 // This pattern is automatically generated from aarch64_ad.m4
13243 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13244 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13245 %{
13246 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13247 ins_cost(1.9 * INSN_COST);
13248 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13249
13250 ins_encode %{
13251 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13252 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13253 %}
13254 ins_pipe(ialu_reg_reg_shift);
13255 %}
13256
13257 // This pattern is automatically generated from aarch64_ad.m4
13258 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13259 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13260 %{
13261 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13262 ins_cost(1.9 * INSN_COST);
13263 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13264
13265 ins_encode %{
13266 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13267 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13268 %}
13269 ins_pipe(ialu_reg_reg_shift);
13270 %}
13271
13272 // This pattern is automatically generated from aarch64_ad.m4
13273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13274 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13275 %{
13276 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13277 ins_cost(1.9 * INSN_COST);
13278 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13279
13280 ins_encode %{
13281 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13282 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13283 %}
13284 ins_pipe(ialu_reg_reg_shift);
13285 %}
13286
13287 // This pattern is automatically generated from aarch64_ad.m4
13288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13289 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13290 %{
13291 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13292 ins_cost(1.9 * INSN_COST);
13293 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13294
13295 ins_encode %{
13296 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13297 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13298 %}
13299 ins_pipe(ialu_reg_reg_shift);
13300 %}
13301
13302 // This pattern is automatically generated from aarch64_ad.m4
13303 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13304 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13305 %{
13306 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13307 ins_cost(1.9 * INSN_COST);
13308 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13309
13310 ins_encode %{
13311 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13312 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13313 %}
13314 ins_pipe(ialu_reg_reg_shift);
13315 %}
13316
13317 // This pattern is automatically generated from aarch64_ad.m4
13318 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13319 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13320 %{
13321 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13322 ins_cost(1.9 * INSN_COST);
13323 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13324
13325 ins_encode %{
13326 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13327 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13328 %}
13329 ins_pipe(ialu_reg_reg_shift);
13330 %}
13331
13332 // This pattern is automatically generated from aarch64_ad.m4
13333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13334 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13335 %{
13336 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13337 ins_cost(1.9 * INSN_COST);
13338 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13339
13340 ins_encode %{
13341 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13342 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13343 %}
13344 ins_pipe(ialu_reg_reg_shift);
13345 %}
13346
13347 // This pattern is automatically generated from aarch64_ad.m4
13348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13349 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13350 %{
13351 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13352 ins_cost(1.9 * INSN_COST);
13353 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13354
13355 ins_encode %{
13356 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13357 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13358 %}
13359 ins_pipe(ialu_reg_reg_shift);
13360 %}
13361
13362 // This pattern is automatically generated from aarch64_ad.m4
13363 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13364 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13365 %{
13366 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13367 ins_cost(1.9 * INSN_COST);
13368 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13369
13370 ins_encode %{
13371 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13372 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13373 %}
13374 ins_pipe(ialu_reg_reg_shift);
13375 %}
13376
13377 // This pattern is automatically generated from aarch64_ad.m4
13378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13379 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13380 %{
13381 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13382 ins_cost(1.9 * INSN_COST);
13383 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13384
13385 ins_encode %{
13386 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13387 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13388 %}
13389 ins_pipe(ialu_reg_reg_shift);
13390 %}
13391
13392 // This pattern is automatically generated from aarch64_ad.m4
13393 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13394 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13395 %{
13396 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13397 ins_cost(1.9 * INSN_COST);
13398 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13399
13400 ins_encode %{
13401 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13402 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13403 %}
13404 ins_pipe(ialu_reg_reg_shift);
13405 %}
13406
13407 // This pattern is automatically generated from aarch64_ad.m4
13408 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13409 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13410 %{
13411 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13412 ins_cost(1.9 * INSN_COST);
13413 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13414
13415 ins_encode %{
13416 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13417 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13418 %}
13419 ins_pipe(ialu_reg_reg_shift);
13420 %}
13421
13422 // This pattern is automatically generated from aarch64_ad.m4
13423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13424 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13425 %{
13426 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13427 ins_cost(1.9 * INSN_COST);
13428 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13429
13430 ins_encode %{
13431 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13432 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13433 %}
13434 ins_pipe(ialu_reg_reg_shift);
13435 %}
13436
13437 // This pattern is automatically generated from aarch64_ad.m4
13438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13439 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13440 %{
13441 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13442 ins_cost(1.9 * INSN_COST);
13443 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13444
13445 ins_encode %{
13446 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13447 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13448 %}
13449 ins_pipe(ialu_reg_reg_shift);
13450 %}
13451
13452 // This pattern is automatically generated from aarch64_ad.m4
13453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13454 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13455 %{
13456 effect(DEF dst, USE src1, USE src2, USE cr);
13457 ins_cost(INSN_COST * 2);
13458 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13459
13460 ins_encode %{
13461 __ cselw($dst$$Register,
13462 $src1$$Register,
13463 $src2$$Register,
13464 Assembler::LT);
13465 %}
13466 ins_pipe(icond_reg_reg);
13467 %}
13468
13469 // This pattern is automatically generated from aarch64_ad.m4
13470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13471 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13472 %{
13473 effect(DEF dst, USE src1, USE src2, USE cr);
13474 ins_cost(INSN_COST * 2);
13475 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13476
13477 ins_encode %{
13478 __ cselw($dst$$Register,
13479 $src1$$Register,
13480 $src2$$Register,
13481 Assembler::GT);
13482 %}
13483 ins_pipe(icond_reg_reg);
13484 %}
13485
13486 // This pattern is automatically generated from aarch64_ad.m4
13487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13488 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13489 %{
13490 effect(DEF dst, USE src1, USE cr);
13491 ins_cost(INSN_COST * 2);
13492 format %{ "cselw $dst, $src1, zr lt\t" %}
13493
13494 ins_encode %{
13495 __ cselw($dst$$Register,
13496 $src1$$Register,
13497 zr,
13498 Assembler::LT);
13499 %}
13500 ins_pipe(icond_reg);
13501 %}
13502
13503 // This pattern is automatically generated from aarch64_ad.m4
13504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13505 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13506 %{
13507 effect(DEF dst, USE src1, USE cr);
13508 ins_cost(INSN_COST * 2);
13509 format %{ "cselw $dst, $src1, zr gt\t" %}
13510
13511 ins_encode %{
13512 __ cselw($dst$$Register,
13513 $src1$$Register,
13514 zr,
13515 Assembler::GT);
13516 %}
13517 ins_pipe(icond_reg);
13518 %}
13519
13520 // This pattern is automatically generated from aarch64_ad.m4
13521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13522 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13523 %{
13524 effect(DEF dst, USE src1, USE cr);
13525 ins_cost(INSN_COST * 2);
13526 format %{ "csincw $dst, $src1, zr le\t" %}
13527
13528 ins_encode %{
13529 __ csincw($dst$$Register,
13530 $src1$$Register,
13531 zr,
13532 Assembler::LE);
13533 %}
13534 ins_pipe(icond_reg);
13535 %}
13536
13537 // This pattern is automatically generated from aarch64_ad.m4
13538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13539 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13540 %{
13541 effect(DEF dst, USE src1, USE cr);
13542 ins_cost(INSN_COST * 2);
13543 format %{ "csincw $dst, $src1, zr gt\t" %}
13544
13545 ins_encode %{
13546 __ csincw($dst$$Register,
13547 $src1$$Register,
13548 zr,
13549 Assembler::GT);
13550 %}
13551 ins_pipe(icond_reg);
13552 %}
13553
13554 // This pattern is automatically generated from aarch64_ad.m4
13555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13556 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13557 %{
13558 effect(DEF dst, USE src1, USE cr);
13559 ins_cost(INSN_COST * 2);
13560 format %{ "csinvw $dst, $src1, zr lt\t" %}
13561
13562 ins_encode %{
13563 __ csinvw($dst$$Register,
13564 $src1$$Register,
13565 zr,
13566 Assembler::LT);
13567 %}
13568 ins_pipe(icond_reg);
13569 %}
13570
13571 // This pattern is automatically generated from aarch64_ad.m4
13572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13573 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13574 %{
13575 effect(DEF dst, USE src1, USE cr);
13576 ins_cost(INSN_COST * 2);
13577 format %{ "csinvw $dst, $src1, zr ge\t" %}
13578
13579 ins_encode %{
13580 __ csinvw($dst$$Register,
13581 $src1$$Register,
13582 zr,
13583 Assembler::GE);
13584 %}
13585 ins_pipe(icond_reg);
13586 %}
13587
13588 // This pattern is automatically generated from aarch64_ad.m4
13589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13590 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13591 %{
13592 match(Set dst (MinI src imm));
13593 ins_cost(INSN_COST * 3);
13594 expand %{
13595 rFlagsReg cr;
13596 compI_reg_imm0(cr, src);
13597 cmovI_reg_imm0_lt(dst, src, cr);
13598 %}
13599 %}
13600
13601 // This pattern is automatically generated from aarch64_ad.m4
13602 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13603 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13604 %{
13605 match(Set dst (MinI imm src));
13606 ins_cost(INSN_COST * 3);
13607 expand %{
13608 rFlagsReg cr;
13609 compI_reg_imm0(cr, src);
13610 cmovI_reg_imm0_lt(dst, src, cr);
13611 %}
13612 %}
13613
13614 // This pattern is automatically generated from aarch64_ad.m4
13615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13616 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13617 %{
13618 match(Set dst (MinI src imm));
13619 ins_cost(INSN_COST * 3);
13620 expand %{
13621 rFlagsReg cr;
13622 compI_reg_imm0(cr, src);
13623 cmovI_reg_imm1_le(dst, src, cr);
13624 %}
13625 %}
13626
13627 // This pattern is automatically generated from aarch64_ad.m4
13628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13629 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13630 %{
13631 match(Set dst (MinI imm src));
13632 ins_cost(INSN_COST * 3);
13633 expand %{
13634 rFlagsReg cr;
13635 compI_reg_imm0(cr, src);
13636 cmovI_reg_imm1_le(dst, src, cr);
13637 %}
13638 %}
13639
13640 // This pattern is automatically generated from aarch64_ad.m4
13641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13642 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13643 %{
13644 match(Set dst (MinI src imm));
13645 ins_cost(INSN_COST * 3);
13646 expand %{
13647 rFlagsReg cr;
13648 compI_reg_imm0(cr, src);
13649 cmovI_reg_immM1_lt(dst, src, cr);
13650 %}
13651 %}
13652
13653 // This pattern is automatically generated from aarch64_ad.m4
13654 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13655 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13656 %{
13657 match(Set dst (MinI imm src));
13658 ins_cost(INSN_COST * 3);
13659 expand %{
13660 rFlagsReg cr;
13661 compI_reg_imm0(cr, src);
13662 cmovI_reg_immM1_lt(dst, src, cr);
13663 %}
13664 %}
13665
13666 // This pattern is automatically generated from aarch64_ad.m4
13667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13668 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13669 %{
13670 match(Set dst (MaxI src imm));
13671 ins_cost(INSN_COST * 3);
13672 expand %{
13673 rFlagsReg cr;
13674 compI_reg_imm0(cr, src);
13675 cmovI_reg_imm0_gt(dst, src, cr);
13676 %}
13677 %}
13678
13679 // This pattern is automatically generated from aarch64_ad.m4
13680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13681 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13682 %{
13683 match(Set dst (MaxI imm src));
13684 ins_cost(INSN_COST * 3);
13685 expand %{
13686 rFlagsReg cr;
13687 compI_reg_imm0(cr, src);
13688 cmovI_reg_imm0_gt(dst, src, cr);
13689 %}
13690 %}
13691
13692 // This pattern is automatically generated from aarch64_ad.m4
13693 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13694 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13695 %{
13696 match(Set dst (MaxI src imm));
13697 ins_cost(INSN_COST * 3);
13698 expand %{
13699 rFlagsReg cr;
13700 compI_reg_imm0(cr, src);
13701 cmovI_reg_imm1_gt(dst, src, cr);
13702 %}
13703 %}
13704
13705 // This pattern is automatically generated from aarch64_ad.m4
13706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13707 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13708 %{
13709 match(Set dst (MaxI imm src));
13710 ins_cost(INSN_COST * 3);
13711 expand %{
13712 rFlagsReg cr;
13713 compI_reg_imm0(cr, src);
13714 cmovI_reg_imm1_gt(dst, src, cr);
13715 %}
13716 %}
13717
13718 // This pattern is automatically generated from aarch64_ad.m4
13719 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13720 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13721 %{
13722 match(Set dst (MaxI src imm));
13723 ins_cost(INSN_COST * 3);
13724 expand %{
13725 rFlagsReg cr;
13726 compI_reg_imm0(cr, src);
13727 cmovI_reg_immM1_ge(dst, src, cr);
13728 %}
13729 %}
13730
13731 // This pattern is automatically generated from aarch64_ad.m4
13732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13733 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13734 %{
13735 match(Set dst (MaxI imm src));
13736 ins_cost(INSN_COST * 3);
13737 expand %{
13738 rFlagsReg cr;
13739 compI_reg_imm0(cr, src);
13740 cmovI_reg_immM1_ge(dst, src, cr);
13741 %}
13742 %}
13743
13744 // This pattern is automatically generated from aarch64_ad.m4
13745 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13746 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13747 %{
13748 match(Set dst (ReverseI src));
13749 ins_cost(INSN_COST);
13750 format %{ "rbitw $dst, $src" %}
13751 ins_encode %{
13752 __ rbitw($dst$$Register, $src$$Register);
13753 %}
13754 ins_pipe(ialu_reg);
13755 %}
13756
13757 // This pattern is automatically generated from aarch64_ad.m4
13758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13759 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13760 %{
13761 match(Set dst (ReverseL src));
13762 ins_cost(INSN_COST);
13763 format %{ "rbit $dst, $src" %}
13764 ins_encode %{
13765 __ rbit($dst$$Register, $src$$Register);
13766 %}
13767 ins_pipe(ialu_reg);
13768 %}
13769
13770
13771 // END This section of the file is automatically generated. Do not edit --------------
13772
13773
13774 // ============================================================================
13775 // Floating Point Arithmetic Instructions
13776
13777 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13778 match(Set dst (AddHF src1 src2));
13779 format %{ "faddh $dst, $src1, $src2" %}
13780 ins_encode %{
13781 __ faddh($dst$$FloatRegister,
13782 $src1$$FloatRegister,
13783 $src2$$FloatRegister);
13784 %}
13785 ins_pipe(fp_dop_reg_reg_s);
13786 %}
13787
13788 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13789 match(Set dst (AddF src1 src2));
13790
13791 ins_cost(INSN_COST * 5);
13792 format %{ "fadds $dst, $src1, $src2" %}
13793
13794 ins_encode %{
13795 __ fadds(as_FloatRegister($dst$$reg),
13796 as_FloatRegister($src1$$reg),
13797 as_FloatRegister($src2$$reg));
13798 %}
13799
13800 ins_pipe(fp_dop_reg_reg_s);
13801 %}
13802
13803 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13804 match(Set dst (AddD src1 src2));
13805
13806 ins_cost(INSN_COST * 5);
13807 format %{ "faddd $dst, $src1, $src2" %}
13808
13809 ins_encode %{
13810 __ faddd(as_FloatRegister($dst$$reg),
13811 as_FloatRegister($src1$$reg),
13812 as_FloatRegister($src2$$reg));
13813 %}
13814
13815 ins_pipe(fp_dop_reg_reg_d);
13816 %}
13817
13818 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13819 match(Set dst (SubHF src1 src2));
13820 format %{ "fsubh $dst, $src1, $src2" %}
13821 ins_encode %{
13822 __ fsubh($dst$$FloatRegister,
13823 $src1$$FloatRegister,
13824 $src2$$FloatRegister);
13825 %}
13826 ins_pipe(fp_dop_reg_reg_s);
13827 %}
13828
13829 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13830 match(Set dst (SubF src1 src2));
13831
13832 ins_cost(INSN_COST * 5);
13833 format %{ "fsubs $dst, $src1, $src2" %}
13834
13835 ins_encode %{
13836 __ fsubs(as_FloatRegister($dst$$reg),
13837 as_FloatRegister($src1$$reg),
13838 as_FloatRegister($src2$$reg));
13839 %}
13840
13841 ins_pipe(fp_dop_reg_reg_s);
13842 %}
13843
13844 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13845 match(Set dst (SubD src1 src2));
13846
13847 ins_cost(INSN_COST * 5);
13848 format %{ "fsubd $dst, $src1, $src2" %}
13849
13850 ins_encode %{
13851 __ fsubd(as_FloatRegister($dst$$reg),
13852 as_FloatRegister($src1$$reg),
13853 as_FloatRegister($src2$$reg));
13854 %}
13855
13856 ins_pipe(fp_dop_reg_reg_d);
13857 %}
13858
13859 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13860 match(Set dst (MulHF src1 src2));
13861 format %{ "fmulh $dst, $src1, $src2" %}
13862 ins_encode %{
13863 __ fmulh($dst$$FloatRegister,
13864 $src1$$FloatRegister,
13865 $src2$$FloatRegister);
13866 %}
13867 ins_pipe(fp_dop_reg_reg_s);
13868 %}
13869
13870 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13871 match(Set dst (MulF src1 src2));
13872
13873 ins_cost(INSN_COST * 6);
13874 format %{ "fmuls $dst, $src1, $src2" %}
13875
13876 ins_encode %{
13877 __ fmuls(as_FloatRegister($dst$$reg),
13878 as_FloatRegister($src1$$reg),
13879 as_FloatRegister($src2$$reg));
13880 %}
13881
13882 ins_pipe(fp_dop_reg_reg_s);
13883 %}
13884
13885 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13886 match(Set dst (MulD src1 src2));
13887
13888 ins_cost(INSN_COST * 6);
13889 format %{ "fmuld $dst, $src1, $src2" %}
13890
13891 ins_encode %{
13892 __ fmuld(as_FloatRegister($dst$$reg),
13893 as_FloatRegister($src1$$reg),
13894 as_FloatRegister($src2$$reg));
13895 %}
13896
13897 ins_pipe(fp_dop_reg_reg_d);
13898 %}
13899
13900 // src1 * src2 + src3 (half-precision float)
13901 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13902 match(Set dst (FmaHF src3 (Binary src1 src2)));
13903 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13904 ins_encode %{
13905 assert(UseFMA, "Needs FMA instructions support.");
13906 __ fmaddh($dst$$FloatRegister,
13907 $src1$$FloatRegister,
13908 $src2$$FloatRegister,
13909 $src3$$FloatRegister);
13910 %}
13911 ins_pipe(pipe_class_default);
13912 %}
13913
13914 // src1 * src2 + src3
13915 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13916 match(Set dst (FmaF src3 (Binary src1 src2)));
13917
13918 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13919
13920 ins_encode %{
13921 assert(UseFMA, "Needs FMA instructions support.");
13922 __ fmadds(as_FloatRegister($dst$$reg),
13923 as_FloatRegister($src1$$reg),
13924 as_FloatRegister($src2$$reg),
13925 as_FloatRegister($src3$$reg));
13926 %}
13927
13928 ins_pipe(pipe_class_default);
13929 %}
13930
13931 // src1 * src2 + src3
13932 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13933 match(Set dst (FmaD src3 (Binary src1 src2)));
13934
13935 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13936
13937 ins_encode %{
13938 assert(UseFMA, "Needs FMA instructions support.");
13939 __ fmaddd(as_FloatRegister($dst$$reg),
13940 as_FloatRegister($src1$$reg),
13941 as_FloatRegister($src2$$reg),
13942 as_FloatRegister($src3$$reg));
13943 %}
13944
13945 ins_pipe(pipe_class_default);
13946 %}
13947
13948 // src1 * (-src2) + src3
13949 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13950 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13951 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13952
13953 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13954
13955 ins_encode %{
13956 assert(UseFMA, "Needs FMA instructions support.");
13957 __ fmsubs(as_FloatRegister($dst$$reg),
13958 as_FloatRegister($src1$$reg),
13959 as_FloatRegister($src2$$reg),
13960 as_FloatRegister($src3$$reg));
13961 %}
13962
13963 ins_pipe(pipe_class_default);
13964 %}
13965
13966 // src1 * (-src2) + src3
13967 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13968 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13969 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13970
13971 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13972
13973 ins_encode %{
13974 assert(UseFMA, "Needs FMA instructions support.");
13975 __ fmsubd(as_FloatRegister($dst$$reg),
13976 as_FloatRegister($src1$$reg),
13977 as_FloatRegister($src2$$reg),
13978 as_FloatRegister($src3$$reg));
13979 %}
13980
13981 ins_pipe(pipe_class_default);
13982 %}
13983
13984 // src1 * (-src2) - src3
13985 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13986 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13987 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13988
13989 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13990
13991 ins_encode %{
13992 assert(UseFMA, "Needs FMA instructions support.");
13993 __ fnmadds(as_FloatRegister($dst$$reg),
13994 as_FloatRegister($src1$$reg),
13995 as_FloatRegister($src2$$reg),
13996 as_FloatRegister($src3$$reg));
13997 %}
13998
13999 ins_pipe(pipe_class_default);
14000 %}
14001
14002 // src1 * (-src2) - src3
14003 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14004 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14005 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14006
14007 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14008
14009 ins_encode %{
14010 assert(UseFMA, "Needs FMA instructions support.");
14011 __ fnmaddd(as_FloatRegister($dst$$reg),
14012 as_FloatRegister($src1$$reg),
14013 as_FloatRegister($src2$$reg),
14014 as_FloatRegister($src3$$reg));
14015 %}
14016
14017 ins_pipe(pipe_class_default);
14018 %}
14019
14020 // src1 * src2 - src3
14021 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14022 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14023
14024 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14025
14026 ins_encode %{
14027 assert(UseFMA, "Needs FMA instructions support.");
14028 __ fnmsubs(as_FloatRegister($dst$$reg),
14029 as_FloatRegister($src1$$reg),
14030 as_FloatRegister($src2$$reg),
14031 as_FloatRegister($src3$$reg));
14032 %}
14033
14034 ins_pipe(pipe_class_default);
14035 %}
14036
14037 // src1 * src2 - src3
14038 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14039 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14040
14041 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14042
14043 ins_encode %{
14044 assert(UseFMA, "Needs FMA instructions support.");
14045 // n.b. insn name should be fnmsubd
14046 __ fnmsub(as_FloatRegister($dst$$reg),
14047 as_FloatRegister($src1$$reg),
14048 as_FloatRegister($src2$$reg),
14049 as_FloatRegister($src3$$reg));
14050 %}
14051
14052 ins_pipe(pipe_class_default);
14053 %}
14054
14055 // Math.max(HH)H (half-precision float)
14056 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14057 match(Set dst (MaxHF src1 src2));
14058 format %{ "fmaxh $dst, $src1, $src2" %}
14059 ins_encode %{
14060 __ fmaxh($dst$$FloatRegister,
14061 $src1$$FloatRegister,
14062 $src2$$FloatRegister);
14063 %}
14064 ins_pipe(fp_dop_reg_reg_s);
14065 %}
14066
14067 // Math.min(HH)H (half-precision float)
14068 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14069 match(Set dst (MinHF src1 src2));
14070 format %{ "fminh $dst, $src1, $src2" %}
14071 ins_encode %{
14072 __ fminh($dst$$FloatRegister,
14073 $src1$$FloatRegister,
14074 $src2$$FloatRegister);
14075 %}
14076 ins_pipe(fp_dop_reg_reg_s);
14077 %}
14078
14079 // Math.max(FF)F
14080 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14081 match(Set dst (MaxF src1 src2));
14082
14083 format %{ "fmaxs $dst, $src1, $src2" %}
14084 ins_encode %{
14085 __ fmaxs(as_FloatRegister($dst$$reg),
14086 as_FloatRegister($src1$$reg),
14087 as_FloatRegister($src2$$reg));
14088 %}
14089
14090 ins_pipe(fp_dop_reg_reg_s);
14091 %}
14092
14093 // Math.min(FF)F
14094 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14095 match(Set dst (MinF src1 src2));
14096
14097 format %{ "fmins $dst, $src1, $src2" %}
14098 ins_encode %{
14099 __ fmins(as_FloatRegister($dst$$reg),
14100 as_FloatRegister($src1$$reg),
14101 as_FloatRegister($src2$$reg));
14102 %}
14103
14104 ins_pipe(fp_dop_reg_reg_s);
14105 %}
14106
14107 // Math.max(DD)D
14108 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14109 match(Set dst (MaxD src1 src2));
14110
14111 format %{ "fmaxd $dst, $src1, $src2" %}
14112 ins_encode %{
14113 __ fmaxd(as_FloatRegister($dst$$reg),
14114 as_FloatRegister($src1$$reg),
14115 as_FloatRegister($src2$$reg));
14116 %}
14117
14118 ins_pipe(fp_dop_reg_reg_d);
14119 %}
14120
14121 // Math.min(DD)D
14122 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14123 match(Set dst (MinD src1 src2));
14124
14125 format %{ "fmind $dst, $src1, $src2" %}
14126 ins_encode %{
14127 __ fmind(as_FloatRegister($dst$$reg),
14128 as_FloatRegister($src1$$reg),
14129 as_FloatRegister($src2$$reg));
14130 %}
14131
14132 ins_pipe(fp_dop_reg_reg_d);
14133 %}
14134
14135 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14136 match(Set dst (DivHF src1 src2));
14137 format %{ "fdivh $dst, $src1, $src2" %}
14138 ins_encode %{
14139 __ fdivh($dst$$FloatRegister,
14140 $src1$$FloatRegister,
14141 $src2$$FloatRegister);
14142 %}
14143 ins_pipe(fp_div_s);
14144 %}
14145
14146 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14147 match(Set dst (DivF src1 src2));
14148
14149 ins_cost(INSN_COST * 18);
14150 format %{ "fdivs $dst, $src1, $src2" %}
14151
14152 ins_encode %{
14153 __ fdivs(as_FloatRegister($dst$$reg),
14154 as_FloatRegister($src1$$reg),
14155 as_FloatRegister($src2$$reg));
14156 %}
14157
14158 ins_pipe(fp_div_s);
14159 %}
14160
14161 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14162 match(Set dst (DivD src1 src2));
14163
14164 ins_cost(INSN_COST * 32);
14165 format %{ "fdivd $dst, $src1, $src2" %}
14166
14167 ins_encode %{
14168 __ fdivd(as_FloatRegister($dst$$reg),
14169 as_FloatRegister($src1$$reg),
14170 as_FloatRegister($src2$$reg));
14171 %}
14172
14173 ins_pipe(fp_div_d);
14174 %}
14175
14176 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14177 match(Set dst (NegF src));
14178
14179 ins_cost(INSN_COST * 3);
14180 format %{ "fneg $dst, $src" %}
14181
14182 ins_encode %{
14183 __ fnegs(as_FloatRegister($dst$$reg),
14184 as_FloatRegister($src$$reg));
14185 %}
14186
14187 ins_pipe(fp_uop_s);
14188 %}
14189
14190 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14191 match(Set dst (NegD src));
14192
14193 ins_cost(INSN_COST * 3);
14194 format %{ "fnegd $dst, $src" %}
14195
14196 ins_encode %{
14197 __ fnegd(as_FloatRegister($dst$$reg),
14198 as_FloatRegister($src$$reg));
14199 %}
14200
14201 ins_pipe(fp_uop_d);
14202 %}
14203
14204 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14205 %{
14206 match(Set dst (AbsI src));
14207
14208 effect(KILL cr);
14209 ins_cost(INSN_COST * 2);
14210 format %{ "cmpw $src, zr\n\t"
14211 "cnegw $dst, $src, Assembler::LT\t# int abs"
14212 %}
14213
14214 ins_encode %{
14215 __ cmpw(as_Register($src$$reg), zr);
14216 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14217 %}
14218 ins_pipe(pipe_class_default);
14219 %}
14220
14221 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14222 %{
14223 match(Set dst (AbsL src));
14224
14225 effect(KILL cr);
14226 ins_cost(INSN_COST * 2);
14227 format %{ "cmp $src, zr\n\t"
14228 "cneg $dst, $src, Assembler::LT\t# long abs"
14229 %}
14230
14231 ins_encode %{
14232 __ cmp(as_Register($src$$reg), zr);
14233 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14234 %}
14235 ins_pipe(pipe_class_default);
14236 %}
14237
14238 instruct absF_reg(vRegF dst, vRegF src) %{
14239 match(Set dst (AbsF src));
14240
14241 ins_cost(INSN_COST * 3);
14242 format %{ "fabss $dst, $src" %}
14243 ins_encode %{
14244 __ fabss(as_FloatRegister($dst$$reg),
14245 as_FloatRegister($src$$reg));
14246 %}
14247
14248 ins_pipe(fp_uop_s);
14249 %}
14250
14251 instruct absD_reg(vRegD dst, vRegD src) %{
14252 match(Set dst (AbsD src));
14253
14254 ins_cost(INSN_COST * 3);
14255 format %{ "fabsd $dst, $src" %}
14256 ins_encode %{
14257 __ fabsd(as_FloatRegister($dst$$reg),
14258 as_FloatRegister($src$$reg));
14259 %}
14260
14261 ins_pipe(fp_uop_d);
14262 %}
14263
14264 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14265 match(Set dst (AbsF (SubF src1 src2)));
14266
14267 ins_cost(INSN_COST * 3);
14268 format %{ "fabds $dst, $src1, $src2" %}
14269 ins_encode %{
14270 __ fabds(as_FloatRegister($dst$$reg),
14271 as_FloatRegister($src1$$reg),
14272 as_FloatRegister($src2$$reg));
14273 %}
14274
14275 ins_pipe(fp_uop_s);
14276 %}
14277
14278 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14279 match(Set dst (AbsD (SubD src1 src2)));
14280
14281 ins_cost(INSN_COST * 3);
14282 format %{ "fabdd $dst, $src1, $src2" %}
14283 ins_encode %{
14284 __ fabdd(as_FloatRegister($dst$$reg),
14285 as_FloatRegister($src1$$reg),
14286 as_FloatRegister($src2$$reg));
14287 %}
14288
14289 ins_pipe(fp_uop_d);
14290 %}
14291
14292 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14293 match(Set dst (SqrtD src));
14294
14295 ins_cost(INSN_COST * 50);
14296 format %{ "fsqrtd $dst, $src" %}
14297 ins_encode %{
14298 __ fsqrtd(as_FloatRegister($dst$$reg),
14299 as_FloatRegister($src$$reg));
14300 %}
14301
14302 ins_pipe(fp_div_s);
14303 %}
14304
14305 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14306 match(Set dst (SqrtF src));
14307
14308 ins_cost(INSN_COST * 50);
14309 format %{ "fsqrts $dst, $src" %}
14310 ins_encode %{
14311 __ fsqrts(as_FloatRegister($dst$$reg),
14312 as_FloatRegister($src$$reg));
14313 %}
14314
14315 ins_pipe(fp_div_d);
14316 %}
14317
14318 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14319 match(Set dst (SqrtHF src));
14320 format %{ "fsqrth $dst, $src" %}
14321 ins_encode %{
14322 __ fsqrth($dst$$FloatRegister,
14323 $src$$FloatRegister);
14324 %}
14325 ins_pipe(fp_div_s);
14326 %}
14327
14328 // Math.rint, floor, ceil
14329 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14330 match(Set dst (RoundDoubleMode src rmode));
14331 format %{ "frint $dst, $src, $rmode" %}
14332 ins_encode %{
14333 switch ($rmode$$constant) {
14334 case RoundDoubleModeNode::rmode_rint:
14335 __ frintnd(as_FloatRegister($dst$$reg),
14336 as_FloatRegister($src$$reg));
14337 break;
14338 case RoundDoubleModeNode::rmode_floor:
14339 __ frintmd(as_FloatRegister($dst$$reg),
14340 as_FloatRegister($src$$reg));
14341 break;
14342 case RoundDoubleModeNode::rmode_ceil:
14343 __ frintpd(as_FloatRegister($dst$$reg),
14344 as_FloatRegister($src$$reg));
14345 break;
14346 }
14347 %}
14348 ins_pipe(fp_uop_d);
14349 %}
14350
14351 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14352 match(Set dst (CopySignD src1 (Binary src2 zero)));
14353 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14354 format %{ "CopySignD $dst $src1 $src2" %}
14355 ins_encode %{
14356 FloatRegister dst = as_FloatRegister($dst$$reg),
14357 src1 = as_FloatRegister($src1$$reg),
14358 src2 = as_FloatRegister($src2$$reg),
14359 zero = as_FloatRegister($zero$$reg);
14360 __ fnegd(dst, zero);
14361 __ bsl(dst, __ T8B, src2, src1);
14362 %}
14363 ins_pipe(fp_uop_d);
14364 %}
14365
14366 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14367 match(Set dst (CopySignF src1 src2));
14368 effect(TEMP_DEF dst, USE src1, USE src2);
14369 format %{ "CopySignF $dst $src1 $src2" %}
14370 ins_encode %{
14371 FloatRegister dst = as_FloatRegister($dst$$reg),
14372 src1 = as_FloatRegister($src1$$reg),
14373 src2 = as_FloatRegister($src2$$reg);
14374 __ movi(dst, __ T2S, 0x80, 24);
14375 __ bsl(dst, __ T8B, src2, src1);
14376 %}
14377 ins_pipe(fp_uop_d);
14378 %}
14379
14380 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14381 match(Set dst (SignumD src (Binary zero one)));
14382 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14383 format %{ "signumD $dst, $src" %}
14384 ins_encode %{
14385 FloatRegister src = as_FloatRegister($src$$reg),
14386 dst = as_FloatRegister($dst$$reg),
14387 zero = as_FloatRegister($zero$$reg),
14388 one = as_FloatRegister($one$$reg);
14389 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14390 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14391 // Bit selection instruction gets bit from "one" for each enabled bit in
14392 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14393 // NaN the whole "src" will be copied because "dst" is zero. For all other
14394 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14395 // from "src", and all other bits are copied from 1.0.
14396 __ bsl(dst, __ T8B, one, src);
14397 %}
14398 ins_pipe(fp_uop_d);
14399 %}
14400
14401 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14402 match(Set dst (SignumF src (Binary zero one)));
14403 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14404 format %{ "signumF $dst, $src" %}
14405 ins_encode %{
14406 FloatRegister src = as_FloatRegister($src$$reg),
14407 dst = as_FloatRegister($dst$$reg),
14408 zero = as_FloatRegister($zero$$reg),
14409 one = as_FloatRegister($one$$reg);
14410 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14411 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14412 // Bit selection instruction gets bit from "one" for each enabled bit in
14413 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14414 // NaN the whole "src" will be copied because "dst" is zero. For all other
14415 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14416 // from "src", and all other bits are copied from 1.0.
14417 __ bsl(dst, __ T8B, one, src);
14418 %}
14419 ins_pipe(fp_uop_d);
14420 %}
14421
14422 instruct onspinwait() %{
14423 match(OnSpinWait);
14424 ins_cost(INSN_COST);
14425
14426 format %{ "onspinwait" %}
14427
14428 ins_encode %{
14429 __ spin_wait();
14430 %}
14431 ins_pipe(pipe_class_empty);
14432 %}
14433
14434 // ============================================================================
14435 // Logical Instructions
14436
14437 // Integer Logical Instructions
14438
14439 // And Instructions
14440
14441
14442 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14443 match(Set dst (AndI src1 src2));
14444
14445 format %{ "andw $dst, $src1, $src2\t# int" %}
14446
14447 ins_cost(INSN_COST);
14448 ins_encode %{
14449 __ andw(as_Register($dst$$reg),
14450 as_Register($src1$$reg),
14451 as_Register($src2$$reg));
14452 %}
14453
14454 ins_pipe(ialu_reg_reg);
14455 %}
14456
14457 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14458 match(Set dst (AndI src1 src2));
14459
14460 format %{ "andsw $dst, $src1, $src2\t# int" %}
14461
14462 ins_cost(INSN_COST);
14463 ins_encode %{
14464 __ andw(as_Register($dst$$reg),
14465 as_Register($src1$$reg),
14466 (uint64_t)($src2$$constant));
14467 %}
14468
14469 ins_pipe(ialu_reg_imm);
14470 %}
14471
14472 // Or Instructions
14473
14474 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14475 match(Set dst (OrI src1 src2));
14476
14477 format %{ "orrw $dst, $src1, $src2\t# int" %}
14478
14479 ins_cost(INSN_COST);
14480 ins_encode %{
14481 __ orrw(as_Register($dst$$reg),
14482 as_Register($src1$$reg),
14483 as_Register($src2$$reg));
14484 %}
14485
14486 ins_pipe(ialu_reg_reg);
14487 %}
14488
14489 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14490 match(Set dst (OrI src1 src2));
14491
14492 format %{ "orrw $dst, $src1, $src2\t# int" %}
14493
14494 ins_cost(INSN_COST);
14495 ins_encode %{
14496 __ orrw(as_Register($dst$$reg),
14497 as_Register($src1$$reg),
14498 (uint64_t)($src2$$constant));
14499 %}
14500
14501 ins_pipe(ialu_reg_imm);
14502 %}
14503
14504 // Xor Instructions
14505
14506 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14507 match(Set dst (XorI src1 src2));
14508
14509 format %{ "eorw $dst, $src1, $src2\t# int" %}
14510
14511 ins_cost(INSN_COST);
14512 ins_encode %{
14513 __ eorw(as_Register($dst$$reg),
14514 as_Register($src1$$reg),
14515 as_Register($src2$$reg));
14516 %}
14517
14518 ins_pipe(ialu_reg_reg);
14519 %}
14520
14521 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14522 match(Set dst (XorI src1 src2));
14523
14524 format %{ "eorw $dst, $src1, $src2\t# int" %}
14525
14526 ins_cost(INSN_COST);
14527 ins_encode %{
14528 __ eorw(as_Register($dst$$reg),
14529 as_Register($src1$$reg),
14530 (uint64_t)($src2$$constant));
14531 %}
14532
14533 ins_pipe(ialu_reg_imm);
14534 %}
14535
14536 // Long Logical Instructions
14537 // TODO
14538
14539 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14540 match(Set dst (AndL src1 src2));
14541
14542 format %{ "and $dst, $src1, $src2\t# int" %}
14543
14544 ins_cost(INSN_COST);
14545 ins_encode %{
14546 __ andr(as_Register($dst$$reg),
14547 as_Register($src1$$reg),
14548 as_Register($src2$$reg));
14549 %}
14550
14551 ins_pipe(ialu_reg_reg);
14552 %}
14553
14554 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14555 match(Set dst (AndL src1 src2));
14556
14557 format %{ "and $dst, $src1, $src2\t# int" %}
14558
14559 ins_cost(INSN_COST);
14560 ins_encode %{
14561 __ andr(as_Register($dst$$reg),
14562 as_Register($src1$$reg),
14563 (uint64_t)($src2$$constant));
14564 %}
14565
14566 ins_pipe(ialu_reg_imm);
14567 %}
14568
14569 // Or Instructions
14570
14571 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14572 match(Set dst (OrL src1 src2));
14573
14574 format %{ "orr $dst, $src1, $src2\t# int" %}
14575
14576 ins_cost(INSN_COST);
14577 ins_encode %{
14578 __ orr(as_Register($dst$$reg),
14579 as_Register($src1$$reg),
14580 as_Register($src2$$reg));
14581 %}
14582
14583 ins_pipe(ialu_reg_reg);
14584 %}
14585
14586 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14587 match(Set dst (OrL src1 src2));
14588
14589 format %{ "orr $dst, $src1, $src2\t# int" %}
14590
14591 ins_cost(INSN_COST);
14592 ins_encode %{
14593 __ orr(as_Register($dst$$reg),
14594 as_Register($src1$$reg),
14595 (uint64_t)($src2$$constant));
14596 %}
14597
14598 ins_pipe(ialu_reg_imm);
14599 %}
14600
14601 // Xor Instructions
14602
14603 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14604 match(Set dst (XorL src1 src2));
14605
14606 format %{ "eor $dst, $src1, $src2\t# int" %}
14607
14608 ins_cost(INSN_COST);
14609 ins_encode %{
14610 __ eor(as_Register($dst$$reg),
14611 as_Register($src1$$reg),
14612 as_Register($src2$$reg));
14613 %}
14614
14615 ins_pipe(ialu_reg_reg);
14616 %}
14617
14618 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14619 match(Set dst (XorL src1 src2));
14620
14621 ins_cost(INSN_COST);
14622 format %{ "eor $dst, $src1, $src2\t# int" %}
14623
14624 ins_encode %{
14625 __ eor(as_Register($dst$$reg),
14626 as_Register($src1$$reg),
14627 (uint64_t)($src2$$constant));
14628 %}
14629
14630 ins_pipe(ialu_reg_imm);
14631 %}
14632
14633 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14634 %{
14635 match(Set dst (ConvI2L src));
14636
14637 ins_cost(INSN_COST);
14638 format %{ "sxtw $dst, $src\t# i2l" %}
14639 ins_encode %{
14640 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14641 %}
14642 ins_pipe(ialu_reg_shift);
14643 %}
14644
14645 // this pattern occurs in bigmath arithmetic
14646 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14647 %{
14648 match(Set dst (AndL (ConvI2L src) mask));
14649
14650 ins_cost(INSN_COST);
14651 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14652 ins_encode %{
14653 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14654 %}
14655
14656 ins_pipe(ialu_reg_shift);
14657 %}
14658
14659 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14660 match(Set dst (ConvL2I src));
14661
14662 ins_cost(INSN_COST);
14663 format %{ "movw $dst, $src \t// l2i" %}
14664
14665 ins_encode %{
14666 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14667 %}
14668
14669 ins_pipe(ialu_reg);
14670 %}
14671
14672 instruct convD2F_reg(vRegF dst, vRegD src) %{
14673 match(Set dst (ConvD2F src));
14674
14675 ins_cost(INSN_COST * 5);
14676 format %{ "fcvtd $dst, $src \t// d2f" %}
14677
14678 ins_encode %{
14679 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14680 %}
14681
14682 ins_pipe(fp_d2f);
14683 %}
14684
14685 instruct convF2D_reg(vRegD dst, vRegF src) %{
14686 match(Set dst (ConvF2D src));
14687
14688 ins_cost(INSN_COST * 5);
14689 format %{ "fcvts $dst, $src \t// f2d" %}
14690
14691 ins_encode %{
14692 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14693 %}
14694
14695 ins_pipe(fp_f2d);
14696 %}
14697
14698 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14699 match(Set dst (ConvF2I src));
14700
14701 ins_cost(INSN_COST * 5);
14702 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14703
14704 ins_encode %{
14705 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14706 %}
14707
14708 ins_pipe(fp_f2i);
14709 %}
14710
14711 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14712 match(Set dst (ConvF2L src));
14713
14714 ins_cost(INSN_COST * 5);
14715 format %{ "fcvtzs $dst, $src \t// f2l" %}
14716
14717 ins_encode %{
14718 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14719 %}
14720
14721 ins_pipe(fp_f2l);
14722 %}
14723
14724 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14725 match(Set dst (ConvF2HF src));
14726 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14727 "smov $dst, $tmp\t# move result from $tmp to $dst"
14728 %}
14729 effect(TEMP tmp);
14730 ins_encode %{
14731 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14732 %}
14733 ins_pipe(pipe_slow);
14734 %}
14735
14736 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14737 match(Set dst (ConvHF2F src));
14738 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14739 "fcvt $dst, $tmp\t# convert half to single precision"
14740 %}
14741 effect(TEMP tmp);
14742 ins_encode %{
14743 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14744 %}
14745 ins_pipe(pipe_slow);
14746 %}
14747
14748 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14749 match(Set dst (ConvI2F src));
14750
14751 ins_cost(INSN_COST * 5);
14752 format %{ "scvtfws $dst, $src \t// i2f" %}
14753
14754 ins_encode %{
14755 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14756 %}
14757
14758 ins_pipe(fp_i2f);
14759 %}
14760
14761 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14762 match(Set dst (ConvL2F src));
14763
14764 ins_cost(INSN_COST * 5);
14765 format %{ "scvtfs $dst, $src \t// l2f" %}
14766
14767 ins_encode %{
14768 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14769 %}
14770
14771 ins_pipe(fp_l2f);
14772 %}
14773
14774 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14775 match(Set dst (ConvD2I src));
14776
14777 ins_cost(INSN_COST * 5);
14778 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14779
14780 ins_encode %{
14781 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14782 %}
14783
14784 ins_pipe(fp_d2i);
14785 %}
14786
14787 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14788 match(Set dst (ConvD2L src));
14789
14790 ins_cost(INSN_COST * 5);
14791 format %{ "fcvtzd $dst, $src \t// d2l" %}
14792
14793 ins_encode %{
14794 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14795 %}
14796
14797 ins_pipe(fp_d2l);
14798 %}
14799
14800 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14801 match(Set dst (ConvI2D src));
14802
14803 ins_cost(INSN_COST * 5);
14804 format %{ "scvtfwd $dst, $src \t// i2d" %}
14805
14806 ins_encode %{
14807 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14808 %}
14809
14810 ins_pipe(fp_i2d);
14811 %}
14812
14813 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14814 match(Set dst (ConvL2D src));
14815
14816 ins_cost(INSN_COST * 5);
14817 format %{ "scvtfd $dst, $src \t// l2d" %}
14818
14819 ins_encode %{
14820 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14821 %}
14822
14823 ins_pipe(fp_l2d);
14824 %}
14825
14826 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14827 %{
14828 match(Set dst (RoundD src));
14829 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14830 format %{ "java_round_double $dst,$src"%}
14831 ins_encode %{
14832 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14833 as_FloatRegister($ftmp$$reg));
14834 %}
14835 ins_pipe(pipe_slow);
14836 %}
14837
14838 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14839 %{
14840 match(Set dst (RoundF src));
14841 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14842 format %{ "java_round_float $dst,$src"%}
14843 ins_encode %{
14844 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14845 as_FloatRegister($ftmp$$reg));
14846 %}
14847 ins_pipe(pipe_slow);
14848 %}
14849
14850 // stack <-> reg and reg <-> reg shuffles with no conversion
14851
14852 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14853
14854 match(Set dst (MoveF2I src));
14855
14856 effect(DEF dst, USE src);
14857
14858 ins_cost(4 * INSN_COST);
14859
14860 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14861
14862 ins_encode %{
14863 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14864 %}
14865
14866 ins_pipe(iload_reg_reg);
14867
14868 %}
14869
14870 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14871
14872 match(Set dst (MoveI2F src));
14873
14874 effect(DEF dst, USE src);
14875
14876 ins_cost(4 * INSN_COST);
14877
14878 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14879
14880 ins_encode %{
14881 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14882 %}
14883
14884 ins_pipe(pipe_class_memory);
14885
14886 %}
14887
14888 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14889
14890 match(Set dst (MoveD2L src));
14891
14892 effect(DEF dst, USE src);
14893
14894 ins_cost(4 * INSN_COST);
14895
14896 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14897
14898 ins_encode %{
14899 __ ldr($dst$$Register, Address(sp, $src$$disp));
14900 %}
14901
14902 ins_pipe(iload_reg_reg);
14903
14904 %}
14905
14906 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14907
14908 match(Set dst (MoveL2D src));
14909
14910 effect(DEF dst, USE src);
14911
14912 ins_cost(4 * INSN_COST);
14913
14914 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14915
14916 ins_encode %{
14917 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14918 %}
14919
14920 ins_pipe(pipe_class_memory);
14921
14922 %}
14923
14924 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14925
14926 match(Set dst (MoveF2I src));
14927
14928 effect(DEF dst, USE src);
14929
14930 ins_cost(INSN_COST);
14931
14932 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14933
14934 ins_encode %{
14935 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14936 %}
14937
14938 ins_pipe(pipe_class_memory);
14939
14940 %}
14941
14942 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14943
14944 match(Set dst (MoveI2F src));
14945
14946 effect(DEF dst, USE src);
14947
14948 ins_cost(INSN_COST);
14949
14950 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14951
14952 ins_encode %{
14953 __ strw($src$$Register, Address(sp, $dst$$disp));
14954 %}
14955
14956 ins_pipe(istore_reg_reg);
14957
14958 %}
14959
14960 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14961
14962 match(Set dst (MoveD2L src));
14963
14964 effect(DEF dst, USE src);
14965
14966 ins_cost(INSN_COST);
14967
14968 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14969
14970 ins_encode %{
14971 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14972 %}
14973
14974 ins_pipe(pipe_class_memory);
14975
14976 %}
14977
14978 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14979
14980 match(Set dst (MoveL2D src));
14981
14982 effect(DEF dst, USE src);
14983
14984 ins_cost(INSN_COST);
14985
14986 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14987
14988 ins_encode %{
14989 __ str($src$$Register, Address(sp, $dst$$disp));
14990 %}
14991
14992 ins_pipe(istore_reg_reg);
14993
14994 %}
14995
14996 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14997
14998 match(Set dst (MoveF2I src));
14999
15000 effect(DEF dst, USE src);
15001
15002 ins_cost(INSN_COST);
15003
15004 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15005
15006 ins_encode %{
15007 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15008 %}
15009
15010 ins_pipe(fp_f2i);
15011
15012 %}
15013
15014 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15015
15016 match(Set dst (MoveI2F src));
15017
15018 effect(DEF dst, USE src);
15019
15020 ins_cost(INSN_COST);
15021
15022 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15023
15024 ins_encode %{
15025 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15026 %}
15027
15028 ins_pipe(fp_i2f);
15029
15030 %}
15031
15032 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15033
15034 match(Set dst (MoveD2L src));
15035
15036 effect(DEF dst, USE src);
15037
15038 ins_cost(INSN_COST);
15039
15040 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15041
15042 ins_encode %{
15043 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15044 %}
15045
15046 ins_pipe(fp_d2l);
15047
15048 %}
15049
15050 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15051
15052 match(Set dst (MoveL2D src));
15053
15054 effect(DEF dst, USE src);
15055
15056 ins_cost(INSN_COST);
15057
15058 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15059
15060 ins_encode %{
15061 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15062 %}
15063
15064 ins_pipe(fp_l2d);
15065
15066 %}
15067
15068 // ============================================================================
15069 // clearing of an array
15070
15071 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15072 %{
15073 match(Set dummy (ClearArray cnt base));
15074 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15075
15076 ins_cost(4 * INSN_COST);
15077 format %{ "ClearArray $cnt, $base" %}
15078
15079 ins_encode %{
15080 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15081 if (tpc == nullptr) {
15082 ciEnv::current()->record_failure("CodeCache is full");
15083 return;
15084 }
15085 %}
15086
15087 ins_pipe(pipe_class_memory);
15088 %}
15089
15090 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15091 %{
15092 predicate((uint64_t)n->in(2)->get_long()
15093 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15094 match(Set dummy (ClearArray cnt base));
15095 effect(TEMP temp, USE_KILL base, KILL cr);
15096
15097 ins_cost(4 * INSN_COST);
15098 format %{ "ClearArray $cnt, $base" %}
15099
15100 ins_encode %{
15101 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15102 if (tpc == nullptr) {
15103 ciEnv::current()->record_failure("CodeCache is full");
15104 return;
15105 }
15106 %}
15107
15108 ins_pipe(pipe_class_memory);
15109 %}
15110
15111 // ============================================================================
15112 // Overflow Math Instructions
15113
15114 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15115 %{
15116 match(Set cr (OverflowAddI op1 op2));
15117
15118 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15119 ins_cost(INSN_COST);
15120 ins_encode %{
15121 __ cmnw($op1$$Register, $op2$$Register);
15122 %}
15123
15124 ins_pipe(icmp_reg_reg);
15125 %}
15126
15127 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15128 %{
15129 match(Set cr (OverflowAddI op1 op2));
15130
15131 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15132 ins_cost(INSN_COST);
15133 ins_encode %{
15134 __ cmnw($op1$$Register, $op2$$constant);
15135 %}
15136
15137 ins_pipe(icmp_reg_imm);
15138 %}
15139
15140 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15141 %{
15142 match(Set cr (OverflowAddL op1 op2));
15143
15144 format %{ "cmn $op1, $op2\t# overflow check long" %}
15145 ins_cost(INSN_COST);
15146 ins_encode %{
15147 __ cmn($op1$$Register, $op2$$Register);
15148 %}
15149
15150 ins_pipe(icmp_reg_reg);
15151 %}
15152
15153 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15154 %{
15155 match(Set cr (OverflowAddL op1 op2));
15156
15157 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15158 ins_cost(INSN_COST);
15159 ins_encode %{
15160 __ adds(zr, $op1$$Register, $op2$$constant);
15161 %}
15162
15163 ins_pipe(icmp_reg_imm);
15164 %}
15165
15166 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15167 %{
15168 match(Set cr (OverflowSubI op1 op2));
15169
15170 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15171 ins_cost(INSN_COST);
15172 ins_encode %{
15173 __ cmpw($op1$$Register, $op2$$Register);
15174 %}
15175
15176 ins_pipe(icmp_reg_reg);
15177 %}
15178
15179 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15180 %{
15181 match(Set cr (OverflowSubI op1 op2));
15182
15183 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15184 ins_cost(INSN_COST);
15185 ins_encode %{
15186 __ cmpw($op1$$Register, $op2$$constant);
15187 %}
15188
15189 ins_pipe(icmp_reg_imm);
15190 %}
15191
15192 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15193 %{
15194 match(Set cr (OverflowSubL op1 op2));
15195
15196 format %{ "cmp $op1, $op2\t# overflow check long" %}
15197 ins_cost(INSN_COST);
15198 ins_encode %{
15199 __ cmp($op1$$Register, $op2$$Register);
15200 %}
15201
15202 ins_pipe(icmp_reg_reg);
15203 %}
15204
15205 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15206 %{
15207 match(Set cr (OverflowSubL op1 op2));
15208
15209 format %{ "cmp $op1, $op2\t# overflow check long" %}
15210 ins_cost(INSN_COST);
15211 ins_encode %{
15212 __ subs(zr, $op1$$Register, $op2$$constant);
15213 %}
15214
15215 ins_pipe(icmp_reg_imm);
15216 %}
15217
15218 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15219 %{
15220 match(Set cr (OverflowSubI zero op1));
15221
15222 format %{ "cmpw zr, $op1\t# overflow check int" %}
15223 ins_cost(INSN_COST);
15224 ins_encode %{
15225 __ cmpw(zr, $op1$$Register);
15226 %}
15227
15228 ins_pipe(icmp_reg_imm);
15229 %}
15230
15231 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15232 %{
15233 match(Set cr (OverflowSubL zero op1));
15234
15235 format %{ "cmp zr, $op1\t# overflow check long" %}
15236 ins_cost(INSN_COST);
15237 ins_encode %{
15238 __ cmp(zr, $op1$$Register);
15239 %}
15240
15241 ins_pipe(icmp_reg_imm);
15242 %}
15243
15244 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15245 %{
15246 match(Set cr (OverflowMulI op1 op2));
15247
15248 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15249 "cmp rscratch1, rscratch1, sxtw\n\t"
15250 "movw rscratch1, #0x80000000\n\t"
15251 "cselw rscratch1, rscratch1, zr, NE\n\t"
15252 "cmpw rscratch1, #1" %}
15253 ins_cost(5 * INSN_COST);
15254 ins_encode %{
15255 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15256 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15257 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15258 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15259 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15260 %}
15261
15262 ins_pipe(pipe_slow);
15263 %}
15264
15265 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15266 %{
15267 match(If cmp (OverflowMulI op1 op2));
15268 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15269 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15270 effect(USE labl, KILL cr);
15271
15272 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15273 "cmp rscratch1, rscratch1, sxtw\n\t"
15274 "b$cmp $labl" %}
15275 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15276 ins_encode %{
15277 Label* L = $labl$$label;
15278 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15279 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15280 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15281 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15282 %}
15283
15284 ins_pipe(pipe_serial);
15285 %}
15286
15287 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15288 %{
15289 match(Set cr (OverflowMulL op1 op2));
15290
15291 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15292 "smulh rscratch2, $op1, $op2\n\t"
15293 "cmp rscratch2, rscratch1, ASR #63\n\t"
15294 "movw rscratch1, #0x80000000\n\t"
15295 "cselw rscratch1, rscratch1, zr, NE\n\t"
15296 "cmpw rscratch1, #1" %}
15297 ins_cost(6 * INSN_COST);
15298 ins_encode %{
15299 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15300 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15301 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15302 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15303 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15304 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15305 %}
15306
15307 ins_pipe(pipe_slow);
15308 %}
15309
15310 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15311 %{
15312 match(If cmp (OverflowMulL op1 op2));
15313 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15314 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15315 effect(USE labl, KILL cr);
15316
15317 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15318 "smulh rscratch2, $op1, $op2\n\t"
15319 "cmp rscratch2, rscratch1, ASR #63\n\t"
15320 "b$cmp $labl" %}
15321 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15322 ins_encode %{
15323 Label* L = $labl$$label;
15324 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15325 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15326 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15327 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15328 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15329 %}
15330
15331 ins_pipe(pipe_serial);
15332 %}
15333
15334 // ============================================================================
15335 // Compare Instructions
15336
15337 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15338 %{
15339 match(Set cr (CmpI op1 op2));
15340
15341 effect(DEF cr, USE op1, USE op2);
15342
15343 ins_cost(INSN_COST);
15344 format %{ "cmpw $op1, $op2" %}
15345
15346 ins_encode(aarch64_enc_cmpw(op1, op2));
15347
15348 ins_pipe(icmp_reg_reg);
15349 %}
15350
15351 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15352 %{
15353 match(Set cr (CmpI op1 zero));
15354
15355 effect(DEF cr, USE op1);
15356
15357 ins_cost(INSN_COST);
15358 format %{ "cmpw $op1, 0" %}
15359
15360 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15361
15362 ins_pipe(icmp_reg_imm);
15363 %}
15364
15365 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15366 %{
15367 match(Set cr (CmpI op1 op2));
15368
15369 effect(DEF cr, USE op1);
15370
15371 ins_cost(INSN_COST);
15372 format %{ "cmpw $op1, $op2" %}
15373
15374 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15375
15376 ins_pipe(icmp_reg_imm);
15377 %}
15378
15379 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15380 %{
15381 match(Set cr (CmpI op1 op2));
15382
15383 effect(DEF cr, USE op1);
15384
15385 ins_cost(INSN_COST * 2);
15386 format %{ "cmpw $op1, $op2" %}
15387
15388 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15389
15390 ins_pipe(icmp_reg_imm);
15391 %}
15392
15393 // Unsigned compare Instructions; really, same as signed compare
15394 // except it should only be used to feed an If or a CMovI which takes a
15395 // cmpOpU.
15396
15397 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15398 %{
15399 match(Set cr (CmpU op1 op2));
15400
15401 effect(DEF cr, USE op1, USE op2);
15402
15403 ins_cost(INSN_COST);
15404 format %{ "cmpw $op1, $op2\t# unsigned" %}
15405
15406 ins_encode(aarch64_enc_cmpw(op1, op2));
15407
15408 ins_pipe(icmp_reg_reg);
15409 %}
15410
15411 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15412 %{
15413 match(Set cr (CmpU op1 zero));
15414
15415 effect(DEF cr, USE op1);
15416
15417 ins_cost(INSN_COST);
15418 format %{ "cmpw $op1, #0\t# unsigned" %}
15419
15420 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15421
15422 ins_pipe(icmp_reg_imm);
15423 %}
15424
15425 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15426 %{
15427 match(Set cr (CmpU op1 op2));
15428
15429 effect(DEF cr, USE op1);
15430
15431 ins_cost(INSN_COST);
15432 format %{ "cmpw $op1, $op2\t# unsigned" %}
15433
15434 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15435
15436 ins_pipe(icmp_reg_imm);
15437 %}
15438
15439 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15440 %{
15441 match(Set cr (CmpU op1 op2));
15442
15443 effect(DEF cr, USE op1);
15444
15445 ins_cost(INSN_COST * 2);
15446 format %{ "cmpw $op1, $op2\t# unsigned" %}
15447
15448 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15449
15450 ins_pipe(icmp_reg_imm);
15451 %}
15452
15453 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15454 %{
15455 match(Set cr (CmpL op1 op2));
15456
15457 effect(DEF cr, USE op1, USE op2);
15458
15459 ins_cost(INSN_COST);
15460 format %{ "cmp $op1, $op2" %}
15461
15462 ins_encode(aarch64_enc_cmp(op1, op2));
15463
15464 ins_pipe(icmp_reg_reg);
15465 %}
15466
15467 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15468 %{
15469 match(Set cr (CmpL op1 zero));
15470
15471 effect(DEF cr, USE op1);
15472
15473 ins_cost(INSN_COST);
15474 format %{ "tst $op1" %}
15475
15476 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15477
15478 ins_pipe(icmp_reg_imm);
15479 %}
15480
15481 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15482 %{
15483 match(Set cr (CmpL op1 op2));
15484
15485 effect(DEF cr, USE op1);
15486
15487 ins_cost(INSN_COST);
15488 format %{ "cmp $op1, $op2" %}
15489
15490 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15491
15492 ins_pipe(icmp_reg_imm);
15493 %}
15494
15495 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15496 %{
15497 match(Set cr (CmpL op1 op2));
15498
15499 effect(DEF cr, USE op1);
15500
15501 ins_cost(INSN_COST * 2);
15502 format %{ "cmp $op1, $op2" %}
15503
15504 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15505
15506 ins_pipe(icmp_reg_imm);
15507 %}
15508
15509 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15510 %{
15511 match(Set cr (CmpUL op1 op2));
15512
15513 effect(DEF cr, USE op1, USE op2);
15514
15515 ins_cost(INSN_COST);
15516 format %{ "cmp $op1, $op2" %}
15517
15518 ins_encode(aarch64_enc_cmp(op1, op2));
15519
15520 ins_pipe(icmp_reg_reg);
15521 %}
15522
15523 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15524 %{
15525 match(Set cr (CmpUL op1 zero));
15526
15527 effect(DEF cr, USE op1);
15528
15529 ins_cost(INSN_COST);
15530 format %{ "tst $op1" %}
15531
15532 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15533
15534 ins_pipe(icmp_reg_imm);
15535 %}
15536
15537 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15538 %{
15539 match(Set cr (CmpUL op1 op2));
15540
15541 effect(DEF cr, USE op1);
15542
15543 ins_cost(INSN_COST);
15544 format %{ "cmp $op1, $op2" %}
15545
15546 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15547
15548 ins_pipe(icmp_reg_imm);
15549 %}
15550
15551 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15552 %{
15553 match(Set cr (CmpUL op1 op2));
15554
15555 effect(DEF cr, USE op1);
15556
15557 ins_cost(INSN_COST * 2);
15558 format %{ "cmp $op1, $op2" %}
15559
15560 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15561
15562 ins_pipe(icmp_reg_imm);
15563 %}
15564
15565 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15566 %{
15567 match(Set cr (CmpP op1 op2));
15568
15569 effect(DEF cr, USE op1, USE op2);
15570
15571 ins_cost(INSN_COST);
15572 format %{ "cmp $op1, $op2\t // ptr" %}
15573
15574 ins_encode(aarch64_enc_cmpp(op1, op2));
15575
15576 ins_pipe(icmp_reg_reg);
15577 %}
15578
15579 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15580 %{
15581 match(Set cr (CmpN op1 op2));
15582
15583 effect(DEF cr, USE op1, USE op2);
15584
15585 ins_cost(INSN_COST);
15586 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15587
15588 ins_encode(aarch64_enc_cmpn(op1, op2));
15589
15590 ins_pipe(icmp_reg_reg);
15591 %}
15592
15593 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15594 %{
15595 match(Set cr (CmpP op1 zero));
15596
15597 effect(DEF cr, USE op1, USE zero);
15598
15599 ins_cost(INSN_COST);
15600 format %{ "cmp $op1, 0\t // ptr" %}
15601
15602 ins_encode(aarch64_enc_testp(op1));
15603
15604 ins_pipe(icmp_reg_imm);
15605 %}
15606
15607 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15608 %{
15609 match(Set cr (CmpN op1 zero));
15610
15611 effect(DEF cr, USE op1, USE zero);
15612
15613 ins_cost(INSN_COST);
15614 format %{ "cmp $op1, 0\t // compressed ptr" %}
15615
15616 ins_encode(aarch64_enc_testn(op1));
15617
15618 ins_pipe(icmp_reg_imm);
15619 %}
15620
15621 // FP comparisons
15622 //
15623 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15624 // using normal cmpOp. See declaration of rFlagsReg for details.
15625
15626 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15627 %{
15628 match(Set cr (CmpF src1 src2));
15629
15630 ins_cost(3 * INSN_COST);
15631 format %{ "fcmps $src1, $src2" %}
15632
15633 ins_encode %{
15634 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15635 %}
15636
15637 ins_pipe(pipe_class_compare);
15638 %}
15639
15640 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15641 %{
15642 match(Set cr (CmpF src1 src2));
15643
15644 ins_cost(3 * INSN_COST);
15645 format %{ "fcmps $src1, 0.0" %}
15646
15647 ins_encode %{
15648 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15649 %}
15650
15651 ins_pipe(pipe_class_compare);
15652 %}
15653 // FROM HERE
15654
15655 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15656 %{
15657 match(Set cr (CmpD src1 src2));
15658
15659 ins_cost(3 * INSN_COST);
15660 format %{ "fcmpd $src1, $src2" %}
15661
15662 ins_encode %{
15663 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15664 %}
15665
15666 ins_pipe(pipe_class_compare);
15667 %}
15668
15669 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15670 %{
15671 match(Set cr (CmpD src1 src2));
15672
15673 ins_cost(3 * INSN_COST);
15674 format %{ "fcmpd $src1, 0.0" %}
15675
15676 ins_encode %{
15677 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15678 %}
15679
15680 ins_pipe(pipe_class_compare);
15681 %}
15682
15683 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15684 %{
15685 match(Set dst (CmpF3 src1 src2));
15686 effect(KILL cr);
15687
15688 ins_cost(5 * INSN_COST);
15689 format %{ "fcmps $src1, $src2\n\t"
15690 "csinvw($dst, zr, zr, eq\n\t"
15691 "csnegw($dst, $dst, $dst, lt)"
15692 %}
15693
15694 ins_encode %{
15695 Label done;
15696 FloatRegister s1 = as_FloatRegister($src1$$reg);
15697 FloatRegister s2 = as_FloatRegister($src2$$reg);
15698 Register d = as_Register($dst$$reg);
15699 __ fcmps(s1, s2);
15700 // installs 0 if EQ else -1
15701 __ csinvw(d, zr, zr, Assembler::EQ);
15702 // keeps -1 if less or unordered else installs 1
15703 __ csnegw(d, d, d, Assembler::LT);
15704 __ bind(done);
15705 %}
15706
15707 ins_pipe(pipe_class_default);
15708
15709 %}
15710
15711 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15712 %{
15713 match(Set dst (CmpD3 src1 src2));
15714 effect(KILL cr);
15715
15716 ins_cost(5 * INSN_COST);
15717 format %{ "fcmpd $src1, $src2\n\t"
15718 "csinvw($dst, zr, zr, eq\n\t"
15719 "csnegw($dst, $dst, $dst, lt)"
15720 %}
15721
15722 ins_encode %{
15723 Label done;
15724 FloatRegister s1 = as_FloatRegister($src1$$reg);
15725 FloatRegister s2 = as_FloatRegister($src2$$reg);
15726 Register d = as_Register($dst$$reg);
15727 __ fcmpd(s1, s2);
15728 // installs 0 if EQ else -1
15729 __ csinvw(d, zr, zr, Assembler::EQ);
15730 // keeps -1 if less or unordered else installs 1
15731 __ csnegw(d, d, d, Assembler::LT);
15732 __ bind(done);
15733 %}
15734 ins_pipe(pipe_class_default);
15735
15736 %}
15737
15738 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15739 %{
15740 match(Set dst (CmpF3 src1 zero));
15741 effect(KILL cr);
15742
15743 ins_cost(5 * INSN_COST);
15744 format %{ "fcmps $src1, 0.0\n\t"
15745 "csinvw($dst, zr, zr, eq\n\t"
15746 "csnegw($dst, $dst, $dst, lt)"
15747 %}
15748
15749 ins_encode %{
15750 Label done;
15751 FloatRegister s1 = as_FloatRegister($src1$$reg);
15752 Register d = as_Register($dst$$reg);
15753 __ fcmps(s1, 0.0);
15754 // installs 0 if EQ else -1
15755 __ csinvw(d, zr, zr, Assembler::EQ);
15756 // keeps -1 if less or unordered else installs 1
15757 __ csnegw(d, d, d, Assembler::LT);
15758 __ bind(done);
15759 %}
15760
15761 ins_pipe(pipe_class_default);
15762
15763 %}
15764
15765 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15766 %{
15767 match(Set dst (CmpD3 src1 zero));
15768 effect(KILL cr);
15769
15770 ins_cost(5 * INSN_COST);
15771 format %{ "fcmpd $src1, 0.0\n\t"
15772 "csinvw($dst, zr, zr, eq\n\t"
15773 "csnegw($dst, $dst, $dst, lt)"
15774 %}
15775
15776 ins_encode %{
15777 Label done;
15778 FloatRegister s1 = as_FloatRegister($src1$$reg);
15779 Register d = as_Register($dst$$reg);
15780 __ fcmpd(s1, 0.0);
15781 // installs 0 if EQ else -1
15782 __ csinvw(d, zr, zr, Assembler::EQ);
15783 // keeps -1 if less or unordered else installs 1
15784 __ csnegw(d, d, d, Assembler::LT);
15785 __ bind(done);
15786 %}
15787 ins_pipe(pipe_class_default);
15788
15789 %}
15790
15791 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15792 %{
15793 match(Set dst (CmpLTMask p q));
15794 effect(KILL cr);
15795
15796 ins_cost(3 * INSN_COST);
15797
15798 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15799 "csetw $dst, lt\n\t"
15800 "subw $dst, zr, $dst"
15801 %}
15802
15803 ins_encode %{
15804 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15805 __ csetw(as_Register($dst$$reg), Assembler::LT);
15806 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15807 %}
15808
15809 ins_pipe(ialu_reg_reg);
15810 %}
15811
15812 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15813 %{
15814 match(Set dst (CmpLTMask src zero));
15815 effect(KILL cr);
15816
15817 ins_cost(INSN_COST);
15818
15819 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15820
15821 ins_encode %{
15822 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15823 %}
15824
15825 ins_pipe(ialu_reg_shift);
15826 %}
15827
15828 // ============================================================================
15829 // Max and Min
15830
15831 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15832
15833 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15834 %{
15835 effect(DEF cr, USE src);
15836 ins_cost(INSN_COST);
15837 format %{ "cmpw $src, 0" %}
15838
15839 ins_encode %{
15840 __ cmpw($src$$Register, 0);
15841 %}
15842 ins_pipe(icmp_reg_imm);
15843 %}
15844
15845 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15846 %{
15847 match(Set dst (MinI src1 src2));
15848 ins_cost(INSN_COST * 3);
15849
15850 expand %{
15851 rFlagsReg cr;
15852 compI_reg_reg(cr, src1, src2);
15853 cmovI_reg_reg_lt(dst, src1, src2, cr);
15854 %}
15855 %}
15856
15857 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15858 %{
15859 match(Set dst (MaxI src1 src2));
15860 ins_cost(INSN_COST * 3);
15861
15862 expand %{
15863 rFlagsReg cr;
15864 compI_reg_reg(cr, src1, src2);
15865 cmovI_reg_reg_gt(dst, src1, src2, cr);
15866 %}
15867 %}
15868
15869
15870 // ============================================================================
15871 // Branch Instructions
15872
15873 // Direct Branch.
15874 instruct branch(label lbl)
15875 %{
15876 match(Goto);
15877
15878 effect(USE lbl);
15879
15880 ins_cost(BRANCH_COST);
15881 format %{ "b $lbl" %}
15882
15883 ins_encode(aarch64_enc_b(lbl));
15884
15885 ins_pipe(pipe_branch);
15886 %}
15887
15888 // Conditional Near Branch
15889 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15890 %{
15891 // Same match rule as `branchConFar'.
15892 match(If cmp cr);
15893
15894 effect(USE lbl);
15895
15896 ins_cost(BRANCH_COST);
15897 // If set to 1 this indicates that the current instruction is a
15898 // short variant of a long branch. This avoids using this
15899 // instruction in first-pass matching. It will then only be used in
15900 // the `Shorten_branches' pass.
15901 // ins_short_branch(1);
15902 format %{ "b$cmp $lbl" %}
15903
15904 ins_encode(aarch64_enc_br_con(cmp, lbl));
15905
15906 ins_pipe(pipe_branch_cond);
15907 %}
15908
15909 // Conditional Near Branch Unsigned
15910 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15911 %{
15912 // Same match rule as `branchConFar'.
15913 match(If cmp cr);
15914
15915 effect(USE lbl);
15916
15917 ins_cost(BRANCH_COST);
15918 // If set to 1 this indicates that the current instruction is a
15919 // short variant of a long branch. This avoids using this
15920 // instruction in first-pass matching. It will then only be used in
15921 // the `Shorten_branches' pass.
15922 // ins_short_branch(1);
15923 format %{ "b$cmp $lbl\t# unsigned" %}
15924
15925 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15926
15927 ins_pipe(pipe_branch_cond);
15928 %}
15929
15930 // Make use of CBZ and CBNZ. These instructions, as well as being
15931 // shorter than (cmp; branch), have the additional benefit of not
15932 // killing the flags.
15933
15934 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15935 match(If cmp (CmpI op1 op2));
15936 effect(USE labl);
15937
15938 ins_cost(BRANCH_COST);
15939 format %{ "cbw$cmp $op1, $labl" %}
15940 ins_encode %{
15941 Label* L = $labl$$label;
15942 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15943 if (cond == Assembler::EQ)
15944 __ cbzw($op1$$Register, *L);
15945 else
15946 __ cbnzw($op1$$Register, *L);
15947 %}
15948 ins_pipe(pipe_cmp_branch);
15949 %}
15950
15951 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15952 match(If cmp (CmpL op1 op2));
15953 effect(USE labl);
15954
15955 ins_cost(BRANCH_COST);
15956 format %{ "cb$cmp $op1, $labl" %}
15957 ins_encode %{
15958 Label* L = $labl$$label;
15959 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15960 if (cond == Assembler::EQ)
15961 __ cbz($op1$$Register, *L);
15962 else
15963 __ cbnz($op1$$Register, *L);
15964 %}
15965 ins_pipe(pipe_cmp_branch);
15966 %}
15967
15968 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15969 match(If cmp (CmpP op1 op2));
15970 effect(USE labl);
15971
15972 ins_cost(BRANCH_COST);
15973 format %{ "cb$cmp $op1, $labl" %}
15974 ins_encode %{
15975 Label* L = $labl$$label;
15976 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15977 if (cond == Assembler::EQ)
15978 __ cbz($op1$$Register, *L);
15979 else
15980 __ cbnz($op1$$Register, *L);
15981 %}
15982 ins_pipe(pipe_cmp_branch);
15983 %}
15984
15985 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15986 match(If cmp (CmpN op1 op2));
15987 effect(USE labl);
15988
15989 ins_cost(BRANCH_COST);
15990 format %{ "cbw$cmp $op1, $labl" %}
15991 ins_encode %{
15992 Label* L = $labl$$label;
15993 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15994 if (cond == Assembler::EQ)
15995 __ cbzw($op1$$Register, *L);
15996 else
15997 __ cbnzw($op1$$Register, *L);
15998 %}
15999 ins_pipe(pipe_cmp_branch);
16000 %}
16001
16002 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16003 match(If cmp (CmpP (DecodeN oop) zero));
16004 effect(USE labl);
16005
16006 ins_cost(BRANCH_COST);
16007 format %{ "cb$cmp $oop, $labl" %}
16008 ins_encode %{
16009 Label* L = $labl$$label;
16010 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16011 if (cond == Assembler::EQ)
16012 __ cbzw($oop$$Register, *L);
16013 else
16014 __ cbnzw($oop$$Register, *L);
16015 %}
16016 ins_pipe(pipe_cmp_branch);
16017 %}
16018
16019 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16020 match(If cmp (CmpU op1 op2));
16021 effect(USE labl);
16022
16023 ins_cost(BRANCH_COST);
16024 format %{ "cbw$cmp $op1, $labl" %}
16025 ins_encode %{
16026 Label* L = $labl$$label;
16027 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16028 if (cond == Assembler::EQ || cond == Assembler::LS) {
16029 __ cbzw($op1$$Register, *L);
16030 } else {
16031 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16032 __ cbnzw($op1$$Register, *L);
16033 }
16034 %}
16035 ins_pipe(pipe_cmp_branch);
16036 %}
16037
16038 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16039 match(If cmp (CmpUL op1 op2));
16040 effect(USE labl);
16041
16042 ins_cost(BRANCH_COST);
16043 format %{ "cb$cmp $op1, $labl" %}
16044 ins_encode %{
16045 Label* L = $labl$$label;
16046 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16047 if (cond == Assembler::EQ || cond == Assembler::LS) {
16048 __ cbz($op1$$Register, *L);
16049 } else {
16050 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16051 __ cbnz($op1$$Register, *L);
16052 }
16053 %}
16054 ins_pipe(pipe_cmp_branch);
16055 %}
16056
16057 // Test bit and Branch
16058
16059 // Patterns for short (< 32KiB) variants
16060 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16061 match(If cmp (CmpL op1 op2));
16062 effect(USE labl);
16063
16064 ins_cost(BRANCH_COST);
16065 format %{ "cb$cmp $op1, $labl # long" %}
16066 ins_encode %{
16067 Label* L = $labl$$label;
16068 Assembler::Condition cond =
16069 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16070 __ tbr(cond, $op1$$Register, 63, *L);
16071 %}
16072 ins_pipe(pipe_cmp_branch);
16073 ins_short_branch(1);
16074 %}
16075
16076 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16077 match(If cmp (CmpI op1 op2));
16078 effect(USE labl);
16079
16080 ins_cost(BRANCH_COST);
16081 format %{ "cb$cmp $op1, $labl # int" %}
16082 ins_encode %{
16083 Label* L = $labl$$label;
16084 Assembler::Condition cond =
16085 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16086 __ tbr(cond, $op1$$Register, 31, *L);
16087 %}
16088 ins_pipe(pipe_cmp_branch);
16089 ins_short_branch(1);
16090 %}
16091
16092 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16093 match(If cmp (CmpL (AndL op1 op2) op3));
16094 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16095 effect(USE labl);
16096
16097 ins_cost(BRANCH_COST);
16098 format %{ "tb$cmp $op1, $op2, $labl" %}
16099 ins_encode %{
16100 Label* L = $labl$$label;
16101 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16102 int bit = exact_log2_long($op2$$constant);
16103 __ tbr(cond, $op1$$Register, bit, *L);
16104 %}
16105 ins_pipe(pipe_cmp_branch);
16106 ins_short_branch(1);
16107 %}
16108
16109 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16110 match(If cmp (CmpI (AndI op1 op2) op3));
16111 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16112 effect(USE labl);
16113
16114 ins_cost(BRANCH_COST);
16115 format %{ "tb$cmp $op1, $op2, $labl" %}
16116 ins_encode %{
16117 Label* L = $labl$$label;
16118 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16119 int bit = exact_log2((juint)$op2$$constant);
16120 __ tbr(cond, $op1$$Register, bit, *L);
16121 %}
16122 ins_pipe(pipe_cmp_branch);
16123 ins_short_branch(1);
16124 %}
16125
16126 // And far variants
16127 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16128 match(If cmp (CmpL op1 op2));
16129 effect(USE labl);
16130
16131 ins_cost(BRANCH_COST);
16132 format %{ "cb$cmp $op1, $labl # long" %}
16133 ins_encode %{
16134 Label* L = $labl$$label;
16135 Assembler::Condition cond =
16136 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16137 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16138 %}
16139 ins_pipe(pipe_cmp_branch);
16140 %}
16141
16142 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16143 match(If cmp (CmpI op1 op2));
16144 effect(USE labl);
16145
16146 ins_cost(BRANCH_COST);
16147 format %{ "cb$cmp $op1, $labl # int" %}
16148 ins_encode %{
16149 Label* L = $labl$$label;
16150 Assembler::Condition cond =
16151 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16152 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16153 %}
16154 ins_pipe(pipe_cmp_branch);
16155 %}
16156
16157 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16158 match(If cmp (CmpL (AndL op1 op2) op3));
16159 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16160 effect(USE labl);
16161
16162 ins_cost(BRANCH_COST);
16163 format %{ "tb$cmp $op1, $op2, $labl" %}
16164 ins_encode %{
16165 Label* L = $labl$$label;
16166 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16167 int bit = exact_log2_long($op2$$constant);
16168 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16169 %}
16170 ins_pipe(pipe_cmp_branch);
16171 %}
16172
16173 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16174 match(If cmp (CmpI (AndI op1 op2) op3));
16175 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16176 effect(USE labl);
16177
16178 ins_cost(BRANCH_COST);
16179 format %{ "tb$cmp $op1, $op2, $labl" %}
16180 ins_encode %{
16181 Label* L = $labl$$label;
16182 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16183 int bit = exact_log2((juint)$op2$$constant);
16184 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16185 %}
16186 ins_pipe(pipe_cmp_branch);
16187 %}
16188
16189 // Test bits
16190
16191 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16192 match(Set cr (CmpL (AndL op1 op2) op3));
16193 predicate(Assembler::operand_valid_for_logical_immediate
16194 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16195
16196 ins_cost(INSN_COST);
16197 format %{ "tst $op1, $op2 # long" %}
16198 ins_encode %{
16199 __ tst($op1$$Register, $op2$$constant);
16200 %}
16201 ins_pipe(ialu_reg_reg);
16202 %}
16203
16204 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16205 match(Set cr (CmpI (AndI op1 op2) op3));
16206 predicate(Assembler::operand_valid_for_logical_immediate
16207 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16208
16209 ins_cost(INSN_COST);
16210 format %{ "tst $op1, $op2 # int" %}
16211 ins_encode %{
16212 __ tstw($op1$$Register, $op2$$constant);
16213 %}
16214 ins_pipe(ialu_reg_reg);
16215 %}
16216
16217 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16218 match(Set cr (CmpL (AndL op1 op2) op3));
16219
16220 ins_cost(INSN_COST);
16221 format %{ "tst $op1, $op2 # long" %}
16222 ins_encode %{
16223 __ tst($op1$$Register, $op2$$Register);
16224 %}
16225 ins_pipe(ialu_reg_reg);
16226 %}
16227
16228 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16229 match(Set cr (CmpI (AndI op1 op2) op3));
16230
16231 ins_cost(INSN_COST);
16232 format %{ "tstw $op1, $op2 # int" %}
16233 ins_encode %{
16234 __ tstw($op1$$Register, $op2$$Register);
16235 %}
16236 ins_pipe(ialu_reg_reg);
16237 %}
16238
16239
16240 // Conditional Far Branch
16241 // Conditional Far Branch Unsigned
16242 // TODO: fixme
16243
16244 // counted loop end branch near
16245 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16246 %{
16247 match(CountedLoopEnd cmp cr);
16248
16249 effect(USE lbl);
16250
16251 ins_cost(BRANCH_COST);
16252 // short variant.
16253 // ins_short_branch(1);
16254 format %{ "b$cmp $lbl \t// counted loop end" %}
16255
16256 ins_encode(aarch64_enc_br_con(cmp, lbl));
16257
16258 ins_pipe(pipe_branch);
16259 %}
16260
16261 // counted loop end branch far
16262 // TODO: fixme
16263
16264 // ============================================================================
16265 // inlined locking and unlocking
16266
16267 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16268 %{
16269 match(Set cr (FastLock object box));
16270 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16271
16272 ins_cost(5 * INSN_COST);
16273 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16274
16275 ins_encode %{
16276 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16277 %}
16278
16279 ins_pipe(pipe_serial);
16280 %}
16281
16282 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16283 %{
16284 match(Set cr (FastUnlock object box));
16285 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16286
16287 ins_cost(5 * INSN_COST);
16288 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16289
16290 ins_encode %{
16291 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16292 %}
16293
16294 ins_pipe(pipe_serial);
16295 %}
16296
16297 // ============================================================================
16298 // Safepoint Instructions
16299
16300 // TODO
16301 // provide a near and far version of this code
16302
16303 instruct safePoint(rFlagsReg cr, iRegP poll)
16304 %{
16305 match(SafePoint poll);
16306 effect(KILL cr);
16307
16308 format %{
16309 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16310 %}
16311 ins_encode %{
16312 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16313 %}
16314 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16315 %}
16316
16317
16318 // ============================================================================
16319 // Procedure Call/Return Instructions
16320
16321 // Call Java Static Instruction
16322
16323 instruct CallStaticJavaDirect(method meth)
16324 %{
16325 match(CallStaticJava);
16326
16327 effect(USE meth);
16328
16329 ins_cost(CALL_COST);
16330
16331 format %{ "call,static $meth \t// ==> " %}
16332
16333 ins_encode(aarch64_enc_java_static_call(meth),
16334 aarch64_enc_call_epilog);
16335
16336 ins_pipe(pipe_class_call);
16337 %}
16338
16339 // TO HERE
16340
16341 // Call Java Dynamic Instruction
16342 instruct CallDynamicJavaDirect(method meth)
16343 %{
16344 match(CallDynamicJava);
16345
16346 effect(USE meth);
16347
16348 ins_cost(CALL_COST);
16349
16350 format %{ "CALL,dynamic $meth \t// ==> " %}
16351
16352 ins_encode(aarch64_enc_java_dynamic_call(meth),
16353 aarch64_enc_call_epilog);
16354
16355 ins_pipe(pipe_class_call);
16356 %}
16357
16358 // Call Runtime Instruction
16359
16360 instruct CallRuntimeDirect(method meth)
16361 %{
16362 match(CallRuntime);
16363
16364 effect(USE meth);
16365
16366 ins_cost(CALL_COST);
16367
16368 format %{ "CALL, runtime $meth" %}
16369
16370 ins_encode( aarch64_enc_java_to_runtime(meth) );
16371
16372 ins_pipe(pipe_class_call);
16373 %}
16374
16375 // Call Runtime Instruction
16376
16377 instruct CallLeafDirect(method meth)
16378 %{
16379 match(CallLeaf);
16380
16381 effect(USE meth);
16382
16383 ins_cost(CALL_COST);
16384
16385 format %{ "CALL, runtime leaf $meth" %}
16386
16387 ins_encode( aarch64_enc_java_to_runtime(meth) );
16388
16389 ins_pipe(pipe_class_call);
16390 %}
16391
16392 // Call Runtime Instruction without safepoint and with vector arguments
16393 instruct CallLeafDirectVector(method meth)
16394 %{
16395 match(CallLeafVector);
16396
16397 effect(USE meth);
16398
16399 ins_cost(CALL_COST);
16400
16401 format %{ "CALL, runtime leaf vector $meth" %}
16402
16403 ins_encode(aarch64_enc_java_to_runtime(meth));
16404
16405 ins_pipe(pipe_class_call);
16406 %}
16407
16408 // Call Runtime Instruction
16409
16410 instruct CallLeafNoFPDirect(method meth)
16411 %{
16412 match(CallLeafNoFP);
16413
16414 effect(USE meth);
16415
16416 ins_cost(CALL_COST);
16417
16418 format %{ "CALL, runtime leaf nofp $meth" %}
16419
16420 ins_encode( aarch64_enc_java_to_runtime(meth) );
16421
16422 ins_pipe(pipe_class_call);
16423 %}
16424
16425 // Tail Call; Jump from runtime stub to Java code.
16426 // Also known as an 'interprocedural jump'.
16427 // Target of jump will eventually return to caller.
16428 // TailJump below removes the return address.
16429 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16430 // emitted just above the TailCall which has reset rfp to the caller state.
16431 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16432 %{
16433 match(TailCall jump_target method_ptr);
16434
16435 ins_cost(CALL_COST);
16436
16437 format %{ "br $jump_target\t# $method_ptr holds method" %}
16438
16439 ins_encode(aarch64_enc_tail_call(jump_target));
16440
16441 ins_pipe(pipe_class_call);
16442 %}
16443
16444 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16445 %{
16446 match(TailJump jump_target ex_oop);
16447
16448 ins_cost(CALL_COST);
16449
16450 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16451
16452 ins_encode(aarch64_enc_tail_jmp(jump_target));
16453
16454 ins_pipe(pipe_class_call);
16455 %}
16456
16457 // Forward exception.
16458 instruct ForwardExceptionjmp()
16459 %{
16460 match(ForwardException);
16461 ins_cost(CALL_COST);
16462
16463 format %{ "b forward_exception_stub" %}
16464 ins_encode %{
16465 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16466 %}
16467 ins_pipe(pipe_class_call);
16468 %}
16469
16470 // Create exception oop: created by stack-crawling runtime code.
16471 // Created exception is now available to this handler, and is setup
16472 // just prior to jumping to this handler. No code emitted.
16473 // TODO check
16474 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16475 instruct CreateException(iRegP_R0 ex_oop)
16476 %{
16477 match(Set ex_oop (CreateEx));
16478
16479 format %{ " -- \t// exception oop; no code emitted" %}
16480
16481 size(0);
16482
16483 ins_encode( /*empty*/ );
16484
16485 ins_pipe(pipe_class_empty);
16486 %}
16487
16488 // Rethrow exception: The exception oop will come in the first
16489 // argument position. Then JUMP (not call) to the rethrow stub code.
16490 instruct RethrowException() %{
16491 match(Rethrow);
16492 ins_cost(CALL_COST);
16493
16494 format %{ "b rethrow_stub" %}
16495
16496 ins_encode( aarch64_enc_rethrow() );
16497
16498 ins_pipe(pipe_class_call);
16499 %}
16500
16501
16502 // Return Instruction
16503 // epilog node loads ret address into lr as part of frame pop
16504 instruct Ret()
16505 %{
16506 match(Return);
16507
16508 format %{ "ret\t// return register" %}
16509
16510 ins_encode( aarch64_enc_ret() );
16511
16512 ins_pipe(pipe_branch);
16513 %}
16514
16515 // Die now.
16516 instruct ShouldNotReachHere() %{
16517 match(Halt);
16518
16519 ins_cost(CALL_COST);
16520 format %{ "ShouldNotReachHere" %}
16521
16522 ins_encode %{
16523 if (is_reachable()) {
16524 const char* str = __ code_string(_halt_reason);
16525 __ stop(str);
16526 }
16527 %}
16528
16529 ins_pipe(pipe_class_default);
16530 %}
16531
16532 // ============================================================================
16533 // Partial Subtype Check
16534 //
16535 // superklass array for an instance of the superklass. Set a hidden
16536 // internal cache on a hit (cache is checked with exposed code in
16537 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16538 // encoding ALSO sets flags.
16539
16540 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16541 %{
16542 match(Set result (PartialSubtypeCheck sub super));
16543 predicate(!UseSecondarySupersTable);
16544 effect(KILL cr, KILL temp);
16545
16546 ins_cost(20 * INSN_COST); // slightly larger than the next version
16547 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16548
16549 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16550
16551 opcode(0x1); // Force zero of result reg on hit
16552
16553 ins_pipe(pipe_class_memory);
16554 %}
16555
16556 // Two versions of partialSubtypeCheck, both used when we need to
16557 // search for a super class in the secondary supers array. The first
16558 // is used when we don't know _a priori_ the class being searched
16559 // for. The second, far more common, is used when we do know: this is
16560 // used for instanceof, checkcast, and any case where C2 can determine
16561 // it by constant propagation.
16562
16563 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16564 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16565 rFlagsReg cr)
16566 %{
16567 match(Set result (PartialSubtypeCheck sub super));
16568 predicate(UseSecondarySupersTable);
16569 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16570
16571 ins_cost(10 * INSN_COST); // slightly larger than the next version
16572 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16573
16574 ins_encode %{
16575 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16576 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16577 $vtemp$$FloatRegister,
16578 $result$$Register, /*L_success*/nullptr);
16579 %}
16580
16581 ins_pipe(pipe_class_memory);
16582 %}
16583
16584 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16585 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16586 rFlagsReg cr)
16587 %{
16588 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16589 predicate(UseSecondarySupersTable);
16590 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16591
16592 ins_cost(5 * INSN_COST); // smaller than the next version
16593 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16594
16595 ins_encode %{
16596 bool success = false;
16597 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16598 if (InlineSecondarySupersTest) {
16599 success =
16600 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16601 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16602 $vtemp$$FloatRegister,
16603 $result$$Register,
16604 super_klass_slot);
16605 } else {
16606 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16607 success = (call != nullptr);
16608 }
16609 if (!success) {
16610 ciEnv::current()->record_failure("CodeCache is full");
16611 return;
16612 }
16613 %}
16614
16615 ins_pipe(pipe_class_memory);
16616 %}
16617
16618 // Intrisics for String.compareTo()
16619
16620 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16621 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16622 %{
16623 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16624 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16625 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16626
16627 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16628 ins_encode %{
16629 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16630 __ string_compare($str1$$Register, $str2$$Register,
16631 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16632 $tmp1$$Register, $tmp2$$Register,
16633 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16634 %}
16635 ins_pipe(pipe_class_memory);
16636 %}
16637
16638 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16639 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16640 %{
16641 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16642 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16643 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16644
16645 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16646 ins_encode %{
16647 __ string_compare($str1$$Register, $str2$$Register,
16648 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16649 $tmp1$$Register, $tmp2$$Register,
16650 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16651 %}
16652 ins_pipe(pipe_class_memory);
16653 %}
16654
16655 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16656 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16657 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16658 %{
16659 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16660 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16661 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16662 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16663
16664 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16665 ins_encode %{
16666 __ string_compare($str1$$Register, $str2$$Register,
16667 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16668 $tmp1$$Register, $tmp2$$Register,
16669 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16670 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16671 %}
16672 ins_pipe(pipe_class_memory);
16673 %}
16674
16675 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16676 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16677 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16678 %{
16679 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16680 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16681 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16682 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16683
16684 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16685 ins_encode %{
16686 __ string_compare($str1$$Register, $str2$$Register,
16687 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16688 $tmp1$$Register, $tmp2$$Register,
16689 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16690 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16691 %}
16692 ins_pipe(pipe_class_memory);
16693 %}
16694
16695 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16696 // these string_compare variants as NEON register type for convenience so that the prototype of
16697 // string_compare can be shared with all variants.
16698
16699 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16700 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16701 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16702 pRegGov_P1 pgtmp2, rFlagsReg cr)
16703 %{
16704 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16705 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16706 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16707 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16708
16709 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16710 ins_encode %{
16711 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16712 __ string_compare($str1$$Register, $str2$$Register,
16713 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16714 $tmp1$$Register, $tmp2$$Register,
16715 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16716 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16717 StrIntrinsicNode::LL);
16718 %}
16719 ins_pipe(pipe_class_memory);
16720 %}
16721
16722 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16723 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16724 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16725 pRegGov_P1 pgtmp2, rFlagsReg cr)
16726 %{
16727 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16728 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16729 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16730 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16731
16732 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16733 ins_encode %{
16734 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16735 __ string_compare($str1$$Register, $str2$$Register,
16736 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16737 $tmp1$$Register, $tmp2$$Register,
16738 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16739 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16740 StrIntrinsicNode::LU);
16741 %}
16742 ins_pipe(pipe_class_memory);
16743 %}
16744
16745 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16746 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16747 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16748 pRegGov_P1 pgtmp2, rFlagsReg cr)
16749 %{
16750 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16751 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16752 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16753 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16754
16755 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16756 ins_encode %{
16757 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16758 __ string_compare($str1$$Register, $str2$$Register,
16759 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16760 $tmp1$$Register, $tmp2$$Register,
16761 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16762 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16763 StrIntrinsicNode::UL);
16764 %}
16765 ins_pipe(pipe_class_memory);
16766 %}
16767
16768 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16769 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16770 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16771 pRegGov_P1 pgtmp2, rFlagsReg cr)
16772 %{
16773 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16774 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16775 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16776 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16777
16778 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16779 ins_encode %{
16780 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16781 __ string_compare($str1$$Register, $str2$$Register,
16782 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16783 $tmp1$$Register, $tmp2$$Register,
16784 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16785 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16786 StrIntrinsicNode::UU);
16787 %}
16788 ins_pipe(pipe_class_memory);
16789 %}
16790
16791 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16792 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16793 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16794 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16795 %{
16796 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16797 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16798 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16799 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16800 TEMP vtmp0, TEMP vtmp1, KILL cr);
16801 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16802 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16803
16804 ins_encode %{
16805 __ string_indexof($str1$$Register, $str2$$Register,
16806 $cnt1$$Register, $cnt2$$Register,
16807 $tmp1$$Register, $tmp2$$Register,
16808 $tmp3$$Register, $tmp4$$Register,
16809 $tmp5$$Register, $tmp6$$Register,
16810 -1, $result$$Register, StrIntrinsicNode::UU);
16811 %}
16812 ins_pipe(pipe_class_memory);
16813 %}
16814
16815 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16816 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16817 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16818 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16819 %{
16820 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16821 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16822 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16823 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16824 TEMP vtmp0, TEMP vtmp1, KILL cr);
16825 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16826 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16827
16828 ins_encode %{
16829 __ string_indexof($str1$$Register, $str2$$Register,
16830 $cnt1$$Register, $cnt2$$Register,
16831 $tmp1$$Register, $tmp2$$Register,
16832 $tmp3$$Register, $tmp4$$Register,
16833 $tmp5$$Register, $tmp6$$Register,
16834 -1, $result$$Register, StrIntrinsicNode::LL);
16835 %}
16836 ins_pipe(pipe_class_memory);
16837 %}
16838
16839 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16840 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16841 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16842 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16843 %{
16844 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16845 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16846 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16847 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16848 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16849 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16850 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16851
16852 ins_encode %{
16853 __ string_indexof($str1$$Register, $str2$$Register,
16854 $cnt1$$Register, $cnt2$$Register,
16855 $tmp1$$Register, $tmp2$$Register,
16856 $tmp3$$Register, $tmp4$$Register,
16857 $tmp5$$Register, $tmp6$$Register,
16858 -1, $result$$Register, StrIntrinsicNode::UL);
16859 %}
16860 ins_pipe(pipe_class_memory);
16861 %}
16862
16863 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16864 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16865 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16866 %{
16867 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16868 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16869 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16870 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16871 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16872 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16873
16874 ins_encode %{
16875 int icnt2 = (int)$int_cnt2$$constant;
16876 __ string_indexof($str1$$Register, $str2$$Register,
16877 $cnt1$$Register, zr,
16878 $tmp1$$Register, $tmp2$$Register,
16879 $tmp3$$Register, $tmp4$$Register, zr, zr,
16880 icnt2, $result$$Register, StrIntrinsicNode::UU);
16881 %}
16882 ins_pipe(pipe_class_memory);
16883 %}
16884
16885 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16886 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16887 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16888 %{
16889 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16890 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16891 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16892 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16893 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16894 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16895
16896 ins_encode %{
16897 int icnt2 = (int)$int_cnt2$$constant;
16898 __ string_indexof($str1$$Register, $str2$$Register,
16899 $cnt1$$Register, zr,
16900 $tmp1$$Register, $tmp2$$Register,
16901 $tmp3$$Register, $tmp4$$Register, zr, zr,
16902 icnt2, $result$$Register, StrIntrinsicNode::LL);
16903 %}
16904 ins_pipe(pipe_class_memory);
16905 %}
16906
16907 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16908 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16909 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16910 %{
16911 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16912 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16913 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16914 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16915 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16916 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16917
16918 ins_encode %{
16919 int icnt2 = (int)$int_cnt2$$constant;
16920 __ string_indexof($str1$$Register, $str2$$Register,
16921 $cnt1$$Register, zr,
16922 $tmp1$$Register, $tmp2$$Register,
16923 $tmp3$$Register, $tmp4$$Register, zr, zr,
16924 icnt2, $result$$Register, StrIntrinsicNode::UL);
16925 %}
16926 ins_pipe(pipe_class_memory);
16927 %}
16928
16929 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16930 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16931 iRegINoSp tmp3, rFlagsReg cr)
16932 %{
16933 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16934 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16935 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16936 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16937
16938 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16939
16940 ins_encode %{
16941 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16942 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16943 $tmp3$$Register);
16944 %}
16945 ins_pipe(pipe_class_memory);
16946 %}
16947
16948 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16949 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16950 iRegINoSp tmp3, rFlagsReg cr)
16951 %{
16952 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16953 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16954 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16955 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16956
16957 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16958
16959 ins_encode %{
16960 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16961 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16962 $tmp3$$Register);
16963 %}
16964 ins_pipe(pipe_class_memory);
16965 %}
16966
16967 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16968 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16969 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16970 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16971 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16972 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16973 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16974 ins_encode %{
16975 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16976 $result$$Register, $ztmp1$$FloatRegister,
16977 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16978 $ptmp$$PRegister, true /* isL */);
16979 %}
16980 ins_pipe(pipe_class_memory);
16981 %}
16982
16983 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16984 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16985 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16986 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16987 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16988 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16989 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16990 ins_encode %{
16991 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16992 $result$$Register, $ztmp1$$FloatRegister,
16993 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16994 $ptmp$$PRegister, false /* isL */);
16995 %}
16996 ins_pipe(pipe_class_memory);
16997 %}
16998
16999 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17000 iRegI_R0 result, rFlagsReg cr)
17001 %{
17002 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17003 match(Set result (StrEquals (Binary str1 str2) cnt));
17004 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17005
17006 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17007 ins_encode %{
17008 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17009 __ string_equals($str1$$Register, $str2$$Register,
17010 $result$$Register, $cnt$$Register);
17011 %}
17012 ins_pipe(pipe_class_memory);
17013 %}
17014
17015 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17016 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17017 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17018 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17019 iRegP_R10 tmp, rFlagsReg cr)
17020 %{
17021 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17022 match(Set result (AryEq ary1 ary2));
17023 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17024 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17025 TEMP vtmp6, TEMP vtmp7, KILL cr);
17026
17027 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17028 ins_encode %{
17029 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17030 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17031 $result$$Register, $tmp$$Register, 1);
17032 if (tpc == nullptr) {
17033 ciEnv::current()->record_failure("CodeCache is full");
17034 return;
17035 }
17036 %}
17037 ins_pipe(pipe_class_memory);
17038 %}
17039
17040 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17041 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17042 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17043 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17044 iRegP_R10 tmp, rFlagsReg cr)
17045 %{
17046 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17047 match(Set result (AryEq ary1 ary2));
17048 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17049 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17050 TEMP vtmp6, TEMP vtmp7, KILL cr);
17051
17052 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17053 ins_encode %{
17054 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17055 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17056 $result$$Register, $tmp$$Register, 2);
17057 if (tpc == nullptr) {
17058 ciEnv::current()->record_failure("CodeCache is full");
17059 return;
17060 }
17061 %}
17062 ins_pipe(pipe_class_memory);
17063 %}
17064
17065 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17066 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17067 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17068 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17069 %{
17070 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17071 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17072 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17073
17074 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
17075 ins_encode %{
17076 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17077 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17078 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17079 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17080 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17081 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17082 (BasicType)$basic_type$$constant);
17083 if (tpc == nullptr) {
17084 ciEnv::current()->record_failure("CodeCache is full");
17085 return;
17086 }
17087 %}
17088 ins_pipe(pipe_class_memory);
17089 %}
17090
17091 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17092 %{
17093 match(Set result (CountPositives ary1 len));
17094 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17095 format %{ "count positives byte[] $ary1,$len -> $result" %}
17096 ins_encode %{
17097 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17098 if (tpc == nullptr) {
17099 ciEnv::current()->record_failure("CodeCache is full");
17100 return;
17101 }
17102 %}
17103 ins_pipe( pipe_slow );
17104 %}
17105
17106 // fast char[] to byte[] compression
17107 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17108 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17109 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17110 iRegI_R0 result, rFlagsReg cr)
17111 %{
17112 match(Set result (StrCompressedCopy src (Binary dst len)));
17113 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17114 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17115
17116 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17117 ins_encode %{
17118 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17119 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17120 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17121 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17122 %}
17123 ins_pipe(pipe_slow);
17124 %}
17125
17126 // fast byte[] to char[] inflation
17127 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17128 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17129 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17130 %{
17131 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17132 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17133 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17134 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17135
17136 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17137 ins_encode %{
17138 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17139 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17140 $vtmp2$$FloatRegister, $tmp$$Register);
17141 if (tpc == nullptr) {
17142 ciEnv::current()->record_failure("CodeCache is full");
17143 return;
17144 }
17145 %}
17146 ins_pipe(pipe_class_memory);
17147 %}
17148
17149 // encode char[] to byte[] in ISO_8859_1
17150 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17151 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17152 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17153 iRegI_R0 result, rFlagsReg cr)
17154 %{
17155 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17156 match(Set result (EncodeISOArray src (Binary dst len)));
17157 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17158 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17159
17160 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17161 ins_encode %{
17162 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17163 $result$$Register, false,
17164 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17165 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17166 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17167 %}
17168 ins_pipe(pipe_class_memory);
17169 %}
17170
17171 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17172 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17173 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17174 iRegI_R0 result, rFlagsReg cr)
17175 %{
17176 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17177 match(Set result (EncodeISOArray src (Binary dst len)));
17178 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17179 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17180
17181 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17182 ins_encode %{
17183 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17184 $result$$Register, true,
17185 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17186 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17187 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17188 %}
17189 ins_pipe(pipe_class_memory);
17190 %}
17191
17192 //----------------------------- CompressBits/ExpandBits ------------------------
17193
17194 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17195 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17196 match(Set dst (CompressBits src mask));
17197 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17198 format %{ "mov $tsrc, $src\n\t"
17199 "mov $tmask, $mask\n\t"
17200 "bext $tdst, $tsrc, $tmask\n\t"
17201 "mov $dst, $tdst"
17202 %}
17203 ins_encode %{
17204 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17205 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17206 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17207 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17208 %}
17209 ins_pipe(pipe_slow);
17210 %}
17211
17212 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17213 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17214 match(Set dst (CompressBits (LoadI mem) mask));
17215 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17216 format %{ "ldrs $tsrc, $mem\n\t"
17217 "ldrs $tmask, $mask\n\t"
17218 "bext $tdst, $tsrc, $tmask\n\t"
17219 "mov $dst, $tdst"
17220 %}
17221 ins_encode %{
17222 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17223 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17224 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17225 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17226 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17227 %}
17228 ins_pipe(pipe_slow);
17229 %}
17230
17231 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17232 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17233 match(Set dst (CompressBits src mask));
17234 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17235 format %{ "mov $tsrc, $src\n\t"
17236 "mov $tmask, $mask\n\t"
17237 "bext $tdst, $tsrc, $tmask\n\t"
17238 "mov $dst, $tdst"
17239 %}
17240 ins_encode %{
17241 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17242 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17243 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17244 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17245 %}
17246 ins_pipe(pipe_slow);
17247 %}
17248
17249 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17250 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17251 match(Set dst (CompressBits (LoadL mem) mask));
17252 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17253 format %{ "ldrd $tsrc, $mem\n\t"
17254 "ldrd $tmask, $mask\n\t"
17255 "bext $tdst, $tsrc, $tmask\n\t"
17256 "mov $dst, $tdst"
17257 %}
17258 ins_encode %{
17259 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17260 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17261 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17262 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17263 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17264 %}
17265 ins_pipe(pipe_slow);
17266 %}
17267
17268 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17269 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17270 match(Set dst (ExpandBits src mask));
17271 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17272 format %{ "mov $tsrc, $src\n\t"
17273 "mov $tmask, $mask\n\t"
17274 "bdep $tdst, $tsrc, $tmask\n\t"
17275 "mov $dst, $tdst"
17276 %}
17277 ins_encode %{
17278 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17279 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17280 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17281 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17282 %}
17283 ins_pipe(pipe_slow);
17284 %}
17285
17286 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17287 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17288 match(Set dst (ExpandBits (LoadI mem) mask));
17289 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17290 format %{ "ldrs $tsrc, $mem\n\t"
17291 "ldrs $tmask, $mask\n\t"
17292 "bdep $tdst, $tsrc, $tmask\n\t"
17293 "mov $dst, $tdst"
17294 %}
17295 ins_encode %{
17296 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17297 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17298 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17299 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17300 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17301 %}
17302 ins_pipe(pipe_slow);
17303 %}
17304
17305 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17306 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17307 match(Set dst (ExpandBits src mask));
17308 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17309 format %{ "mov $tsrc, $src\n\t"
17310 "mov $tmask, $mask\n\t"
17311 "bdep $tdst, $tsrc, $tmask\n\t"
17312 "mov $dst, $tdst"
17313 %}
17314 ins_encode %{
17315 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17316 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17317 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17318 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17319 %}
17320 ins_pipe(pipe_slow);
17321 %}
17322
17323
17324 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17325 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17326 match(Set dst (ExpandBits (LoadL mem) mask));
17327 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17328 format %{ "ldrd $tsrc, $mem\n\t"
17329 "ldrd $tmask, $mask\n\t"
17330 "bdep $tdst, $tsrc, $tmask\n\t"
17331 "mov $dst, $tdst"
17332 %}
17333 ins_encode %{
17334 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17335 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17336 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17337 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17338 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17339 %}
17340 ins_pipe(pipe_slow);
17341 %}
17342
17343 //----------------------------- Reinterpret ----------------------------------
17344 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17345 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17346 match(Set dst (ReinterpretHF2S src));
17347 format %{ "reinterpretHF2S $dst, $src" %}
17348 ins_encode %{
17349 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17350 %}
17351 ins_pipe(pipe_slow);
17352 %}
17353
17354 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17355 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17356 match(Set dst (ReinterpretS2HF src));
17357 format %{ "reinterpretS2HF $dst, $src" %}
17358 ins_encode %{
17359 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17360 %}
17361 ins_pipe(pipe_slow);
17362 %}
17363
17364 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17365 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17366 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
17367 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
17368 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
17369 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17370 // can be omitted in this pattern, resulting in -
17371 // fcvt $dst, $src // Convert float to half-precision float
17372 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17373 %{
17374 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17375 format %{ "convF2HFAndS2HF $dst, $src" %}
17376 ins_encode %{
17377 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17378 %}
17379 ins_pipe(pipe_slow);
17380 %}
17381
17382 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17383 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17384 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
17385 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17386 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
17387 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17388 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17389 // resulting in -
17390 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
17391 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17392 %{
17393 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17394 format %{ "convHF2SAndHF2F $dst, $src" %}
17395 ins_encode %{
17396 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17397 %}
17398 ins_pipe(pipe_slow);
17399 %}
17400
17401 // ============================================================================
17402 // This name is KNOWN by the ADLC and cannot be changed.
17403 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17404 // for this guy.
17405 instruct tlsLoadP(thread_RegP dst)
17406 %{
17407 match(Set dst (ThreadLocal));
17408
17409 ins_cost(0);
17410
17411 format %{ " -- \t// $dst=Thread::current(), empty" %}
17412
17413 size(0);
17414
17415 ins_encode( /*empty*/ );
17416
17417 ins_pipe(pipe_class_empty);
17418 %}
17419
17420 //----------PEEPHOLE RULES-----------------------------------------------------
17421 // These must follow all instruction definitions as they use the names
17422 // defined in the instructions definitions.
17423 //
17424 // peepmatch ( root_instr_name [preceding_instruction]* );
17425 //
17426 // peepconstraint %{
17427 // (instruction_number.operand_name relational_op instruction_number.operand_name
17428 // [, ...] );
17429 // // instruction numbers are zero-based using left to right order in peepmatch
17430 //
17431 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17432 // // provide an instruction_number.operand_name for each operand that appears
17433 // // in the replacement instruction's match rule
17434 //
17435 // ---------VM FLAGS---------------------------------------------------------
17436 //
17437 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17438 //
17439 // Each peephole rule is given an identifying number starting with zero and
17440 // increasing by one in the order seen by the parser. An individual peephole
17441 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17442 // on the command-line.
17443 //
17444 // ---------CURRENT LIMITATIONS----------------------------------------------
17445 //
17446 // Only match adjacent instructions in same basic block
17447 // Only equality constraints
17448 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17449 // Only one replacement instruction
17450 //
17451 // ---------EXAMPLE----------------------------------------------------------
17452 //
17453 // // pertinent parts of existing instructions in architecture description
17454 // instruct movI(iRegINoSp dst, iRegI src)
17455 // %{
17456 // match(Set dst (CopyI src));
17457 // %}
17458 //
17459 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17460 // %{
17461 // match(Set dst (AddI dst src));
17462 // effect(KILL cr);
17463 // %}
17464 //
17465 // // Change (inc mov) to lea
17466 // peephole %{
17467 // // increment preceded by register-register move
17468 // peepmatch ( incI_iReg movI );
17469 // // require that the destination register of the increment
17470 // // match the destination register of the move
17471 // peepconstraint ( 0.dst == 1.dst );
17472 // // construct a replacement instruction that sets
17473 // // the destination to ( move's source register + one )
17474 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17475 // %}
17476 //
17477
17478 // Implementation no longer uses movX instructions since
17479 // machine-independent system no longer uses CopyX nodes.
17480 //
17481 // peephole
17482 // %{
17483 // peepmatch (incI_iReg movI);
17484 // peepconstraint (0.dst == 1.dst);
17485 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17486 // %}
17487
17488 // peephole
17489 // %{
17490 // peepmatch (decI_iReg movI);
17491 // peepconstraint (0.dst == 1.dst);
17492 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17493 // %}
17494
17495 // peephole
17496 // %{
17497 // peepmatch (addI_iReg_imm movI);
17498 // peepconstraint (0.dst == 1.dst);
17499 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17500 // %}
17501
17502 // peephole
17503 // %{
17504 // peepmatch (incL_iReg movL);
17505 // peepconstraint (0.dst == 1.dst);
17506 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17507 // %}
17508
17509 // peephole
17510 // %{
17511 // peepmatch (decL_iReg movL);
17512 // peepconstraint (0.dst == 1.dst);
17513 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17514 // %}
17515
17516 // peephole
17517 // %{
17518 // peepmatch (addL_iReg_imm movL);
17519 // peepconstraint (0.dst == 1.dst);
17520 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17521 // %}
17522
17523 // peephole
17524 // %{
17525 // peepmatch (addP_iReg_imm movP);
17526 // peepconstraint (0.dst == 1.dst);
17527 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17528 // %}
17529
17530 // // Change load of spilled value to only a spill
17531 // instruct storeI(memory mem, iRegI src)
17532 // %{
17533 // match(Set mem (StoreI mem src));
17534 // %}
17535 //
17536 // instruct loadI(iRegINoSp dst, memory mem)
17537 // %{
17538 // match(Set dst (LoadI mem));
17539 // %}
17540 //
17541
17542 //----------SMARTSPILL RULES---------------------------------------------------
17543 // These must follow all instruction definitions as they use the names
17544 // defined in the instructions definitions.
17545
17546 // Local Variables:
17547 // mode: c++
17548 // End: