1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // Copyright 2025 Arm Limited and/or its affiliates.
5 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 //
7 // This code is free software; you can redistribute it and/or modify it
8 // under the terms of the GNU General Public License version 2 only, as
9 // published by the Free Software Foundation.
10 //
11 // This code is distributed in the hope that it will be useful, but WITHOUT
12 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 // version 2 for more details (a copy is included in the LICENSE file that
15 // accompanied this code).
16 //
17 // You should have received a copy of the GNU General Public License version
18 // 2 along with this work; if not, write to the Free Software Foundation,
19 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 //
21 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 // or visit www.oracle.com if you need additional information or have any
23 // questions.
24 //
25 //
26
27 // AArch64 Architecture Description File
28
29 //----------REGISTER DEFINITION BLOCK------------------------------------------
30 // This information is used by the matcher and the register allocator to
31 // describe individual registers and classes of registers within the target
32 // architecture.
33
34 register %{
35 //----------Architecture Description Register Definitions----------------------
36 // General Registers
37 // "reg_def" name ( register save type, C convention save type,
38 // ideal register type, encoding );
39 // Register Save Types:
40 //
41 // NS = No-Save: The register allocator assumes that these registers
42 // can be used without saving upon entry to the method, &
43 // that they do not need to be saved at call sites.
44 //
45 // SOC = Save-On-Call: The register allocator assumes that these registers
46 // can be used without saving upon entry to the method,
47 // but that they must be saved at call sites.
48 //
49 // SOE = Save-On-Entry: The register allocator assumes that these registers
50 // must be saved before using them upon entry to the
51 // method, but they do not need to be saved at call
52 // sites.
53 //
54 // AS = Always-Save: The register allocator assumes that these registers
55 // must be saved before using them upon entry to the
56 // method, & that they must be saved at call sites.
57 //
58 // Ideal Register Type is used to determine how to save & restore a
59 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
60 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
61 //
62 // The encoding number is the actual bit-pattern placed into the opcodes.
63
64 // We must define the 64 bit int registers in two 32 bit halves, the
65 // real lower register and a virtual upper half register. upper halves
66 // are used by the register allocator but are not actually supplied as
67 // operands to memory ops.
68 //
69 // follow the C1 compiler in making registers
70 //
71 // r0-r7,r10-r26 volatile (caller save)
72 // r27-r32 system (no save, no allocate)
73 // r8-r9 non-allocatable (so we can use them as scratch regs)
74 //
75 // as regards Java usage. we don't use any callee save registers
76 // because this makes it difficult to de-optimise a frame (see comment
77 // in x86 implementation of Deoptimization::unwind_callee_save_values)
78 //
79
80 // General Registers
81
82 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
83 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
84 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
85 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
86 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
87 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
88 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
89 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
90 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
91 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
92 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
93 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
94 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
95 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
96 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
97 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
98 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
99 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
100 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
101 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
102 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
103 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
104 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
105 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
106 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
107 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
108 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
109 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
110 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
111 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
112 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
113 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
114 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
115 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
116 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
117 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
118 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
119 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
120 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
121 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
122 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
123 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
124 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
125 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
126 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
127 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
128 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
129 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
130 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
131 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
132 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
133 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
134 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
135 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
136 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
137 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
138 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
139 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
140 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
141 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
142 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
143 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
144 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
145 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
146
147 // ----------------------------
148 // Float/Double/Vector Registers
149 // ----------------------------
150
151 // Double Registers
152
153 // The rules of ADL require that double registers be defined in pairs.
154 // Each pair must be two 32-bit values, but not necessarily a pair of
155 // single float registers. In each pair, ADLC-assigned register numbers
156 // must be adjacent, with the lower number even. Finally, when the
157 // CPU stores such a register pair to memory, the word associated with
158 // the lower ADLC-assigned number must be stored to the lower address.
159
160 // AArch64 has 32 floating-point registers. Each can store a vector of
161 // single or double precision floating-point values up to 8 * 32
162 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
163 // use the first float or double element of the vector.
164
165 // for Java use float registers v0-v15 are always save on call whereas
166 // the platform ABI treats v8-v15 as callee save). float registers
167 // v16-v31 are SOC as per the platform spec
168
169 // For SVE vector registers, we simply extend vector register size to 8
170 // 'logical' slots. This is nominally 256 bits but it actually covers
171 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
172 // bits. The 'physical' SVE vector register length is detected during
173 // startup, so the register allocator is able to identify the correct
174 // number of bytes needed for an SVE spill/unspill.
175 // Note that a vector register with 4 slots denotes a 128-bit NEON
176 // register allowing it to be distinguished from the corresponding SVE
177 // vector register when the SVE vector length is 128 bits.
178
179 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
180 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
181 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
182 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
183
184 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
185 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
186 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
187 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
188
189 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
190 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
191 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
192 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
193
194 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
195 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
196 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
197 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
198
199 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
200 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
201 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
202 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
203
204 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
205 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
206 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
207 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
208
209 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
210 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
211 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
212 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
213
214 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
215 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
216 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
217 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
218
219 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
220 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
221 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
222 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
223
224 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
225 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
226 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
227 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
228
229 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
230 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
231 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
232 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
233
234 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
235 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
236 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
237 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
238
239 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
240 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
241 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
242 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
243
244 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
245 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
246 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
247 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
248
249 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
250 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
251 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
252 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
253
254 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
255 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
256 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
257 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
258
259 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
260 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
261 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
262 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
263
264 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
265 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
266 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
267 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
268
269 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
270 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
271 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
272 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
273
274 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
275 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
276 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
277 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
278
279 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
280 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
281 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
282 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
283
284 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
285 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
286 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
287 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
288
289 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
290 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
291 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
292 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
293
294 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
295 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
296 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
297 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
298
299 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
300 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
301 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
302 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
303
304 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
305 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
306 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
307 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
308
309 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
310 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
311 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
312 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
313
314 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
315 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
316 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
317 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
318
319 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
320 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
321 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
322 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
323
324 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
325 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
326 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
327 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
328
329 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
330 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
331 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
332 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
333
334 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
335 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
336 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
337 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
338
339 // ----------------------------
340 // SVE Predicate Registers
341 // ----------------------------
342 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
343 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
344 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
345 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
346 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
347 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
348 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
349 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
350 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
351 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
352 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
353 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
354 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
355 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
356 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
357 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
358
359 // ----------------------------
360 // Special Registers
361 // ----------------------------
362
363 // the AArch64 CSPR status flag register is not directly accessible as
364 // instruction operand. the FPSR status flag register is a system
365 // register which can be written/read using MSR/MRS but again does not
366 // appear as an operand (a code identifying the FSPR occurs as an
367 // immediate value in the instruction).
368
369 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
370
371 // Specify priority of register selection within phases of register
372 // allocation. Highest priority is first. A useful heuristic is to
373 // give registers a low priority when they are required by machine
374 // instructions, like EAX and EDX on I486, and choose no-save registers
375 // before save-on-call, & save-on-call before save-on-entry. Registers
376 // which participate in fixed calling sequences should come last.
377 // Registers which are used as pairs must fall on an even boundary.
378
379 alloc_class chunk0(
380 // volatiles
381 R10, R10_H,
382 R11, R11_H,
383 R12, R12_H,
384 R13, R13_H,
385 R14, R14_H,
386 R15, R15_H,
387 R16, R16_H,
388 R17, R17_H,
389 R18, R18_H,
390
391 // arg registers
392 R0, R0_H,
393 R1, R1_H,
394 R2, R2_H,
395 R3, R3_H,
396 R4, R4_H,
397 R5, R5_H,
398 R6, R6_H,
399 R7, R7_H,
400
401 // non-volatiles
402 R19, R19_H,
403 R20, R20_H,
404 R21, R21_H,
405 R22, R22_H,
406 R23, R23_H,
407 R24, R24_H,
408 R25, R25_H,
409 R26, R26_H,
410
411 // non-allocatable registers
412
413 R27, R27_H, // heapbase
414 R28, R28_H, // thread
415 R29, R29_H, // fp
416 R30, R30_H, // lr
417 R31, R31_H, // sp
418 R8, R8_H, // rscratch1
419 R9, R9_H, // rscratch2
420 );
421
422 alloc_class chunk1(
423
424 // no save
425 V16, V16_H, V16_J, V16_K,
426 V17, V17_H, V17_J, V17_K,
427 V18, V18_H, V18_J, V18_K,
428 V19, V19_H, V19_J, V19_K,
429 V20, V20_H, V20_J, V20_K,
430 V21, V21_H, V21_J, V21_K,
431 V22, V22_H, V22_J, V22_K,
432 V23, V23_H, V23_J, V23_K,
433 V24, V24_H, V24_J, V24_K,
434 V25, V25_H, V25_J, V25_K,
435 V26, V26_H, V26_J, V26_K,
436 V27, V27_H, V27_J, V27_K,
437 V28, V28_H, V28_J, V28_K,
438 V29, V29_H, V29_J, V29_K,
439 V30, V30_H, V30_J, V30_K,
440 V31, V31_H, V31_J, V31_K,
441
442 // arg registers
443 V0, V0_H, V0_J, V0_K,
444 V1, V1_H, V1_J, V1_K,
445 V2, V2_H, V2_J, V2_K,
446 V3, V3_H, V3_J, V3_K,
447 V4, V4_H, V4_J, V4_K,
448 V5, V5_H, V5_J, V5_K,
449 V6, V6_H, V6_J, V6_K,
450 V7, V7_H, V7_J, V7_K,
451
452 // non-volatiles
453 V8, V8_H, V8_J, V8_K,
454 V9, V9_H, V9_J, V9_K,
455 V10, V10_H, V10_J, V10_K,
456 V11, V11_H, V11_J, V11_K,
457 V12, V12_H, V12_J, V12_K,
458 V13, V13_H, V13_J, V13_K,
459 V14, V14_H, V14_J, V14_K,
460 V15, V15_H, V15_J, V15_K,
461 );
462
463 alloc_class chunk2 (
464 // Governing predicates for load/store and arithmetic
465 P0,
466 P1,
467 P2,
468 P3,
469 P4,
470 P5,
471 P6,
472
473 // Extra predicates
474 P8,
475 P9,
476 P10,
477 P11,
478 P12,
479 P13,
480 P14,
481 P15,
482
483 // Preserved for all-true predicate
484 P7,
485 );
486
487 alloc_class chunk3(RFLAGS);
488
489 //----------Architecture Description Register Classes--------------------------
490 // Several register classes are automatically defined based upon information in
491 // this architecture description.
492 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
493 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
494 //
495
496 // Class for all 32 bit general purpose registers
497 reg_class all_reg32(
498 R0,
499 R1,
500 R2,
501 R3,
502 R4,
503 R5,
504 R6,
505 R7,
506 R10,
507 R11,
508 R12,
509 R13,
510 R14,
511 R15,
512 R16,
513 R17,
514 R18,
515 R19,
516 R20,
517 R21,
518 R22,
519 R23,
520 R24,
521 R25,
522 R26,
523 R27,
524 R28,
525 R29,
526 R30,
527 R31
528 );
529
530
531 // Class for all 32 bit integer registers (excluding SP which
532 // will never be used as an integer register)
533 reg_class any_reg32 %{
534 return _ANY_REG32_mask;
535 %}
536
537 // Singleton class for R0 int register
538 reg_class int_r0_reg(R0);
539
540 // Singleton class for R2 int register
541 reg_class int_r2_reg(R2);
542
543 // Singleton class for R3 int register
544 reg_class int_r3_reg(R3);
545
546 // Singleton class for R4 int register
547 reg_class int_r4_reg(R4);
548
549 // Singleton class for R31 int register
550 reg_class int_r31_reg(R31);
551
552 // Class for all 64 bit general purpose registers
553 reg_class all_reg(
554 R0, R0_H,
555 R1, R1_H,
556 R2, R2_H,
557 R3, R3_H,
558 R4, R4_H,
559 R5, R5_H,
560 R6, R6_H,
561 R7, R7_H,
562 R10, R10_H,
563 R11, R11_H,
564 R12, R12_H,
565 R13, R13_H,
566 R14, R14_H,
567 R15, R15_H,
568 R16, R16_H,
569 R17, R17_H,
570 R18, R18_H,
571 R19, R19_H,
572 R20, R20_H,
573 R21, R21_H,
574 R22, R22_H,
575 R23, R23_H,
576 R24, R24_H,
577 R25, R25_H,
578 R26, R26_H,
579 R27, R27_H,
580 R28, R28_H,
581 R29, R29_H,
582 R30, R30_H,
583 R31, R31_H
584 );
585
586 // Class for all long integer registers (including SP)
587 reg_class any_reg %{
588 return _ANY_REG_mask;
589 %}
590
591 // Class for non-allocatable 32 bit registers
592 reg_class non_allocatable_reg32(
593 #ifdef R18_RESERVED
594 // See comment in register_aarch64.hpp
595 R18, // tls on Windows
596 #endif
597 R28, // thread
598 R30, // lr
599 R31 // sp
600 );
601
602 // Class for non-allocatable 64 bit registers
603 reg_class non_allocatable_reg(
604 #ifdef R18_RESERVED
605 // See comment in register_aarch64.hpp
606 R18, R18_H, // tls on Windows, platform register on macOS
607 #endif
608 R28, R28_H, // thread
609 R30, R30_H, // lr
610 R31, R31_H // sp
611 );
612
613 // Class for all non-special integer registers
614 reg_class no_special_reg32 %{
615 return _NO_SPECIAL_REG32_mask;
616 %}
617
618 // Class for all non-special long integer registers
619 reg_class no_special_reg %{
620 return _NO_SPECIAL_REG_mask;
621 %}
622
623 // Class for 64 bit register r0
624 reg_class r0_reg(
625 R0, R0_H
626 );
627
628 // Class for 64 bit register r1
629 reg_class r1_reg(
630 R1, R1_H
631 );
632
633 // Class for 64 bit register r2
634 reg_class r2_reg(
635 R2, R2_H
636 );
637
638 // Class for 64 bit register r3
639 reg_class r3_reg(
640 R3, R3_H
641 );
642
643 // Class for 64 bit register r4
644 reg_class r4_reg(
645 R4, R4_H
646 );
647
648 // Class for 64 bit register r5
649 reg_class r5_reg(
650 R5, R5_H
651 );
652
653 // Class for 64 bit register r10
654 reg_class r10_reg(
655 R10, R10_H
656 );
657
658 // Class for 64 bit register r11
659 reg_class r11_reg(
660 R11, R11_H
661 );
662
663 // Class for method register
664 reg_class method_reg(
665 R12, R12_H
666 );
667
668 // Class for thread register
669 reg_class thread_reg(
670 R28, R28_H
671 );
672
673 // Class for frame pointer register
674 reg_class fp_reg(
675 R29, R29_H
676 );
677
678 // Class for link register
679 reg_class lr_reg(
680 R30, R30_H
681 );
682
683 // Class for long sp register
684 reg_class sp_reg(
685 R31, R31_H
686 );
687
688 // Class for all pointer registers
689 reg_class ptr_reg %{
690 return _PTR_REG_mask;
691 %}
692
693 // Class for all non_special pointer registers
694 reg_class no_special_ptr_reg %{
695 return _NO_SPECIAL_PTR_REG_mask;
696 %}
697
698 // Class for all non_special pointer registers (excluding rfp)
699 reg_class no_special_no_rfp_ptr_reg %{
700 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
701 %}
702
703 // Class for all float registers
704 reg_class float_reg(
705 V0,
706 V1,
707 V2,
708 V3,
709 V4,
710 V5,
711 V6,
712 V7,
713 V8,
714 V9,
715 V10,
716 V11,
717 V12,
718 V13,
719 V14,
720 V15,
721 V16,
722 V17,
723 V18,
724 V19,
725 V20,
726 V21,
727 V22,
728 V23,
729 V24,
730 V25,
731 V26,
732 V27,
733 V28,
734 V29,
735 V30,
736 V31
737 );
738
739 // Double precision float registers have virtual `high halves' that
740 // are needed by the allocator.
741 // Class for all double registers
742 reg_class double_reg(
743 V0, V0_H,
744 V1, V1_H,
745 V2, V2_H,
746 V3, V3_H,
747 V4, V4_H,
748 V5, V5_H,
749 V6, V6_H,
750 V7, V7_H,
751 V8, V8_H,
752 V9, V9_H,
753 V10, V10_H,
754 V11, V11_H,
755 V12, V12_H,
756 V13, V13_H,
757 V14, V14_H,
758 V15, V15_H,
759 V16, V16_H,
760 V17, V17_H,
761 V18, V18_H,
762 V19, V19_H,
763 V20, V20_H,
764 V21, V21_H,
765 V22, V22_H,
766 V23, V23_H,
767 V24, V24_H,
768 V25, V25_H,
769 V26, V26_H,
770 V27, V27_H,
771 V28, V28_H,
772 V29, V29_H,
773 V30, V30_H,
774 V31, V31_H
775 );
776
777 // Class for all SVE vector registers.
778 reg_class vectora_reg (
779 V0, V0_H, V0_J, V0_K,
780 V1, V1_H, V1_J, V1_K,
781 V2, V2_H, V2_J, V2_K,
782 V3, V3_H, V3_J, V3_K,
783 V4, V4_H, V4_J, V4_K,
784 V5, V5_H, V5_J, V5_K,
785 V6, V6_H, V6_J, V6_K,
786 V7, V7_H, V7_J, V7_K,
787 V8, V8_H, V8_J, V8_K,
788 V9, V9_H, V9_J, V9_K,
789 V10, V10_H, V10_J, V10_K,
790 V11, V11_H, V11_J, V11_K,
791 V12, V12_H, V12_J, V12_K,
792 V13, V13_H, V13_J, V13_K,
793 V14, V14_H, V14_J, V14_K,
794 V15, V15_H, V15_J, V15_K,
795 V16, V16_H, V16_J, V16_K,
796 V17, V17_H, V17_J, V17_K,
797 V18, V18_H, V18_J, V18_K,
798 V19, V19_H, V19_J, V19_K,
799 V20, V20_H, V20_J, V20_K,
800 V21, V21_H, V21_J, V21_K,
801 V22, V22_H, V22_J, V22_K,
802 V23, V23_H, V23_J, V23_K,
803 V24, V24_H, V24_J, V24_K,
804 V25, V25_H, V25_J, V25_K,
805 V26, V26_H, V26_J, V26_K,
806 V27, V27_H, V27_J, V27_K,
807 V28, V28_H, V28_J, V28_K,
808 V29, V29_H, V29_J, V29_K,
809 V30, V30_H, V30_J, V30_K,
810 V31, V31_H, V31_J, V31_K,
811 );
812
813 // Class for all 64bit vector registers
814 reg_class vectord_reg(
815 V0, V0_H,
816 V1, V1_H,
817 V2, V2_H,
818 V3, V3_H,
819 V4, V4_H,
820 V5, V5_H,
821 V6, V6_H,
822 V7, V7_H,
823 V8, V8_H,
824 V9, V9_H,
825 V10, V10_H,
826 V11, V11_H,
827 V12, V12_H,
828 V13, V13_H,
829 V14, V14_H,
830 V15, V15_H,
831 V16, V16_H,
832 V17, V17_H,
833 V18, V18_H,
834 V19, V19_H,
835 V20, V20_H,
836 V21, V21_H,
837 V22, V22_H,
838 V23, V23_H,
839 V24, V24_H,
840 V25, V25_H,
841 V26, V26_H,
842 V27, V27_H,
843 V28, V28_H,
844 V29, V29_H,
845 V30, V30_H,
846 V31, V31_H
847 );
848
849 // Class for all 128bit vector registers
850 reg_class vectorx_reg(
851 V0, V0_H, V0_J, V0_K,
852 V1, V1_H, V1_J, V1_K,
853 V2, V2_H, V2_J, V2_K,
854 V3, V3_H, V3_J, V3_K,
855 V4, V4_H, V4_J, V4_K,
856 V5, V5_H, V5_J, V5_K,
857 V6, V6_H, V6_J, V6_K,
858 V7, V7_H, V7_J, V7_K,
859 V8, V8_H, V8_J, V8_K,
860 V9, V9_H, V9_J, V9_K,
861 V10, V10_H, V10_J, V10_K,
862 V11, V11_H, V11_J, V11_K,
863 V12, V12_H, V12_J, V12_K,
864 V13, V13_H, V13_J, V13_K,
865 V14, V14_H, V14_J, V14_K,
866 V15, V15_H, V15_J, V15_K,
867 V16, V16_H, V16_J, V16_K,
868 V17, V17_H, V17_J, V17_K,
869 V18, V18_H, V18_J, V18_K,
870 V19, V19_H, V19_J, V19_K,
871 V20, V20_H, V20_J, V20_K,
872 V21, V21_H, V21_J, V21_K,
873 V22, V22_H, V22_J, V22_K,
874 V23, V23_H, V23_J, V23_K,
875 V24, V24_H, V24_J, V24_K,
876 V25, V25_H, V25_J, V25_K,
877 V26, V26_H, V26_J, V26_K,
878 V27, V27_H, V27_J, V27_K,
879 V28, V28_H, V28_J, V28_K,
880 V29, V29_H, V29_J, V29_K,
881 V30, V30_H, V30_J, V30_K,
882 V31, V31_H, V31_J, V31_K
883 );
884
885 // Class for vector register V10
886 reg_class v10_veca_reg(
887 V10, V10_H, V10_J, V10_K
888 );
889
890 // Class for vector register V11
891 reg_class v11_veca_reg(
892 V11, V11_H, V11_J, V11_K
893 );
894
895 // Class for vector register V12
896 reg_class v12_veca_reg(
897 V12, V12_H, V12_J, V12_K
898 );
899
900 // Class for vector register V13
901 reg_class v13_veca_reg(
902 V13, V13_H, V13_J, V13_K
903 );
904
905 // Class for vector register V17
906 reg_class v17_veca_reg(
907 V17, V17_H, V17_J, V17_K
908 );
909
910 // Class for vector register V18
911 reg_class v18_veca_reg(
912 V18, V18_H, V18_J, V18_K
913 );
914
915 // Class for vector register V23
916 reg_class v23_veca_reg(
917 V23, V23_H, V23_J, V23_K
918 );
919
920 // Class for vector register V24
921 reg_class v24_veca_reg(
922 V24, V24_H, V24_J, V24_K
923 );
924
925 // Class for 128 bit register v0
926 reg_class v0_reg(
927 V0, V0_H
928 );
929
930 // Class for 128 bit register v1
931 reg_class v1_reg(
932 V1, V1_H
933 );
934
935 // Class for 128 bit register v2
936 reg_class v2_reg(
937 V2, V2_H
938 );
939
940 // Class for 128 bit register v3
941 reg_class v3_reg(
942 V3, V3_H
943 );
944
945 // Class for 128 bit register v4
946 reg_class v4_reg(
947 V4, V4_H
948 );
949
950 // Class for 128 bit register v5
951 reg_class v5_reg(
952 V5, V5_H
953 );
954
955 // Class for 128 bit register v6
956 reg_class v6_reg(
957 V6, V6_H
958 );
959
960 // Class for 128 bit register v7
961 reg_class v7_reg(
962 V7, V7_H
963 );
964
965 // Class for 128 bit register v8
966 reg_class v8_reg(
967 V8, V8_H
968 );
969
970 // Class for 128 bit register v9
971 reg_class v9_reg(
972 V9, V9_H
973 );
974
975 // Class for 128 bit register v10
976 reg_class v10_reg(
977 V10, V10_H
978 );
979
980 // Class for 128 bit register v11
981 reg_class v11_reg(
982 V11, V11_H
983 );
984
985 // Class for 128 bit register v12
986 reg_class v12_reg(
987 V12, V12_H
988 );
989
990 // Class for 128 bit register v13
991 reg_class v13_reg(
992 V13, V13_H
993 );
994
995 // Class for 128 bit register v14
996 reg_class v14_reg(
997 V14, V14_H
998 );
999
1000 // Class for 128 bit register v15
1001 reg_class v15_reg(
1002 V15, V15_H
1003 );
1004
1005 // Class for 128 bit register v16
1006 reg_class v16_reg(
1007 V16, V16_H
1008 );
1009
1010 // Class for 128 bit register v17
1011 reg_class v17_reg(
1012 V17, V17_H
1013 );
1014
1015 // Class for 128 bit register v18
1016 reg_class v18_reg(
1017 V18, V18_H
1018 );
1019
1020 // Class for 128 bit register v19
1021 reg_class v19_reg(
1022 V19, V19_H
1023 );
1024
1025 // Class for 128 bit register v20
1026 reg_class v20_reg(
1027 V20, V20_H
1028 );
1029
1030 // Class for 128 bit register v21
1031 reg_class v21_reg(
1032 V21, V21_H
1033 );
1034
1035 // Class for 128 bit register v22
1036 reg_class v22_reg(
1037 V22, V22_H
1038 );
1039
1040 // Class for 128 bit register v23
1041 reg_class v23_reg(
1042 V23, V23_H
1043 );
1044
1045 // Class for 128 bit register v24
1046 reg_class v24_reg(
1047 V24, V24_H
1048 );
1049
1050 // Class for 128 bit register v25
1051 reg_class v25_reg(
1052 V25, V25_H
1053 );
1054
1055 // Class for 128 bit register v26
1056 reg_class v26_reg(
1057 V26, V26_H
1058 );
1059
1060 // Class for 128 bit register v27
1061 reg_class v27_reg(
1062 V27, V27_H
1063 );
1064
1065 // Class for 128 bit register v28
1066 reg_class v28_reg(
1067 V28, V28_H
1068 );
1069
1070 // Class for 128 bit register v29
1071 reg_class v29_reg(
1072 V29, V29_H
1073 );
1074
1075 // Class for 128 bit register v30
1076 reg_class v30_reg(
1077 V30, V30_H
1078 );
1079
1080 // Class for 128 bit register v31
1081 reg_class v31_reg(
1082 V31, V31_H
1083 );
1084
1085 // Class for all SVE predicate registers.
1086 reg_class pr_reg (
1087 P0,
1088 P1,
1089 P2,
1090 P3,
1091 P4,
1092 P5,
1093 P6,
1094 // P7, non-allocatable, preserved with all elements preset to TRUE.
1095 P8,
1096 P9,
1097 P10,
1098 P11,
1099 P12,
1100 P13,
1101 P14,
1102 P15
1103 );
1104
1105 // Class for SVE governing predicate registers, which are used
1106 // to determine the active elements of a predicated instruction.
1107 reg_class gov_pr (
1108 P0,
1109 P1,
1110 P2,
1111 P3,
1112 P4,
1113 P5,
1114 P6,
1115 // P7, non-allocatable, preserved with all elements preset to TRUE.
1116 );
1117
1118 reg_class p0_reg(P0);
1119 reg_class p1_reg(P1);
1120
1121 // Singleton class for condition codes
1122 reg_class int_flags(RFLAGS);
1123
1124 %}
1125
1126 //----------DEFINITION BLOCK---------------------------------------------------
1127 // Define name --> value mappings to inform the ADLC of an integer valued name
1128 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1129 // Format:
1130 // int_def <name> ( <int_value>, <expression>);
1131 // Generated Code in ad_<arch>.hpp
1132 // #define <name> (<expression>)
1133 // // value == <int_value>
1134 // Generated code in ad_<arch>.cpp adlc_verification()
1135 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1136 //
1137
1138 // we follow the ppc-aix port in using a simple cost model which ranks
1139 // register operations as cheap, memory ops as more expensive and
1140 // branches as most expensive. the first two have a low as well as a
1141 // normal cost. huge cost appears to be a way of saying don't do
1142 // something
1143
1144 definitions %{
1145 // The default cost (of a register move instruction).
1146 int_def INSN_COST ( 100, 100);
1147 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1148 int_def CALL_COST ( 200, 2 * INSN_COST);
1149 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1150 %}
1151
1152
1153 //----------SOURCE BLOCK-------------------------------------------------------
1154 // This is a block of C++ code which provides values, functions, and
1155 // definitions necessary in the rest of the architecture description
1156
1157 source_hpp %{
1158
1159 #include "asm/macroAssembler.hpp"
1160 #include "gc/shared/barrierSetAssembler.hpp"
1161 #include "gc/shared/cardTable.hpp"
1162 #include "gc/shared/cardTableBarrierSet.hpp"
1163 #include "gc/shared/collectedHeap.hpp"
1164 #include "opto/addnode.hpp"
1165 #include "opto/convertnode.hpp"
1166 #include "runtime/objectMonitor.hpp"
1167
1168 extern RegMask _ANY_REG32_mask;
1169 extern RegMask _ANY_REG_mask;
1170 extern RegMask _PTR_REG_mask;
1171 extern RegMask _NO_SPECIAL_REG32_mask;
1172 extern RegMask _NO_SPECIAL_REG_mask;
1173 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1174 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1175
1176 class CallStubImpl {
1177
1178 //--------------------------------------------------------------
1179 //---< Used for optimization in Compile::shorten_branches >---
1180 //--------------------------------------------------------------
1181
1182 public:
1183 // Size of call trampoline stub.
1184 static uint size_call_trampoline() {
1185 return 0; // no call trampolines on this platform
1186 }
1187
1188 // number of relocations needed by a call trampoline stub
1189 static uint reloc_call_trampoline() {
1190 return 0; // no call trampolines on this platform
1191 }
1192 };
1193
1194 class HandlerImpl {
1195
1196 public:
1197
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_deopt_handler() {
1201 // count one branch instruction and one far call instruction sequence
1202 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1203 }
1204 };
1205
1206 class Node::PD {
1207 public:
1208 enum NodeFlags {
1209 _last_flag = Node::_last_flag
1210 };
1211 };
1212
1213 bool is_CAS(int opcode, bool maybe_volatile);
1214
1215 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1216
1217 bool unnecessary_acquire(const Node *barrier);
1218 bool needs_acquiring_load(const Node *load);
1219
1220 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1221
1222 bool unnecessary_release(const Node *barrier);
1223 bool unnecessary_volatile(const Node *barrier);
1224 bool needs_releasing_store(const Node *store);
1225
1226 // predicate controlling translation of CompareAndSwapX
1227 bool needs_acquiring_load_exclusive(const Node *load);
1228
1229 // predicate controlling addressing modes
1230 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1231
1232 // Convert BootTest condition to Assembler condition.
1233 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1234 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1235 %}
1236
1237 source %{
1238
1239 // Derived RegMask with conditionally allocatable registers
1240
1241 void PhaseOutput::pd_perform_mach_node_analysis() {
1242 }
1243
1244 int MachNode::pd_alignment_required() const {
1245 return 1;
1246 }
1247
1248 int MachNode::compute_padding(int current_offset) const {
1249 return 0;
1250 }
1251
1252 RegMask _ANY_REG32_mask;
1253 RegMask _ANY_REG_mask;
1254 RegMask _PTR_REG_mask;
1255 RegMask _NO_SPECIAL_REG32_mask;
1256 RegMask _NO_SPECIAL_REG_mask;
1257 RegMask _NO_SPECIAL_PTR_REG_mask;
1258 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1259
1260 void reg_mask_init() {
1261 // We derive below RegMask(s) from the ones which are auto-generated from
1262 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1263 // registers conditionally reserved.
1264
1265 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1266 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1267
1268 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1269
1270 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1271
1272 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1273 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1274
1275 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1276 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1277
1278 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1279 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1280
1281 // r27 is not allocatable when compressed oops is on and heapbase is not
1282 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1283 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1284 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1285 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1286 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1287 }
1288
1289 // r29 is not allocatable when PreserveFramePointer is on
1290 if (PreserveFramePointer) {
1291 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1292 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1293 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1294 }
1295
1296 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1297 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 // Optimizaton of volatile gets and puts
1301 // -------------------------------------
1302 //
1303 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1304 // use to implement volatile reads and writes. For a volatile read
1305 // we simply need
1306 //
1307 // ldar<x>
1308 //
1309 // and for a volatile write we need
1310 //
1311 // stlr<x>
1312 //
1313 // Alternatively, we can implement them by pairing a normal
1314 // load/store with a memory barrier. For a volatile read we need
1315 //
1316 // ldr<x>
1317 // dmb ishld
1318 //
1319 // for a volatile write
1320 //
1321 // dmb ish
1322 // str<x>
1323 // dmb ish
1324 //
1325 // We can also use ldaxr and stlxr to implement compare and swap CAS
1326 // sequences. These are normally translated to an instruction
1327 // sequence like the following
1328 //
1329 // dmb ish
1330 // retry:
1331 // ldxr<x> rval raddr
1332 // cmp rval rold
1333 // b.ne done
1334 // stlxr<x> rval, rnew, rold
1335 // cbnz rval retry
1336 // done:
1337 // cset r0, eq
1338 // dmb ishld
1339 //
1340 // Note that the exclusive store is already using an stlxr
1341 // instruction. That is required to ensure visibility to other
1342 // threads of the exclusive write (assuming it succeeds) before that
1343 // of any subsequent writes.
1344 //
1345 // The following instruction sequence is an improvement on the above
1346 //
1347 // retry:
1348 // ldaxr<x> rval raddr
1349 // cmp rval rold
1350 // b.ne done
1351 // stlxr<x> rval, rnew, rold
1352 // cbnz rval retry
1353 // done:
1354 // cset r0, eq
1355 //
1356 // We don't need the leading dmb ish since the stlxr guarantees
1357 // visibility of prior writes in the case that the swap is
1358 // successful. Crucially we don't have to worry about the case where
1359 // the swap is not successful since no valid program should be
1360 // relying on visibility of prior changes by the attempting thread
1361 // in the case where the CAS fails.
1362 //
1363 // Similarly, we don't need the trailing dmb ishld if we substitute
1364 // an ldaxr instruction since that will provide all the guarantees we
1365 // require regarding observation of changes made by other threads
1366 // before any change to the CAS address observed by the load.
1367 //
1368 // In order to generate the desired instruction sequence we need to
1369 // be able to identify specific 'signature' ideal graph node
1370 // sequences which i) occur as a translation of a volatile reads or
1371 // writes or CAS operations and ii) do not occur through any other
1372 // translation or graph transformation. We can then provide
1373 // alternative aldc matching rules which translate these node
1374 // sequences to the desired machine code sequences. Selection of the
1375 // alternative rules can be implemented by predicates which identify
1376 // the relevant node sequences.
1377 //
1378 // The ideal graph generator translates a volatile read to the node
1379 // sequence
1380 //
1381 // LoadX[mo_acquire]
1382 // MemBarAcquire
1383 //
1384 // As a special case when using the compressed oops optimization we
1385 // may also see this variant
1386 //
1387 // LoadN[mo_acquire]
1388 // DecodeN
1389 // MemBarAcquire
1390 //
1391 // A volatile write is translated to the node sequence
1392 //
1393 // MemBarRelease
1394 // StoreX[mo_release] {CardMark}-optional
1395 // MemBarVolatile
1396 //
1397 // n.b. the above node patterns are generated with a strict
1398 // 'signature' configuration of input and output dependencies (see
1399 // the predicates below for exact details). The card mark may be as
1400 // simple as a few extra nodes or, in a few GC configurations, may
1401 // include more complex control flow between the leading and
1402 // trailing memory barriers. However, whatever the card mark
1403 // configuration these signatures are unique to translated volatile
1404 // reads/stores -- they will not appear as a result of any other
1405 // bytecode translation or inlining nor as a consequence of
1406 // optimizing transforms.
1407 //
1408 // We also want to catch inlined unsafe volatile gets and puts and
1409 // be able to implement them using either ldar<x>/stlr<x> or some
1410 // combination of ldr<x>/stlr<x> and dmb instructions.
1411 //
1412 // Inlined unsafe volatiles puts manifest as a minor variant of the
1413 // normal volatile put node sequence containing an extra cpuorder
1414 // membar
1415 //
1416 // MemBarRelease
1417 // MemBarCPUOrder
1418 // StoreX[mo_release] {CardMark}-optional
1419 // MemBarCPUOrder
1420 // MemBarVolatile
1421 //
1422 // n.b. as an aside, a cpuorder membar is not itself subject to
1423 // matching and translation by adlc rules. However, the rule
1424 // predicates need to detect its presence in order to correctly
1425 // select the desired adlc rules.
1426 //
1427 // Inlined unsafe volatile gets manifest as a slightly different
1428 // node sequence to a normal volatile get because of the
1429 // introduction of some CPUOrder memory barriers to bracket the
1430 // Load. However, but the same basic skeleton of a LoadX feeding a
1431 // MemBarAcquire, possibly through an optional DecodeN, is still
1432 // present
1433 //
1434 // MemBarCPUOrder
1435 // || \\
1436 // MemBarCPUOrder LoadX[mo_acquire]
1437 // || |
1438 // || {DecodeN} optional
1439 // || /
1440 // MemBarAcquire
1441 //
1442 // In this case the acquire membar does not directly depend on the
1443 // load. However, we can be sure that the load is generated from an
1444 // inlined unsafe volatile get if we see it dependent on this unique
1445 // sequence of membar nodes. Similarly, given an acquire membar we
1446 // can know that it was added because of an inlined unsafe volatile
1447 // get if it is fed and feeds a cpuorder membar and if its feed
1448 // membar also feeds an acquiring load.
1449 //
1450 // Finally an inlined (Unsafe) CAS operation is translated to the
1451 // following ideal graph
1452 //
1453 // MemBarRelease
1454 // MemBarCPUOrder
1455 // CompareAndSwapX {CardMark}-optional
1456 // MemBarCPUOrder
1457 // MemBarAcquire
1458 //
1459 // So, where we can identify these volatile read and write
1460 // signatures we can choose to plant either of the above two code
1461 // sequences. For a volatile read we can simply plant a normal
1462 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1463 // also choose to inhibit translation of the MemBarAcquire and
1464 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1465 //
1466 // When we recognise a volatile store signature we can choose to
1467 // plant at a dmb ish as a translation for the MemBarRelease, a
1468 // normal str<x> and then a dmb ish for the MemBarVolatile.
1469 // Alternatively, we can inhibit translation of the MemBarRelease
1470 // and MemBarVolatile and instead plant a simple stlr<x>
1471 // instruction.
1472 //
1473 // when we recognise a CAS signature we can choose to plant a dmb
1474 // ish as a translation for the MemBarRelease, the conventional
1475 // macro-instruction sequence for the CompareAndSwap node (which
1476 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1477 // Alternatively, we can elide generation of the dmb instructions
1478 // and plant the alternative CompareAndSwap macro-instruction
1479 // sequence (which uses ldaxr<x>).
1480 //
1481 // Of course, the above only applies when we see these signature
1482 // configurations. We still want to plant dmb instructions in any
1483 // other cases where we may see a MemBarAcquire, MemBarRelease or
1484 // MemBarVolatile. For example, at the end of a constructor which
1485 // writes final/volatile fields we will see a MemBarRelease
1486 // instruction and this needs a 'dmb ish' lest we risk the
1487 // constructed object being visible without making the
1488 // final/volatile field writes visible.
1489 //
1490 // n.b. the translation rules below which rely on detection of the
1491 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1492 // If we see anything other than the signature configurations we
1493 // always just translate the loads and stores to ldr<x> and str<x>
1494 // and translate acquire, release and volatile membars to the
1495 // relevant dmb instructions.
1496 //
1497
1498 // is_CAS(int opcode, bool maybe_volatile)
1499 //
1500 // return true if opcode is one of the possible CompareAndSwapX
1501 // values otherwise false.
1502
1503 bool is_CAS(int opcode, bool maybe_volatile)
1504 {
1505 switch(opcode) {
1506 // We handle these
1507 case Op_CompareAndSwapI:
1508 case Op_CompareAndSwapL:
1509 case Op_CompareAndSwapP:
1510 case Op_CompareAndSwapN:
1511 case Op_ShenandoahCompareAndSwapP:
1512 case Op_ShenandoahCompareAndSwapN:
1513 case Op_CompareAndSwapB:
1514 case Op_CompareAndSwapS:
1515 case Op_GetAndSetI:
1516 case Op_GetAndSetL:
1517 case Op_GetAndSetP:
1518 case Op_GetAndSetN:
1519 case Op_GetAndAddI:
1520 case Op_GetAndAddL:
1521 return true;
1522 case Op_CompareAndExchangeI:
1523 case Op_CompareAndExchangeN:
1524 case Op_CompareAndExchangeB:
1525 case Op_CompareAndExchangeS:
1526 case Op_CompareAndExchangeL:
1527 case Op_CompareAndExchangeP:
1528 case Op_WeakCompareAndSwapB:
1529 case Op_WeakCompareAndSwapS:
1530 case Op_WeakCompareAndSwapI:
1531 case Op_WeakCompareAndSwapL:
1532 case Op_WeakCompareAndSwapP:
1533 case Op_WeakCompareAndSwapN:
1534 case Op_ShenandoahWeakCompareAndSwapP:
1535 case Op_ShenandoahWeakCompareAndSwapN:
1536 case Op_ShenandoahCompareAndExchangeP:
1537 case Op_ShenandoahCompareAndExchangeN:
1538 return maybe_volatile;
1539 default:
1540 return false;
1541 }
1542 }
1543
1544 // helper to determine the maximum number of Phi nodes we may need to
1545 // traverse when searching from a card mark membar for the merge mem
1546 // feeding a trailing membar or vice versa
1547
1548 // predicates controlling emit of ldr<x>/ldar<x>
1549
1550 bool unnecessary_acquire(const Node *barrier)
1551 {
1552 assert(barrier->is_MemBar(), "expecting a membar");
1553
1554 MemBarNode* mb = barrier->as_MemBar();
1555
1556 if (mb->trailing_load()) {
1557 return true;
1558 }
1559
1560 if (mb->trailing_load_store()) {
1561 Node* load_store = mb->in(MemBarNode::Precedent);
1562 assert(load_store->is_LoadStore(), "unexpected graph shape");
1563 return is_CAS(load_store->Opcode(), true);
1564 }
1565
1566 return false;
1567 }
1568
1569 bool needs_acquiring_load(const Node *n)
1570 {
1571 assert(n->is_Load(), "expecting a load");
1572 LoadNode *ld = n->as_Load();
1573 return ld->is_acquire();
1574 }
1575
1576 bool unnecessary_release(const Node *n)
1577 {
1578 assert((n->is_MemBar() &&
1579 n->Opcode() == Op_MemBarRelease),
1580 "expecting a release membar");
1581
1582 MemBarNode *barrier = n->as_MemBar();
1583 if (!barrier->leading()) {
1584 return false;
1585 } else {
1586 Node* trailing = barrier->trailing_membar();
1587 MemBarNode* trailing_mb = trailing->as_MemBar();
1588 assert(trailing_mb->trailing(), "Not a trailing membar?");
1589 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1590
1591 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1592 if (mem->is_Store()) {
1593 assert(mem->as_Store()->is_release(), "");
1594 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1595 return true;
1596 } else {
1597 assert(mem->is_LoadStore(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1599 return is_CAS(mem->Opcode(), true);
1600 }
1601 }
1602 return false;
1603 }
1604
1605 bool unnecessary_volatile(const Node *n)
1606 {
1607 // assert n->is_MemBar();
1608 MemBarNode *mbvol = n->as_MemBar();
1609
1610 bool release = mbvol->trailing_store();
1611 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1612 #ifdef ASSERT
1613 if (release) {
1614 Node* leading = mbvol->leading_membar();
1615 assert(leading->Opcode() == Op_MemBarRelease, "");
1616 assert(leading->as_MemBar()->leading_store(), "");
1617 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1618 }
1619 #endif
1620
1621 return release;
1622 }
1623
1624 // predicates controlling emit of str<x>/stlr<x>
1625
1626 bool needs_releasing_store(const Node *n)
1627 {
1628 // assert n->is_Store();
1629 StoreNode *st = n->as_Store();
1630 return st->trailing_membar() != nullptr;
1631 }
1632
1633 // predicate controlling translation of CAS
1634 //
1635 // returns true if CAS needs to use an acquiring load otherwise false
1636
1637 bool needs_acquiring_load_exclusive(const Node *n)
1638 {
1639 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1640 LoadStoreNode* ldst = n->as_LoadStore();
1641 if (is_CAS(n->Opcode(), false)) {
1642 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1643 } else {
1644 return ldst->trailing_membar() != nullptr;
1645 }
1646
1647 // so we can just return true here
1648 return true;
1649 }
1650
1651 #define __ masm->
1652
1653 // advance declarations for helper functions to convert register
1654 // indices to register objects
1655
1656 // the ad file has to provide implementations of certain methods
1657 // expected by the generic code
1658 //
1659 // REQUIRED FUNCTIONALITY
1660
1661 //=============================================================================
1662
1663 // !!!!! Special hack to get all types of calls to specify the byte offset
1664 // from the start of the call to the point where the return address
1665 // will point.
1666
1667 int MachCallStaticJavaNode::ret_addr_offset()
1668 {
1669 // call should be a simple bl
1670 int off = 4;
1671 return off;
1672 }
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else {
1694 return 6 * NativeInstruction::instruction_size;
1695 }
1696 }
1697
1698 //=============================================================================
1699
1700 #ifndef PRODUCT
1701 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1702 st->print("BREAKPOINT");
1703 }
1704 #endif
1705
1706 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1707 __ brk(0);
1708 }
1709
1710 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1711 return MachNode::size(ra_);
1712 }
1713
1714 //=============================================================================
1715
1716 #ifndef PRODUCT
1717 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1718 st->print("nop \t# %d bytes pad for loops and calls", _count);
1719 }
1720 #endif
1721
1722 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1723 for (int i = 0; i < _count; i++) {
1724 __ nop();
1725 }
1726 }
1727
1728 uint MachNopNode::size(PhaseRegAlloc*) const {
1729 return _count * NativeInstruction::instruction_size;
1730 }
1731
1732 //=============================================================================
1733 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1734
1735 int ConstantTable::calculate_table_base_offset() const {
1736 return 0; // absolute addressing, no offset
1737 }
1738
1739 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1740 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1741 ShouldNotReachHere();
1742 }
1743
1744 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1745 // Empty encoding
1746 }
1747
1748 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1749 return 0;
1750 }
1751
1752 #ifndef PRODUCT
1753 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1754 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1755 }
1756 #endif
1757
1758 #ifndef PRODUCT
1759 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1760 Compile* C = ra_->C;
1761
1762 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1763
1764 if (C->output()->need_stack_bang(framesize))
1765 st->print("# stack bang size=%d\n\t", framesize);
1766
1767 if (VM_Version::use_rop_protection()) {
1768 st->print("ldr zr, [lr]\n\t");
1769 st->print("paciaz\n\t");
1770 }
1771 if (framesize < ((1 << 9) + 2 * wordSize)) {
1772 st->print("sub sp, sp, #%d\n\t", framesize);
1773 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1774 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1775 } else {
1776 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1777 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1778 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1779 st->print("sub sp, sp, rscratch1");
1780 }
1781 if (C->stub_function() == nullptr) {
1782 st->print("\n\t");
1783 st->print("ldr rscratch1, [guard]\n\t");
1784 st->print("dmb ishld\n\t");
1785 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1786 st->print("cmp rscratch1, rscratch2\n\t");
1787 st->print("b.eq skip");
1788 st->print("\n\t");
1789 st->print("blr #nmethod_entry_barrier_stub\n\t");
1790 st->print("b skip\n\t");
1791 st->print("guard: int\n\t");
1792 st->print("\n\t");
1793 st->print("skip:\n\t");
1794 }
1795 }
1796 #endif
1797
1798 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1799 Compile* C = ra_->C;
1800
1801 // n.b. frame size includes space for return pc and rfp
1802 const int framesize = C->output()->frame_size_in_bytes();
1803
1804 if (C->clinit_barrier_on_entry()) {
1805 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1806
1807 Label L_skip_barrier;
1808
1809 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1810 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1811 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1812 __ bind(L_skip_barrier);
1813 }
1814
1815 if (C->max_vector_size() > 0) {
1816 __ reinitialize_ptrue();
1817 }
1818
1819 int bangsize = C->output()->bang_size_in_bytes();
1820 if (C->output()->need_stack_bang(bangsize))
1821 __ generate_stack_overflow_check(bangsize);
1822
1823 __ build_frame(framesize);
1824
1825 if (C->stub_function() == nullptr) {
1826 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1827 // Dummy labels for just measuring the code size
1828 Label dummy_slow_path;
1829 Label dummy_continuation;
1830 Label dummy_guard;
1831 Label* slow_path = &dummy_slow_path;
1832 Label* continuation = &dummy_continuation;
1833 Label* guard = &dummy_guard;
1834 if (!Compile::current()->output()->in_scratch_emit_size()) {
1835 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1836 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1837 Compile::current()->output()->add_stub(stub);
1838 slow_path = &stub->entry();
1839 continuation = &stub->continuation();
1840 guard = &stub->guard();
1841 }
1842 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1843 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1844 }
1845
1846 if (VerifyStackAtCalls) {
1847 Unimplemented();
1848 }
1849
1850 C->output()->set_frame_complete(__ offset());
1851
1852 if (C->has_mach_constant_base_node()) {
1853 // NOTE: We set the table base offset here because users might be
1854 // emitted before MachConstantBaseNode.
1855 ConstantTable& constant_table = C->output()->constant_table();
1856 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1857 }
1858 }
1859
1860 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1861 {
1862 return MachNode::size(ra_); // too many variables; just compute it
1863 // the hard way
1864 }
1865
1866 int MachPrologNode::reloc() const
1867 {
1868 return 0;
1869 }
1870
1871 //=============================================================================
1872
1873 #ifndef PRODUCT
1874 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 st->print("# pop frame %d\n\t",framesize);
1879
1880 if (framesize == 0) {
1881 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1882 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1883 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1884 st->print("add sp, sp, #%d\n\t", framesize);
1885 } else {
1886 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1887 st->print("add sp, sp, rscratch1\n\t");
1888 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1889 }
1890 if (VM_Version::use_rop_protection()) {
1891 st->print("autiaz\n\t");
1892 st->print("ldr zr, [lr]\n\t");
1893 }
1894
1895 if (do_polling() && C->is_method_compilation()) {
1896 st->print("# test polling word\n\t");
1897 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1898 st->print("cmp sp, rscratch1\n\t");
1899 st->print("bhi #slow_path");
1900 }
1901 }
1902 #endif
1903
1904 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1905 Compile* C = ra_->C;
1906 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1907
1908 __ remove_frame(framesize);
1909
1910 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1911 __ reserved_stack_check();
1912 }
1913
1914 if (do_polling() && C->is_method_compilation()) {
1915 Label dummy_label;
1916 Label* code_stub = &dummy_label;
1917 if (!C->output()->in_scratch_emit_size()) {
1918 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1919 C->output()->add_stub(stub);
1920 code_stub = &stub->entry();
1921 }
1922 __ relocate(relocInfo::poll_return_type);
1923 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1924 }
1925 }
1926
1927 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1928 // Variable size. Determine dynamically.
1929 return MachNode::size(ra_);
1930 }
1931
1932 int MachEpilogNode::reloc() const {
1933 // Return number of relocatable values contained in this instruction.
1934 return 1; // 1 for polling page.
1935 }
1936
1937 const Pipeline * MachEpilogNode::pipeline() const {
1938 return MachNode::pipeline_class();
1939 }
1940
1941 //=============================================================================
1942
1943 static enum RC rc_class(OptoReg::Name reg) {
1944
1945 if (reg == OptoReg::Bad) {
1946 return rc_bad;
1947 }
1948
1949 // we have 32 int registers * 2 halves
1950 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1951
1952 if (reg < slots_of_int_registers) {
1953 return rc_int;
1954 }
1955
1956 // we have 32 float register * 8 halves
1957 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1958 if (reg < slots_of_int_registers + slots_of_float_registers) {
1959 return rc_float;
1960 }
1961
1962 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1963 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1964 return rc_predicate;
1965 }
1966
1967 // Between predicate regs & stack is the flags.
1968 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1969
1970 return rc_stack;
1971 }
1972
1973 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1974 Compile* C = ra_->C;
1975
1976 // Get registers to move.
1977 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1978 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1979 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1980 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1981
1982 enum RC src_hi_rc = rc_class(src_hi);
1983 enum RC src_lo_rc = rc_class(src_lo);
1984 enum RC dst_hi_rc = rc_class(dst_hi);
1985 enum RC dst_lo_rc = rc_class(dst_lo);
1986
1987 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1988
1989 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1990 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1991 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1992 "expected aligned-adjacent pairs");
1993 }
1994
1995 if (src_lo == dst_lo && src_hi == dst_hi) {
1996 return 0; // Self copy, no move.
1997 }
1998
1999 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2000 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2001 int src_offset = ra_->reg2offset(src_lo);
2002 int dst_offset = ra_->reg2offset(dst_lo);
2003
2004 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2005 uint ireg = ideal_reg();
2006 if (ireg == Op_VecA && masm) {
2007 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2008 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2009 // stack->stack
2010 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2011 sve_vector_reg_size_in_bytes);
2012 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2013 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2014 sve_vector_reg_size_in_bytes);
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2017 sve_vector_reg_size_in_bytes);
2018 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2019 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2020 as_FloatRegister(Matcher::_regEncode[src_lo]),
2021 as_FloatRegister(Matcher::_regEncode[src_lo]));
2022 } else {
2023 ShouldNotReachHere();
2024 }
2025 } else if (masm) {
2026 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2027 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2028 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2029 // stack->stack
2030 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2031 if (ireg == Op_VecD) {
2032 __ unspill(rscratch1, true, src_offset);
2033 __ spill(rscratch1, true, dst_offset);
2034 } else {
2035 __ spill_copy128(src_offset, dst_offset);
2036 }
2037 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2038 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2039 ireg == Op_VecD ? __ T8B : __ T16B,
2040 as_FloatRegister(Matcher::_regEncode[src_lo]));
2041 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2042 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2043 ireg == Op_VecD ? __ D : __ Q,
2044 ra_->reg2offset(dst_lo));
2045 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2046 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2047 ireg == Op_VecD ? __ D : __ Q,
2048 ra_->reg2offset(src_lo));
2049 } else {
2050 ShouldNotReachHere();
2051 }
2052 }
2053 } else if (masm) {
2054 switch (src_lo_rc) {
2055 case rc_int:
2056 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2057 if (is64) {
2058 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2059 as_Register(Matcher::_regEncode[src_lo]));
2060 } else {
2061 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2062 as_Register(Matcher::_regEncode[src_lo]));
2063 }
2064 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2065 if (is64) {
2066 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2067 as_Register(Matcher::_regEncode[src_lo]));
2068 } else {
2069 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2070 as_Register(Matcher::_regEncode[src_lo]));
2071 }
2072 } else { // gpr --> stack spill
2073 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2074 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2075 }
2076 break;
2077 case rc_float:
2078 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2079 if (is64) {
2080 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2081 as_FloatRegister(Matcher::_regEncode[src_lo]));
2082 } else {
2083 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2084 as_FloatRegister(Matcher::_regEncode[src_lo]));
2085 }
2086 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2087 if (is64) {
2088 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2089 as_FloatRegister(Matcher::_regEncode[src_lo]));
2090 } else {
2091 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2092 as_FloatRegister(Matcher::_regEncode[src_lo]));
2093 }
2094 } else { // fpr --> stack spill
2095 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2096 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2097 is64 ? __ D : __ S, dst_offset);
2098 }
2099 break;
2100 case rc_stack:
2101 if (dst_lo_rc == rc_int) { // stack --> gpr load
2102 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2103 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2104 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2105 is64 ? __ D : __ S, src_offset);
2106 } else if (dst_lo_rc == rc_predicate) {
2107 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2108 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2109 } else { // stack --> stack copy
2110 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2111 if (ideal_reg() == Op_RegVectMask) {
2112 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2113 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2114 } else {
2115 __ unspill(rscratch1, is64, src_offset);
2116 __ spill(rscratch1, is64, dst_offset);
2117 }
2118 }
2119 break;
2120 case rc_predicate:
2121 if (dst_lo_rc == rc_predicate) {
2122 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2123 } else if (dst_lo_rc == rc_stack) {
2124 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2125 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2126 } else {
2127 assert(false, "bad src and dst rc_class combination.");
2128 ShouldNotReachHere();
2129 }
2130 break;
2131 default:
2132 assert(false, "bad rc_class for spill");
2133 ShouldNotReachHere();
2134 }
2135 }
2136
2137 if (st) {
2138 st->print("spill ");
2139 if (src_lo_rc == rc_stack) {
2140 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2141 } else {
2142 st->print("%s -> ", Matcher::regName[src_lo]);
2143 }
2144 if (dst_lo_rc == rc_stack) {
2145 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2146 } else {
2147 st->print("%s", Matcher::regName[dst_lo]);
2148 }
2149 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2150 int vsize = 0;
2151 switch (ideal_reg()) {
2152 case Op_VecD:
2153 vsize = 64;
2154 break;
2155 case Op_VecX:
2156 vsize = 128;
2157 break;
2158 case Op_VecA:
2159 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2160 break;
2161 default:
2162 assert(false, "bad register type for spill");
2163 ShouldNotReachHere();
2164 }
2165 st->print("\t# vector spill size = %d", vsize);
2166 } else if (ideal_reg() == Op_RegVectMask) {
2167 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2168 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2169 st->print("\t# predicate spill size = %d", vsize);
2170 } else {
2171 st->print("\t# spill size = %d", is64 ? 64 : 32);
2172 }
2173 }
2174
2175 return 0;
2176
2177 }
2178
2179 #ifndef PRODUCT
2180 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2181 if (!ra_)
2182 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2183 else
2184 implementation(nullptr, ra_, false, st);
2185 }
2186 #endif
2187
2188 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2189 implementation(masm, ra_, false, nullptr);
2190 }
2191
2192 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2193 return MachNode::size(ra_);
2194 }
2195
2196 //=============================================================================
2197
2198 #ifndef PRODUCT
2199 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2200 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2201 int reg = ra_->get_reg_first(this);
2202 st->print("add %s, rsp, #%d]\t# box lock",
2203 Matcher::regName[reg], offset);
2204 }
2205 #endif
2206
2207 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2208 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2209 int reg = ra_->get_encode(this);
2210
2211 // This add will handle any 24-bit signed offset. 24 bits allows an
2212 // 8 megabyte stack frame.
2213 __ add(as_Register(reg), sp, offset);
2214 }
2215
2216 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2217 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2218 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2219
2220 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2221 return NativeInstruction::instruction_size;
2222 } else {
2223 return 2 * NativeInstruction::instruction_size;
2224 }
2225 }
2226
2227 //=============================================================================
2228
2229 #ifndef PRODUCT
2230 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2231 {
2232 st->print_cr("# MachUEPNode");
2233 if (UseCompressedClassPointers) {
2234 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2235 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2236 st->print_cr("\tcmpw rscratch1, r10");
2237 } else {
2238 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2239 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2240 st->print_cr("\tcmp rscratch1, r10");
2241 }
2242 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2243 }
2244 #endif
2245
2246 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2247 {
2248 __ ic_check(InteriorEntryAlignment);
2249 }
2250
2251 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2252 {
2253 return MachNode::size(ra_);
2254 }
2255
2256 // REQUIRED EMIT CODE
2257
2258 //=============================================================================
2259
2260 // Emit deopt handler code.
2261 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2262 {
2263 // Note that the code buffer's insts_mark is always relative to insts.
2264 // That's why we must use the macroassembler to generate a handler.
2265 address base = __ start_a_stub(size_deopt_handler());
2266 if (base == nullptr) {
2267 ciEnv::current()->record_failure("CodeCache is full");
2268 return 0; // CodeBuffer::expand failed
2269 }
2270
2271 int offset = __ offset();
2272 Label start;
2273 __ bind(start);
2274 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2275
2276 int entry_offset = __ offset();
2277 __ b(start);
2278
2279 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2280 __ end_a_stub();
2281 return entry_offset;
2282 }
2283
2284 // REQUIRED MATCHER CODE
2285
2286 //=============================================================================
2287
2288 bool Matcher::match_rule_supported(int opcode) {
2289 if (!has_match_rule(opcode))
2290 return false;
2291
2292 switch (opcode) {
2293 case Op_OnSpinWait:
2294 return VM_Version::supports_on_spin_wait();
2295 case Op_CacheWB:
2296 case Op_CacheWBPreSync:
2297 case Op_CacheWBPostSync:
2298 if (!VM_Version::supports_data_cache_line_flush()) {
2299 return false;
2300 }
2301 break;
2302 case Op_ExpandBits:
2303 case Op_CompressBits:
2304 if (!VM_Version::supports_svebitperm()) {
2305 return false;
2306 }
2307 break;
2308 case Op_FmaF:
2309 case Op_FmaD:
2310 case Op_FmaVF:
2311 case Op_FmaVD:
2312 if (!UseFMA) {
2313 return false;
2314 }
2315 break;
2316 case Op_FmaHF:
2317 // UseFMA flag also needs to be checked along with FEAT_FP16
2318 if (!UseFMA || !is_feat_fp16_supported()) {
2319 return false;
2320 }
2321 break;
2322 case Op_AddHF:
2323 case Op_SubHF:
2324 case Op_MulHF:
2325 case Op_DivHF:
2326 case Op_MinHF:
2327 case Op_MaxHF:
2328 case Op_SqrtHF:
2329 // Half-precision floating point scalar operations require FEAT_FP16
2330 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2331 // features are supported.
2332 if (!is_feat_fp16_supported()) {
2333 return false;
2334 }
2335 break;
2336 }
2337
2338 return true; // Per default match rules are supported.
2339 }
2340
2341 const RegMask* Matcher::predicate_reg_mask(void) {
2342 return &_PR_REG_mask;
2343 }
2344
2345 bool Matcher::supports_vector_calling_convention(void) {
2346 return EnableVectorSupport;
2347 }
2348
2349 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2350 assert(EnableVectorSupport, "sanity");
2351 int lo = V0_num;
2352 int hi = V0_H_num;
2353 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2354 hi = V0_K_num;
2355 }
2356 return OptoRegPair(hi, lo);
2357 }
2358
2359 // Is this branch offset short enough that a short branch can be used?
2360 //
2361 // NOTE: If the platform does not provide any short branch variants, then
2362 // this method should return false for offset 0.
2363 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2364 // The passed offset is relative to address of the branch.
2365
2366 return (-32768 <= offset && offset < 32768);
2367 }
2368
2369 // Vector width in bytes.
2370 int Matcher::vector_width_in_bytes(BasicType bt) {
2371 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2372 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2373 // Minimum 2 values in vector
2374 if (size < 2*type2aelembytes(bt)) size = 0;
2375 // But never < 4
2376 if (size < 4) size = 0;
2377 return size;
2378 }
2379
2380 // Limits on vector size (number of elements) loaded into vector.
2381 int Matcher::max_vector_size(const BasicType bt) {
2382 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2383 }
2384
2385 int Matcher::min_vector_size(const BasicType bt) {
2386 // Usually, the shortest vector length supported by AArch64 ISA and
2387 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2388 // vectors in a few special cases.
2389 int size;
2390 switch(bt) {
2391 case T_BOOLEAN:
2392 // Load/store a vector mask with only 2 elements for vector types
2393 // such as "2I/2F/2L/2D".
2394 size = 2;
2395 break;
2396 case T_BYTE:
2397 // Generate a "4B" vector, to support vector cast between "8B/16B"
2398 // and "4S/4I/4L/4F/4D".
2399 size = 4;
2400 break;
2401 case T_SHORT:
2402 // Generate a "2S" vector, to support vector cast between "4S/8S"
2403 // and "2I/2L/2F/2D".
2404 size = 2;
2405 break;
2406 default:
2407 // Limit the min vector length to 64-bit.
2408 size = 8 / type2aelembytes(bt);
2409 // The number of elements in a vector should be at least 2.
2410 size = MAX2(size, 2);
2411 }
2412
2413 int max_size = max_vector_size(bt);
2414 return MIN2(size, max_size);
2415 }
2416
2417 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2418 return Matcher::max_vector_size(bt);
2419 }
2420
2421 // Actual max scalable vector register length.
2422 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2423 return Matcher::max_vector_size(bt);
2424 }
2425
2426 // Vector ideal reg.
2427 uint Matcher::vector_ideal_reg(int len) {
2428 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2429 return Op_VecA;
2430 }
2431 switch(len) {
2432 // For 16-bit/32-bit mask vector, reuse VecD.
2433 case 2:
2434 case 4:
2435 case 8: return Op_VecD;
2436 case 16: return Op_VecX;
2437 }
2438 ShouldNotReachHere();
2439 return 0;
2440 }
2441
2442 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2443 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2444 switch (ideal_reg) {
2445 case Op_VecA: return new vecAOper();
2446 case Op_VecD: return new vecDOper();
2447 case Op_VecX: return new vecXOper();
2448 }
2449 ShouldNotReachHere();
2450 return nullptr;
2451 }
2452
2453 bool Matcher::is_reg2reg_move(MachNode* m) {
2454 return false;
2455 }
2456
2457 bool Matcher::is_generic_vector(MachOper* opnd) {
2458 return opnd->opcode() == VREG;
2459 }
2460
2461 // Return whether or not this register is ever used as an argument.
2462 // This function is used on startup to build the trampoline stubs in
2463 // generateOptoStub. Registers not mentioned will be killed by the VM
2464 // call in the trampoline, and arguments in those registers not be
2465 // available to the callee.
2466 bool Matcher::can_be_java_arg(int reg)
2467 {
2468 return
2469 reg == R0_num || reg == R0_H_num ||
2470 reg == R1_num || reg == R1_H_num ||
2471 reg == R2_num || reg == R2_H_num ||
2472 reg == R3_num || reg == R3_H_num ||
2473 reg == R4_num || reg == R4_H_num ||
2474 reg == R5_num || reg == R5_H_num ||
2475 reg == R6_num || reg == R6_H_num ||
2476 reg == R7_num || reg == R7_H_num ||
2477 reg == V0_num || reg == V0_H_num ||
2478 reg == V1_num || reg == V1_H_num ||
2479 reg == V2_num || reg == V2_H_num ||
2480 reg == V3_num || reg == V3_H_num ||
2481 reg == V4_num || reg == V4_H_num ||
2482 reg == V5_num || reg == V5_H_num ||
2483 reg == V6_num || reg == V6_H_num ||
2484 reg == V7_num || reg == V7_H_num;
2485 }
2486
2487 bool Matcher::is_spillable_arg(int reg)
2488 {
2489 return can_be_java_arg(reg);
2490 }
2491
2492 uint Matcher::int_pressure_limit()
2493 {
2494 // JDK-8183543: When taking the number of available registers as int
2495 // register pressure threshold, the jtreg test:
2496 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2497 // failed due to C2 compilation failure with
2498 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2499 //
2500 // A derived pointer is live at CallNode and then is flagged by RA
2501 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2502 // derived pointers and lastly fail to spill after reaching maximum
2503 // number of iterations. Lowering the default pressure threshold to
2504 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2505 // a high register pressure area of the code so that split_DEF can
2506 // generate DefinitionSpillCopy for the derived pointer.
2507 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2508 if (!PreserveFramePointer) {
2509 // When PreserveFramePointer is off, frame pointer is allocatable,
2510 // but different from other SOC registers, it is excluded from
2511 // fatproj's mask because its save type is No-Save. Decrease 1 to
2512 // ensure high pressure at fatproj when PreserveFramePointer is off.
2513 // See check_pressure_at_fatproj().
2514 default_int_pressure_threshold--;
2515 }
2516 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2517 }
2518
2519 uint Matcher::float_pressure_limit()
2520 {
2521 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2522 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2523 }
2524
2525 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2526 return false;
2527 }
2528
2529 const RegMask& Matcher::divI_proj_mask() {
2530 ShouldNotReachHere();
2531 return RegMask::EMPTY;
2532 }
2533
2534 // Register for MODI projection of divmodI.
2535 const RegMask& Matcher::modI_proj_mask() {
2536 ShouldNotReachHere();
2537 return RegMask::EMPTY;
2538 }
2539
2540 // Register for DIVL projection of divmodL.
2541 const RegMask& Matcher::divL_proj_mask() {
2542 ShouldNotReachHere();
2543 return RegMask::EMPTY;
2544 }
2545
2546 // Register for MODL projection of divmodL.
2547 const RegMask& Matcher::modL_proj_mask() {
2548 ShouldNotReachHere();
2549 return RegMask::EMPTY;
2550 }
2551
2552 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2553 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2554 Node* u = addp->fast_out(i);
2555 if (u->is_LoadStore()) {
2556 // On AArch64, LoadStoreNodes (i.e. compare and swap
2557 // instructions) only take register indirect as an operand, so
2558 // any attempt to use an AddPNode as an input to a LoadStoreNode
2559 // must fail.
2560 return false;
2561 }
2562 if (u->is_Mem()) {
2563 int opsize = u->as_Mem()->memory_size();
2564 assert(opsize > 0, "unexpected memory operand size");
2565 if (u->as_Mem()->memory_size() != (1<<shift)) {
2566 return false;
2567 }
2568 }
2569 }
2570 return true;
2571 }
2572
2573 // Convert BootTest condition to Assembler condition.
2574 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2575 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2576 Assembler::Condition result;
2577 switch(cond) {
2578 case BoolTest::eq:
2579 result = Assembler::EQ; break;
2580 case BoolTest::ne:
2581 result = Assembler::NE; break;
2582 case BoolTest::le:
2583 result = Assembler::LE; break;
2584 case BoolTest::ge:
2585 result = Assembler::GE; break;
2586 case BoolTest::lt:
2587 result = Assembler::LT; break;
2588 case BoolTest::gt:
2589 result = Assembler::GT; break;
2590 case BoolTest::ule:
2591 result = Assembler::LS; break;
2592 case BoolTest::uge:
2593 result = Assembler::HS; break;
2594 case BoolTest::ult:
2595 result = Assembler::LO; break;
2596 case BoolTest::ugt:
2597 result = Assembler::HI; break;
2598 case BoolTest::overflow:
2599 result = Assembler::VS; break;
2600 case BoolTest::no_overflow:
2601 result = Assembler::VC; break;
2602 default:
2603 ShouldNotReachHere();
2604 return Assembler::Condition(-1);
2605 }
2606
2607 // Check conversion
2608 if (cond & BoolTest::unsigned_compare) {
2609 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2610 } else {
2611 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2612 }
2613
2614 return result;
2615 }
2616
2617 // Binary src (Replicate con)
2618 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2619 if (n == nullptr || m == nullptr) {
2620 return false;
2621 }
2622
2623 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2624 return false;
2625 }
2626
2627 Node* imm_node = m->in(1);
2628 if (!imm_node->is_Con()) {
2629 return false;
2630 }
2631
2632 const Type* t = imm_node->bottom_type();
2633 if (!(t->isa_int() || t->isa_long())) {
2634 return false;
2635 }
2636
2637 switch (n->Opcode()) {
2638 case Op_AndV:
2639 case Op_OrV:
2640 case Op_XorV: {
2641 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2642 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2643 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2644 }
2645 case Op_AddVB:
2646 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2647 case Op_AddVS:
2648 case Op_AddVI:
2649 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2650 case Op_AddVL:
2651 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2652 default:
2653 return false;
2654 }
2655 }
2656
2657 // (XorV src (Replicate m1))
2658 // (XorVMask src (MaskAll m1))
2659 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2660 if (n != nullptr && m != nullptr) {
2661 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2662 VectorNode::is_all_ones_vector(m);
2663 }
2664 return false;
2665 }
2666
2667 // Should the matcher clone input 'm' of node 'n'?
2668 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2669 if (is_vshift_con_pattern(n, m) ||
2670 is_vector_bitwise_not_pattern(n, m) ||
2671 is_valid_sve_arith_imm_pattern(n, m) ||
2672 is_encode_and_store_pattern(n, m)) {
2673 mstack.push(m, Visit);
2674 return true;
2675 }
2676 return false;
2677 }
2678
2679 // Should the Matcher clone shifts on addressing modes, expecting them
2680 // to be subsumed into complex addressing expressions or compute them
2681 // into registers?
2682 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2683
2684 // Loads and stores with indirect memory input (e.g., volatile loads and
2685 // stores) do not subsume the input into complex addressing expressions. If
2686 // the addressing expression is input to at least one such load or store, do
2687 // not clone the addressing expression. Query needs_acquiring_load and
2688 // needs_releasing_store as a proxy for indirect memory input, as it is not
2689 // possible to directly query for indirect memory input at this stage.
2690 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2691 Node* n = m->fast_out(i);
2692 if (n->is_Load() && needs_acquiring_load(n)) {
2693 return false;
2694 }
2695 if (n->is_Store() && needs_releasing_store(n)) {
2696 return false;
2697 }
2698 }
2699
2700 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2701 return true;
2702 }
2703
2704 Node *off = m->in(AddPNode::Offset);
2705 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2706 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2707 // Are there other uses besides address expressions?
2708 !is_visited(off)) {
2709 address_visited.set(off->_idx); // Flag as address_visited
2710 mstack.push(off->in(2), Visit);
2711 Node *conv = off->in(1);
2712 if (conv->Opcode() == Op_ConvI2L &&
2713 // Are there other uses besides address expressions?
2714 !is_visited(conv)) {
2715 address_visited.set(conv->_idx); // Flag as address_visited
2716 mstack.push(conv->in(1), Pre_Visit);
2717 } else {
2718 mstack.push(conv, Pre_Visit);
2719 }
2720 address_visited.test_set(m->_idx); // Flag as address_visited
2721 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2722 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2723 return true;
2724 } else if (off->Opcode() == Op_ConvI2L &&
2725 // Are there other uses besides address expressions?
2726 !is_visited(off)) {
2727 address_visited.test_set(m->_idx); // Flag as address_visited
2728 address_visited.set(off->_idx); // Flag as address_visited
2729 mstack.push(off->in(1), Pre_Visit);
2730 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2731 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2732 return true;
2733 }
2734 return false;
2735 }
2736
2737 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2738 { \
2739 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2740 guarantee(DISP == 0, "mode not permitted for volatile"); \
2741 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2742 __ INSN(REG, as_Register(BASE)); \
2743 }
2744
2745
2746 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2747 {
2748 Address::extend scale;
2749
2750 // Hooboy, this is fugly. We need a way to communicate to the
2751 // encoder that the index needs to be sign extended, so we have to
2752 // enumerate all the cases.
2753 switch (opcode) {
2754 case INDINDEXSCALEDI2L:
2755 case INDINDEXSCALEDI2LN:
2756 case INDINDEXI2L:
2757 case INDINDEXI2LN:
2758 scale = Address::sxtw(size);
2759 break;
2760 default:
2761 scale = Address::lsl(size);
2762 }
2763
2764 if (index == -1) {
2765 return Address(base, disp);
2766 } else {
2767 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2768 return Address(base, as_Register(index), scale);
2769 }
2770 }
2771
2772
2773 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2774 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2775 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2776 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2777 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2778
2779 // Used for all non-volatile memory accesses. The use of
2780 // $mem->opcode() to discover whether this pattern uses sign-extended
2781 // offsets is something of a kludge.
2782 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2783 Register reg, int opcode,
2784 Register base, int index, int scale, int disp,
2785 int size_in_memory)
2786 {
2787 Address addr = mem2address(opcode, base, index, scale, disp);
2788 if (addr.getMode() == Address::base_plus_offset) {
2789 /* Fix up any out-of-range offsets. */
2790 assert_different_registers(rscratch1, base);
2791 assert_different_registers(rscratch1, reg);
2792 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2793 }
2794 (masm->*insn)(reg, addr);
2795 }
2796
2797 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2798 FloatRegister reg, int opcode,
2799 Register base, int index, int size, int disp,
2800 int size_in_memory)
2801 {
2802 Address::extend scale;
2803
2804 switch (opcode) {
2805 case INDINDEXSCALEDI2L:
2806 case INDINDEXSCALEDI2LN:
2807 scale = Address::sxtw(size);
2808 break;
2809 default:
2810 scale = Address::lsl(size);
2811 }
2812
2813 if (index == -1) {
2814 // Fix up any out-of-range offsets.
2815 assert_different_registers(rscratch1, base);
2816 Address addr = Address(base, disp);
2817 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2818 (masm->*insn)(reg, addr);
2819 } else {
2820 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2821 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2822 }
2823 }
2824
2825 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2826 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2827 int opcode, Register base, int index, int size, int disp)
2828 {
2829 if (index == -1) {
2830 (masm->*insn)(reg, T, Address(base, disp));
2831 } else {
2832 assert(disp == 0, "unsupported address mode");
2833 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2834 }
2835 }
2836
2837 %}
2838
2839
2840
2841 //----------ENCODING BLOCK-----------------------------------------------------
2842 // This block specifies the encoding classes used by the compiler to
2843 // output byte streams. Encoding classes are parameterized macros
2844 // used by Machine Instruction Nodes in order to generate the bit
2845 // encoding of the instruction. Operands specify their base encoding
2846 // interface with the interface keyword. There are currently
2847 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2848 // COND_INTER. REG_INTER causes an operand to generate a function
2849 // which returns its register number when queried. CONST_INTER causes
2850 // an operand to generate a function which returns the value of the
2851 // constant when queried. MEMORY_INTER causes an operand to generate
2852 // four functions which return the Base Register, the Index Register,
2853 // the Scale Value, and the Offset Value of the operand when queried.
2854 // COND_INTER causes an operand to generate six functions which return
2855 // the encoding code (ie - encoding bits for the instruction)
2856 // associated with each basic boolean condition for a conditional
2857 // instruction.
2858 //
2859 // Instructions specify two basic values for encoding. Again, a
2860 // function is available to check if the constant displacement is an
2861 // oop. They use the ins_encode keyword to specify their encoding
2862 // classes (which must be a sequence of enc_class names, and their
2863 // parameters, specified in the encoding block), and they use the
2864 // opcode keyword to specify, in order, their primary, secondary, and
2865 // tertiary opcode. Only the opcode sections which a particular
2866 // instruction needs for encoding need to be specified.
2867 encode %{
2868 // Build emit functions for each basic byte or larger field in the
2869 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2870 // from C++ code in the enc_class source block. Emit functions will
2871 // live in the main source block for now. In future, we can
2872 // generalize this by adding a syntax that specifies the sizes of
2873 // fields in an order, so that the adlc can build the emit functions
2874 // automagically
2875
2876 // catch all for unimplemented encodings
2877 enc_class enc_unimplemented %{
2878 __ unimplemented("C2 catch all");
2879 %}
2880
2881 // BEGIN Non-volatile memory access
2882
2883 // This encoding class is generated automatically from ad_encode.m4.
2884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2885 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2886 Register dst_reg = as_Register($dst$$reg);
2887 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2888 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2889 %}
2890
2891 // This encoding class is generated automatically from ad_encode.m4.
2892 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2893 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2894 Register dst_reg = as_Register($dst$$reg);
2895 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2896 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2897 %}
2898
2899 // This encoding class is generated automatically from ad_encode.m4.
2900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2901 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2902 Register dst_reg = as_Register($dst$$reg);
2903 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2904 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2905 %}
2906
2907 // This encoding class is generated automatically from ad_encode.m4.
2908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2909 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2910 Register dst_reg = as_Register($dst$$reg);
2911 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2912 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2913 %}
2914
2915 // This encoding class is generated automatically from ad_encode.m4.
2916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2917 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2918 Register dst_reg = as_Register($dst$$reg);
2919 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2920 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2921 %}
2922
2923 // This encoding class is generated automatically from ad_encode.m4.
2924 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2925 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2926 Register dst_reg = as_Register($dst$$reg);
2927 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2928 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2929 %}
2930
2931 // This encoding class is generated automatically from ad_encode.m4.
2932 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2933 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2934 Register dst_reg = as_Register($dst$$reg);
2935 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2937 %}
2938
2939 // This encoding class is generated automatically from ad_encode.m4.
2940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2941 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2942 Register dst_reg = as_Register($dst$$reg);
2943 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2950 Register dst_reg = as_Register($dst$$reg);
2951 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2958 Register dst_reg = as_Register($dst$$reg);
2959 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2960 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2961 %}
2962
2963 // This encoding class is generated automatically from ad_encode.m4.
2964 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2965 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2966 Register dst_reg = as_Register($dst$$reg);
2967 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2968 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2969 %}
2970
2971 // This encoding class is generated automatically from ad_encode.m4.
2972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2973 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2974 Register dst_reg = as_Register($dst$$reg);
2975 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2976 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2977 %}
2978
2979 // This encoding class is generated automatically from ad_encode.m4.
2980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2981 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2982 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2983 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2984 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2985 %}
2986
2987 // This encoding class is generated automatically from ad_encode.m4.
2988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2989 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2990 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2991 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2992 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2993 %}
2994
2995 // This encoding class is generated automatically from ad_encode.m4.
2996 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2997 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2998 Register src_reg = as_Register($src$$reg);
2999 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3000 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3001 %}
3002
3003 // This encoding class is generated automatically from ad_encode.m4.
3004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3005 enc_class aarch64_enc_strb0(memory1 mem) %{
3006 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3013 Register src_reg = as_Register($src$$reg);
3014 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3015 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3016 %}
3017
3018 // This encoding class is generated automatically from ad_encode.m4.
3019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3020 enc_class aarch64_enc_strh0(memory2 mem) %{
3021 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3022 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3023 %}
3024
3025 // This encoding class is generated automatically from ad_encode.m4.
3026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3027 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3028 Register src_reg = as_Register($src$$reg);
3029 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3030 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3031 %}
3032
3033 // This encoding class is generated automatically from ad_encode.m4.
3034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3035 enc_class aarch64_enc_strw0(memory4 mem) %{
3036 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3037 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3038 %}
3039
3040 // This encoding class is generated automatically from ad_encode.m4.
3041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3042 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3043 Register src_reg = as_Register($src$$reg);
3044 // we sometimes get asked to store the stack pointer into the
3045 // current thread -- we cannot do that directly on AArch64
3046 if (src_reg == r31_sp) {
3047 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3048 __ mov(rscratch2, sp);
3049 src_reg = rscratch2;
3050 }
3051 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3052 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3053 %}
3054
3055 // This encoding class is generated automatically from ad_encode.m4.
3056 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3057 enc_class aarch64_enc_str0(memory8 mem) %{
3058 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3059 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3060 %}
3061
3062 // This encoding class is generated automatically from ad_encode.m4.
3063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3064 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3065 FloatRegister src_reg = as_FloatRegister($src$$reg);
3066 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3067 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3068 %}
3069
3070 // This encoding class is generated automatically from ad_encode.m4.
3071 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3072 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3073 FloatRegister src_reg = as_FloatRegister($src$$reg);
3074 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3075 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3076 %}
3077
3078 // This encoding class is generated automatically from ad_encode.m4.
3079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3080 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3081 __ membar(Assembler::StoreStore);
3082 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3083 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3084 %}
3085
3086 // END Non-volatile memory access
3087
3088 // Vector loads and stores
3089 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3090 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3091 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3092 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3093 %}
3094
3095 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3096 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3097 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3098 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3099 %}
3100
3101 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3102 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3103 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3104 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3105 %}
3106
3107 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3108 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3109 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3110 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3111 %}
3112
3113 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3114 FloatRegister src_reg = as_FloatRegister($src$$reg);
3115 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3116 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3117 %}
3118
3119 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3120 FloatRegister src_reg = as_FloatRegister($src$$reg);
3121 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3122 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3123 %}
3124
3125 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3126 FloatRegister src_reg = as_FloatRegister($src$$reg);
3127 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3128 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3129 %}
3130
3131 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3132 FloatRegister src_reg = as_FloatRegister($src$$reg);
3133 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3134 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3135 %}
3136
3137 // volatile loads and stores
3138
3139 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3140 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3141 rscratch1, stlrb);
3142 %}
3143
3144 enc_class aarch64_enc_stlrb0(memory mem) %{
3145 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3146 rscratch1, stlrb);
3147 %}
3148
3149 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3150 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3151 rscratch1, stlrh);
3152 %}
3153
3154 enc_class aarch64_enc_stlrh0(memory mem) %{
3155 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3156 rscratch1, stlrh);
3157 %}
3158
3159 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3160 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3161 rscratch1, stlrw);
3162 %}
3163
3164 enc_class aarch64_enc_stlrw0(memory mem) %{
3165 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, stlrw);
3167 %}
3168
3169 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3170 Register dst_reg = as_Register($dst$$reg);
3171 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3172 rscratch1, ldarb);
3173 __ sxtbw(dst_reg, dst_reg);
3174 %}
3175
3176 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3177 Register dst_reg = as_Register($dst$$reg);
3178 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3179 rscratch1, ldarb);
3180 __ sxtb(dst_reg, dst_reg);
3181 %}
3182
3183 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3184 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3185 rscratch1, ldarb);
3186 %}
3187
3188 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3189 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3190 rscratch1, ldarb);
3191 %}
3192
3193 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3194 Register dst_reg = as_Register($dst$$reg);
3195 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3196 rscratch1, ldarh);
3197 __ sxthw(dst_reg, dst_reg);
3198 %}
3199
3200 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3201 Register dst_reg = as_Register($dst$$reg);
3202 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3203 rscratch1, ldarh);
3204 __ sxth(dst_reg, dst_reg);
3205 %}
3206
3207 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3208 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3209 rscratch1, ldarh);
3210 %}
3211
3212 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3213 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3214 rscratch1, ldarh);
3215 %}
3216
3217 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3218 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3219 rscratch1, ldarw);
3220 %}
3221
3222 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3223 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3224 rscratch1, ldarw);
3225 %}
3226
3227 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3228 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3229 rscratch1, ldar);
3230 %}
3231
3232 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3233 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3234 rscratch1, ldarw);
3235 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3236 %}
3237
3238 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3239 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3240 rscratch1, ldar);
3241 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3242 %}
3243
3244 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3245 Register src_reg = as_Register($src$$reg);
3246 // we sometimes get asked to store the stack pointer into the
3247 // current thread -- we cannot do that directly on AArch64
3248 if (src_reg == r31_sp) {
3249 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3250 __ mov(rscratch2, sp);
3251 src_reg = rscratch2;
3252 }
3253 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3254 rscratch1, stlr);
3255 %}
3256
3257 enc_class aarch64_enc_stlr0(memory mem) %{
3258 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3259 rscratch1, stlr);
3260 %}
3261
3262 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3263 {
3264 FloatRegister src_reg = as_FloatRegister($src$$reg);
3265 __ fmovs(rscratch2, src_reg);
3266 }
3267 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3268 rscratch1, stlrw);
3269 %}
3270
3271 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3272 {
3273 FloatRegister src_reg = as_FloatRegister($src$$reg);
3274 __ fmovd(rscratch2, src_reg);
3275 }
3276 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3277 rscratch1, stlr);
3278 %}
3279
3280 // synchronized read/update encodings
3281
3282 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3283 Register dst_reg = as_Register($dst$$reg);
3284 Register base = as_Register($mem$$base);
3285 int index = $mem$$index;
3286 int scale = $mem$$scale;
3287 int disp = $mem$$disp;
3288 if (index == -1) {
3289 if (disp != 0) {
3290 __ lea(rscratch1, Address(base, disp));
3291 __ ldaxr(dst_reg, rscratch1);
3292 } else {
3293 // TODO
3294 // should we ever get anything other than this case?
3295 __ ldaxr(dst_reg, base);
3296 }
3297 } else {
3298 Register index_reg = as_Register(index);
3299 if (disp == 0) {
3300 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3301 __ ldaxr(dst_reg, rscratch1);
3302 } else {
3303 __ lea(rscratch1, Address(base, disp));
3304 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3305 __ ldaxr(dst_reg, rscratch1);
3306 }
3307 }
3308 %}
3309
3310 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3311 Register src_reg = as_Register($src$$reg);
3312 Register base = as_Register($mem$$base);
3313 int index = $mem$$index;
3314 int scale = $mem$$scale;
3315 int disp = $mem$$disp;
3316 if (index == -1) {
3317 if (disp != 0) {
3318 __ lea(rscratch2, Address(base, disp));
3319 __ stlxr(rscratch1, src_reg, rscratch2);
3320 } else {
3321 // TODO
3322 // should we ever get anything other than this case?
3323 __ stlxr(rscratch1, src_reg, base);
3324 }
3325 } else {
3326 Register index_reg = as_Register(index);
3327 if (disp == 0) {
3328 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3329 __ stlxr(rscratch1, src_reg, rscratch2);
3330 } else {
3331 __ lea(rscratch2, Address(base, disp));
3332 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3333 __ stlxr(rscratch1, src_reg, rscratch2);
3334 }
3335 }
3336 __ cmpw(rscratch1, zr);
3337 %}
3338
3339 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3340 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3341 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3342 Assembler::xword, /*acquire*/ false, /*release*/ true,
3343 /*weak*/ false, noreg);
3344 %}
3345
3346 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3347 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3348 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3349 Assembler::word, /*acquire*/ false, /*release*/ true,
3350 /*weak*/ false, noreg);
3351 %}
3352
3353 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3354 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3355 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3356 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3357 /*weak*/ false, noreg);
3358 %}
3359
3360 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3361 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3362 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3363 Assembler::byte, /*acquire*/ false, /*release*/ true,
3364 /*weak*/ false, noreg);
3365 %}
3366
3367
3368 // The only difference between aarch64_enc_cmpxchg and
3369 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3370 // CompareAndSwap sequence to serve as a barrier on acquiring a
3371 // lock.
3372 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3373 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3374 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3375 Assembler::xword, /*acquire*/ true, /*release*/ true,
3376 /*weak*/ false, noreg);
3377 %}
3378
3379 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3380 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3381 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3382 Assembler::word, /*acquire*/ true, /*release*/ true,
3383 /*weak*/ false, noreg);
3384 %}
3385
3386 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3387 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3388 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3389 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3390 /*weak*/ false, noreg);
3391 %}
3392
3393 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3394 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3395 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3396 Assembler::byte, /*acquire*/ true, /*release*/ true,
3397 /*weak*/ false, noreg);
3398 %}
3399
3400 // auxiliary used for CompareAndSwapX to set result register
3401 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3402 Register res_reg = as_Register($res$$reg);
3403 __ cset(res_reg, Assembler::EQ);
3404 %}
3405
3406 // prefetch encodings
3407
3408 enc_class aarch64_enc_prefetchw(memory mem) %{
3409 Register base = as_Register($mem$$base);
3410 int index = $mem$$index;
3411 int scale = $mem$$scale;
3412 int disp = $mem$$disp;
3413 if (index == -1) {
3414 // Fix up any out-of-range offsets.
3415 assert_different_registers(rscratch1, base);
3416 Address addr = Address(base, disp);
3417 addr = __ legitimize_address(addr, 8, rscratch1);
3418 __ prfm(addr, PSTL1KEEP);
3419 } else {
3420 Register index_reg = as_Register(index);
3421 if (disp == 0) {
3422 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3423 } else {
3424 __ lea(rscratch1, Address(base, disp));
3425 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3426 }
3427 }
3428 %}
3429
3430 // mov encodings
3431
3432 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3433 uint32_t con = (uint32_t)$src$$constant;
3434 Register dst_reg = as_Register($dst$$reg);
3435 if (con == 0) {
3436 __ movw(dst_reg, zr);
3437 } else {
3438 __ movw(dst_reg, con);
3439 }
3440 %}
3441
3442 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3443 Register dst_reg = as_Register($dst$$reg);
3444 uint64_t con = (uint64_t)$src$$constant;
3445 if (con == 0) {
3446 __ mov(dst_reg, zr);
3447 } else {
3448 __ mov(dst_reg, con);
3449 }
3450 %}
3451
3452 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3453 Register dst_reg = as_Register($dst$$reg);
3454 address con = (address)$src$$constant;
3455 if (con == nullptr || con == (address)1) {
3456 ShouldNotReachHere();
3457 } else {
3458 relocInfo::relocType rtype = $src->constant_reloc();
3459 if (rtype == relocInfo::oop_type) {
3460 __ movoop(dst_reg, (jobject)con);
3461 } else if (rtype == relocInfo::metadata_type) {
3462 __ mov_metadata(dst_reg, (Metadata*)con);
3463 } else {
3464 assert(rtype == relocInfo::none, "unexpected reloc type");
3465 if (! __ is_valid_AArch64_address(con) ||
3466 con < (address)(uintptr_t)os::vm_page_size()) {
3467 __ mov(dst_reg, con);
3468 } else {
3469 uint64_t offset;
3470 __ adrp(dst_reg, con, offset);
3471 __ add(dst_reg, dst_reg, offset);
3472 }
3473 }
3474 }
3475 %}
3476
3477 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3478 Register dst_reg = as_Register($dst$$reg);
3479 __ mov(dst_reg, zr);
3480 %}
3481
3482 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3483 Register dst_reg = as_Register($dst$$reg);
3484 __ mov(dst_reg, (uint64_t)1);
3485 %}
3486
3487 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3488 Register dst_reg = as_Register($dst$$reg);
3489 address con = (address)$src$$constant;
3490 if (con == nullptr) {
3491 ShouldNotReachHere();
3492 } else {
3493 relocInfo::relocType rtype = $src->constant_reloc();
3494 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3495 __ set_narrow_oop(dst_reg, (jobject)con);
3496 }
3497 %}
3498
3499 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3500 Register dst_reg = as_Register($dst$$reg);
3501 __ mov(dst_reg, zr);
3502 %}
3503
3504 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3505 Register dst_reg = as_Register($dst$$reg);
3506 address con = (address)$src$$constant;
3507 if (con == nullptr) {
3508 ShouldNotReachHere();
3509 } else {
3510 relocInfo::relocType rtype = $src->constant_reloc();
3511 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3512 __ set_narrow_klass(dst_reg, (Klass *)con);
3513 }
3514 %}
3515
3516 // arithmetic encodings
3517
3518 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3519 Register dst_reg = as_Register($dst$$reg);
3520 Register src_reg = as_Register($src1$$reg);
3521 int32_t con = (int32_t)$src2$$constant;
3522 // add has primary == 0, subtract has primary == 1
3523 if ($primary) { con = -con; }
3524 if (con < 0) {
3525 __ subw(dst_reg, src_reg, -con);
3526 } else {
3527 __ addw(dst_reg, src_reg, con);
3528 }
3529 %}
3530
3531 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3532 Register dst_reg = as_Register($dst$$reg);
3533 Register src_reg = as_Register($src1$$reg);
3534 int32_t con = (int32_t)$src2$$constant;
3535 // add has primary == 0, subtract has primary == 1
3536 if ($primary) { con = -con; }
3537 if (con < 0) {
3538 __ sub(dst_reg, src_reg, -con);
3539 } else {
3540 __ add(dst_reg, src_reg, con);
3541 }
3542 %}
3543
3544 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3545 Register dst_reg = as_Register($dst$$reg);
3546 Register src1_reg = as_Register($src1$$reg);
3547 Register src2_reg = as_Register($src2$$reg);
3548 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3549 %}
3550
3551 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3552 Register dst_reg = as_Register($dst$$reg);
3553 Register src1_reg = as_Register($src1$$reg);
3554 Register src2_reg = as_Register($src2$$reg);
3555 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3556 %}
3557
3558 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3559 Register dst_reg = as_Register($dst$$reg);
3560 Register src1_reg = as_Register($src1$$reg);
3561 Register src2_reg = as_Register($src2$$reg);
3562 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3563 %}
3564
3565 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3566 Register dst_reg = as_Register($dst$$reg);
3567 Register src1_reg = as_Register($src1$$reg);
3568 Register src2_reg = as_Register($src2$$reg);
3569 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3570 %}
3571
3572 // compare instruction encodings
3573
3574 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3575 Register reg1 = as_Register($src1$$reg);
3576 Register reg2 = as_Register($src2$$reg);
3577 __ cmpw(reg1, reg2);
3578 %}
3579
3580 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3581 Register reg = as_Register($src1$$reg);
3582 int32_t val = $src2$$constant;
3583 if (val >= 0) {
3584 __ subsw(zr, reg, val);
3585 } else {
3586 __ addsw(zr, reg, -val);
3587 }
3588 %}
3589
3590 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3591 Register reg1 = as_Register($src1$$reg);
3592 uint32_t val = (uint32_t)$src2$$constant;
3593 __ movw(rscratch1, val);
3594 __ cmpw(reg1, rscratch1);
3595 %}
3596
3597 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3598 Register reg1 = as_Register($src1$$reg);
3599 Register reg2 = as_Register($src2$$reg);
3600 __ cmp(reg1, reg2);
3601 %}
3602
3603 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3604 Register reg = as_Register($src1$$reg);
3605 int64_t val = $src2$$constant;
3606 if (val >= 0) {
3607 __ subs(zr, reg, val);
3608 } else if (val != -val) {
3609 __ adds(zr, reg, -val);
3610 } else {
3611 // aargh, Long.MIN_VALUE is a special case
3612 __ orr(rscratch1, zr, (uint64_t)val);
3613 __ subs(zr, reg, rscratch1);
3614 }
3615 %}
3616
3617 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3618 Register reg1 = as_Register($src1$$reg);
3619 uint64_t val = (uint64_t)$src2$$constant;
3620 __ mov(rscratch1, val);
3621 __ cmp(reg1, rscratch1);
3622 %}
3623
3624 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3625 Register reg1 = as_Register($src1$$reg);
3626 Register reg2 = as_Register($src2$$reg);
3627 __ cmp(reg1, reg2);
3628 %}
3629
3630 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3631 Register reg1 = as_Register($src1$$reg);
3632 Register reg2 = as_Register($src2$$reg);
3633 __ cmpw(reg1, reg2);
3634 %}
3635
3636 enc_class aarch64_enc_testp(iRegP src) %{
3637 Register reg = as_Register($src$$reg);
3638 __ cmp(reg, zr);
3639 %}
3640
3641 enc_class aarch64_enc_testn(iRegN src) %{
3642 Register reg = as_Register($src$$reg);
3643 __ cmpw(reg, zr);
3644 %}
3645
3646 enc_class aarch64_enc_b(label lbl) %{
3647 Label *L = $lbl$$label;
3648 __ b(*L);
3649 %}
3650
3651 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3652 Label *L = $lbl$$label;
3653 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3654 %}
3655
3656 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3657 Label *L = $lbl$$label;
3658 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3659 %}
3660
3661 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3662 %{
3663 Register sub_reg = as_Register($sub$$reg);
3664 Register super_reg = as_Register($super$$reg);
3665 Register temp_reg = as_Register($temp$$reg);
3666 Register result_reg = as_Register($result$$reg);
3667
3668 Label miss;
3669 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3670 nullptr, &miss,
3671 /*set_cond_codes:*/ true);
3672 if ($primary) {
3673 __ mov(result_reg, zr);
3674 }
3675 __ bind(miss);
3676 %}
3677
3678 enc_class aarch64_enc_java_static_call(method meth) %{
3679 address addr = (address)$meth$$method;
3680 address call;
3681 if (!_method) {
3682 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3683 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3684 if (call == nullptr) {
3685 ciEnv::current()->record_failure("CodeCache is full");
3686 return;
3687 }
3688 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3689 // The NOP here is purely to ensure that eliding a call to
3690 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3691 __ nop();
3692 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3693 } else {
3694 int method_index = resolved_method_index(masm);
3695 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3696 : static_call_Relocation::spec(method_index);
3697 call = __ trampoline_call(Address(addr, rspec));
3698 if (call == nullptr) {
3699 ciEnv::current()->record_failure("CodeCache is full");
3700 return;
3701 }
3702 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3703 // Calls of the same statically bound method can share
3704 // a stub to the interpreter.
3705 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3706 } else {
3707 // Emit stub for static call
3708 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3709 if (stub == nullptr) {
3710 ciEnv::current()->record_failure("CodeCache is full");
3711 return;
3712 }
3713 }
3714 }
3715
3716 __ post_call_nop();
3717
3718 // Only non uncommon_trap calls need to reinitialize ptrue.
3719 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3720 __ reinitialize_ptrue();
3721 }
3722 %}
3723
3724 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3725 int method_index = resolved_method_index(masm);
3726 address call = __ ic_call((address)$meth$$method, method_index);
3727 if (call == nullptr) {
3728 ciEnv::current()->record_failure("CodeCache is full");
3729 return;
3730 }
3731 __ post_call_nop();
3732 if (Compile::current()->max_vector_size() > 0) {
3733 __ reinitialize_ptrue();
3734 }
3735 %}
3736
3737 enc_class aarch64_enc_call_epilog() %{
3738 if (VerifyStackAtCalls) {
3739 // Check that stack depth is unchanged: find majik cookie on stack
3740 __ call_Unimplemented();
3741 }
3742 %}
3743
3744 enc_class aarch64_enc_java_to_runtime(method meth) %{
3745 // some calls to generated routines (arraycopy code) are scheduled
3746 // by C2 as runtime calls. if so we can call them using a br (they
3747 // will be in a reachable segment) otherwise we have to use a blr
3748 // which loads the absolute address into a register.
3749 address entry = (address)$meth$$method;
3750 CodeBlob *cb = CodeCache::find_blob(entry);
3751 if (cb) {
3752 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3753 if (call == nullptr) {
3754 ciEnv::current()->record_failure("CodeCache is full");
3755 return;
3756 }
3757 __ post_call_nop();
3758 } else {
3759 Label retaddr;
3760 // Make the anchor frame walkable
3761 __ adr(rscratch2, retaddr);
3762 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3763 __ lea(rscratch1, RuntimeAddress(entry));
3764 __ blr(rscratch1);
3765 __ bind(retaddr);
3766 __ post_call_nop();
3767 }
3768 if (Compile::current()->max_vector_size() > 0) {
3769 __ reinitialize_ptrue();
3770 }
3771 %}
3772
3773 enc_class aarch64_enc_rethrow() %{
3774 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3775 %}
3776
3777 enc_class aarch64_enc_ret() %{
3778 #ifdef ASSERT
3779 if (Compile::current()->max_vector_size() > 0) {
3780 __ verify_ptrue();
3781 }
3782 #endif
3783 __ ret(lr);
3784 %}
3785
3786 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3787 Register target_reg = as_Register($jump_target$$reg);
3788 __ br(target_reg);
3789 %}
3790
3791 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3792 Register target_reg = as_Register($jump_target$$reg);
3793 // exception oop should be in r0
3794 // ret addr has been popped into lr
3795 // callee expects it in r3
3796 __ mov(r3, lr);
3797 __ br(target_reg);
3798 %}
3799
3800 %}
3801
3802 //----------FRAME--------------------------------------------------------------
3803 // Definition of frame structure and management information.
3804 //
3805 // S T A C K L A Y O U T Allocators stack-slot number
3806 // | (to get allocators register number
3807 // G Owned by | | v add OptoReg::stack0())
3808 // r CALLER | |
3809 // o | +--------+ pad to even-align allocators stack-slot
3810 // w V | pad0 | numbers; owned by CALLER
3811 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3812 // h ^ | in | 5
3813 // | | args | 4 Holes in incoming args owned by SELF
3814 // | | | | 3
3815 // | | +--------+
3816 // V | | old out| Empty on Intel, window on Sparc
3817 // | old |preserve| Must be even aligned.
3818 // | SP-+--------+----> Matcher::_old_SP, even aligned
3819 // | | in | 3 area for Intel ret address
3820 // Owned by |preserve| Empty on Sparc.
3821 // SELF +--------+
3822 // | | pad2 | 2 pad to align old SP
3823 // | +--------+ 1
3824 // | | locks | 0
3825 // | +--------+----> OptoReg::stack0(), even aligned
3826 // | | pad1 | 11 pad to align new SP
3827 // | +--------+
3828 // | | | 10
3829 // | | spills | 9 spills
3830 // V | | 8 (pad0 slot for callee)
3831 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3832 // ^ | out | 7
3833 // | | args | 6 Holes in outgoing args owned by CALLEE
3834 // Owned by +--------+
3835 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3836 // | new |preserve| Must be even-aligned.
3837 // | SP-+--------+----> Matcher::_new_SP, even aligned
3838 // | | |
3839 //
3840 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3841 // known from SELF's arguments and the Java calling convention.
3842 // Region 6-7 is determined per call site.
3843 // Note 2: If the calling convention leaves holes in the incoming argument
3844 // area, those holes are owned by SELF. Holes in the outgoing area
3845 // are owned by the CALLEE. Holes should not be necessary in the
3846 // incoming area, as the Java calling convention is completely under
3847 // the control of the AD file. Doubles can be sorted and packed to
3848 // avoid holes. Holes in the outgoing arguments may be necessary for
3849 // varargs C calling conventions.
3850 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3851 // even aligned with pad0 as needed.
3852 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3853 // (the latter is true on Intel but is it false on AArch64?)
3854 // region 6-11 is even aligned; it may be padded out more so that
3855 // the region from SP to FP meets the minimum stack alignment.
3856 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3857 // alignment. Region 11, pad1, may be dynamically extended so that
3858 // SP meets the minimum alignment.
3859
3860 frame %{
3861 // These three registers define part of the calling convention
3862 // between compiled code and the interpreter.
3863
3864 // Inline Cache Register or Method for I2C.
3865 inline_cache_reg(R12);
3866
3867 // Number of stack slots consumed by locking an object
3868 sync_stack_slots(2);
3869
3870 // Compiled code's Frame Pointer
3871 frame_pointer(R31);
3872
3873 // Interpreter stores its frame pointer in a register which is
3874 // stored to the stack by I2CAdaptors.
3875 // I2CAdaptors convert from interpreted java to compiled java.
3876 interpreter_frame_pointer(R29);
3877
3878 // Stack alignment requirement
3879 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3880
3881 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3882 // for calls to C. Supports the var-args backing area for register parms.
3883 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3884
3885 // The after-PROLOG location of the return address. Location of
3886 // return address specifies a type (REG or STACK) and a number
3887 // representing the register number (i.e. - use a register name) or
3888 // stack slot.
3889 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3890 // Otherwise, it is above the locks and verification slot and alignment word
3891 // TODO this may well be correct but need to check why that - 2 is there
3892 // ppc port uses 0 but we definitely need to allow for fixed_slots
3893 // which folds in the space used for monitors
3894 return_addr(STACK - 2 +
3895 align_up((Compile::current()->in_preserve_stack_slots() +
3896 Compile::current()->fixed_slots()),
3897 stack_alignment_in_slots()));
3898
3899 // Location of compiled Java return values. Same as C for now.
3900 return_value
3901 %{
3902 // TODO do we allow ideal_reg == Op_RegN???
3903 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3904 "only return normal values");
3905
3906 static const int lo[Op_RegL + 1] = { // enum name
3907 0, // Op_Node
3908 0, // Op_Set
3909 R0_num, // Op_RegN
3910 R0_num, // Op_RegI
3911 R0_num, // Op_RegP
3912 V0_num, // Op_RegF
3913 V0_num, // Op_RegD
3914 R0_num // Op_RegL
3915 };
3916
3917 static const int hi[Op_RegL + 1] = { // enum name
3918 0, // Op_Node
3919 0, // Op_Set
3920 OptoReg::Bad, // Op_RegN
3921 OptoReg::Bad, // Op_RegI
3922 R0_H_num, // Op_RegP
3923 OptoReg::Bad, // Op_RegF
3924 V0_H_num, // Op_RegD
3925 R0_H_num // Op_RegL
3926 };
3927
3928 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3929 %}
3930 %}
3931
3932 //----------ATTRIBUTES---------------------------------------------------------
3933 //----------Operand Attributes-------------------------------------------------
3934 op_attrib op_cost(1); // Required cost attribute
3935
3936 //----------Instruction Attributes---------------------------------------------
3937 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3938 ins_attrib ins_size(32); // Required size attribute (in bits)
3939 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3940 // a non-matching short branch variant
3941 // of some long branch?
3942 ins_attrib ins_alignment(4); // Required alignment attribute (must
3943 // be a power of 2) specifies the
3944 // alignment that some part of the
3945 // instruction (not necessarily the
3946 // start) requires. If > 1, a
3947 // compute_padding() function must be
3948 // provided for the instruction
3949
3950 // Whether this node is expanded during code emission into a sequence of
3951 // instructions and the first instruction can perform an implicit null check.
3952 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3953
3954 //----------OPERANDS-----------------------------------------------------------
3955 // Operand definitions must precede instruction definitions for correct parsing
3956 // in the ADLC because operands constitute user defined types which are used in
3957 // instruction definitions.
3958
3959 //----------Simple Operands----------------------------------------------------
3960
3961 // Integer operands 32 bit
3962 // 32 bit immediate
3963 operand immI()
3964 %{
3965 match(ConI);
3966
3967 op_cost(0);
3968 format %{ %}
3969 interface(CONST_INTER);
3970 %}
3971
3972 // 32 bit zero
3973 operand immI0()
3974 %{
3975 predicate(n->get_int() == 0);
3976 match(ConI);
3977
3978 op_cost(0);
3979 format %{ %}
3980 interface(CONST_INTER);
3981 %}
3982
3983 // 32 bit unit increment
3984 operand immI_1()
3985 %{
3986 predicate(n->get_int() == 1);
3987 match(ConI);
3988
3989 op_cost(0);
3990 format %{ %}
3991 interface(CONST_INTER);
3992 %}
3993
3994 // 32 bit unit decrement
3995 operand immI_M1()
3996 %{
3997 predicate(n->get_int() == -1);
3998 match(ConI);
3999
4000 op_cost(0);
4001 format %{ %}
4002 interface(CONST_INTER);
4003 %}
4004
4005 // Shift values for add/sub extension shift
4006 operand immIExt()
4007 %{
4008 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4009 match(ConI);
4010
4011 op_cost(0);
4012 format %{ %}
4013 interface(CONST_INTER);
4014 %}
4015
4016 operand immI_gt_1()
4017 %{
4018 predicate(n->get_int() > 1);
4019 match(ConI);
4020
4021 op_cost(0);
4022 format %{ %}
4023 interface(CONST_INTER);
4024 %}
4025
4026 operand immI_le_4()
4027 %{
4028 predicate(n->get_int() <= 4);
4029 match(ConI);
4030
4031 op_cost(0);
4032 format %{ %}
4033 interface(CONST_INTER);
4034 %}
4035
4036 operand immI_16()
4037 %{
4038 predicate(n->get_int() == 16);
4039 match(ConI);
4040
4041 op_cost(0);
4042 format %{ %}
4043 interface(CONST_INTER);
4044 %}
4045
4046 operand immI_24()
4047 %{
4048 predicate(n->get_int() == 24);
4049 match(ConI);
4050
4051 op_cost(0);
4052 format %{ %}
4053 interface(CONST_INTER);
4054 %}
4055
4056 operand immI_32()
4057 %{
4058 predicate(n->get_int() == 32);
4059 match(ConI);
4060
4061 op_cost(0);
4062 format %{ %}
4063 interface(CONST_INTER);
4064 %}
4065
4066 operand immI_48()
4067 %{
4068 predicate(n->get_int() == 48);
4069 match(ConI);
4070
4071 op_cost(0);
4072 format %{ %}
4073 interface(CONST_INTER);
4074 %}
4075
4076 operand immI_56()
4077 %{
4078 predicate(n->get_int() == 56);
4079 match(ConI);
4080
4081 op_cost(0);
4082 format %{ %}
4083 interface(CONST_INTER);
4084 %}
4085
4086 operand immI_255()
4087 %{
4088 predicate(n->get_int() == 255);
4089 match(ConI);
4090
4091 op_cost(0);
4092 format %{ %}
4093 interface(CONST_INTER);
4094 %}
4095
4096 operand immI_65535()
4097 %{
4098 predicate(n->get_int() == 65535);
4099 match(ConI);
4100
4101 op_cost(0);
4102 format %{ %}
4103 interface(CONST_INTER);
4104 %}
4105
4106 operand immI_positive()
4107 %{
4108 predicate(n->get_int() > 0);
4109 match(ConI);
4110
4111 op_cost(0);
4112 format %{ %}
4113 interface(CONST_INTER);
4114 %}
4115
4116 // BoolTest condition for signed compare
4117 operand immI_cmp_cond()
4118 %{
4119 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4120 match(ConI);
4121
4122 op_cost(0);
4123 format %{ %}
4124 interface(CONST_INTER);
4125 %}
4126
4127 // BoolTest condition for unsigned compare
4128 operand immI_cmpU_cond()
4129 %{
4130 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4131 match(ConI);
4132
4133 op_cost(0);
4134 format %{ %}
4135 interface(CONST_INTER);
4136 %}
4137
4138 operand immL_255()
4139 %{
4140 predicate(n->get_long() == 255L);
4141 match(ConL);
4142
4143 op_cost(0);
4144 format %{ %}
4145 interface(CONST_INTER);
4146 %}
4147
4148 operand immL_65535()
4149 %{
4150 predicate(n->get_long() == 65535L);
4151 match(ConL);
4152
4153 op_cost(0);
4154 format %{ %}
4155 interface(CONST_INTER);
4156 %}
4157
4158 operand immL_4294967295()
4159 %{
4160 predicate(n->get_long() == 4294967295L);
4161 match(ConL);
4162
4163 op_cost(0);
4164 format %{ %}
4165 interface(CONST_INTER);
4166 %}
4167
4168 operand immL_bitmask()
4169 %{
4170 predicate((n->get_long() != 0)
4171 && ((n->get_long() & 0xc000000000000000l) == 0)
4172 && is_power_of_2(n->get_long() + 1));
4173 match(ConL);
4174
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 operand immI_bitmask()
4181 %{
4182 predicate((n->get_int() != 0)
4183 && ((n->get_int() & 0xc0000000) == 0)
4184 && is_power_of_2(n->get_int() + 1));
4185 match(ConI);
4186
4187 op_cost(0);
4188 format %{ %}
4189 interface(CONST_INTER);
4190 %}
4191
4192 operand immL_positive_bitmaskI()
4193 %{
4194 predicate((n->get_long() != 0)
4195 && ((julong)n->get_long() < 0x80000000ULL)
4196 && is_power_of_2(n->get_long() + 1));
4197 match(ConL);
4198
4199 op_cost(0);
4200 format %{ %}
4201 interface(CONST_INTER);
4202 %}
4203
4204 // Scale values for scaled offset addressing modes (up to long but not quad)
4205 operand immIScale()
4206 %{
4207 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4208 match(ConI);
4209
4210 op_cost(0);
4211 format %{ %}
4212 interface(CONST_INTER);
4213 %}
4214
4215 // 5 bit signed integer
4216 operand immI5()
4217 %{
4218 predicate(Assembler::is_simm(n->get_int(), 5));
4219 match(ConI);
4220
4221 op_cost(0);
4222 format %{ %}
4223 interface(CONST_INTER);
4224 %}
4225
4226 // 7 bit unsigned integer
4227 operand immIU7()
4228 %{
4229 predicate(Assembler::is_uimm(n->get_int(), 7));
4230 match(ConI);
4231
4232 op_cost(0);
4233 format %{ %}
4234 interface(CONST_INTER);
4235 %}
4236
4237 // Offset for scaled or unscaled immediate loads and stores
4238 operand immIOffset()
4239 %{
4240 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4241 match(ConI);
4242
4243 op_cost(0);
4244 format %{ %}
4245 interface(CONST_INTER);
4246 %}
4247
4248 operand immIOffset1()
4249 %{
4250 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4251 match(ConI);
4252
4253 op_cost(0);
4254 format %{ %}
4255 interface(CONST_INTER);
4256 %}
4257
4258 operand immIOffset2()
4259 %{
4260 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4261 match(ConI);
4262
4263 op_cost(0);
4264 format %{ %}
4265 interface(CONST_INTER);
4266 %}
4267
4268 operand immIOffset4()
4269 %{
4270 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4271 match(ConI);
4272
4273 op_cost(0);
4274 format %{ %}
4275 interface(CONST_INTER);
4276 %}
4277
4278 operand immIOffset8()
4279 %{
4280 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4281 match(ConI);
4282
4283 op_cost(0);
4284 format %{ %}
4285 interface(CONST_INTER);
4286 %}
4287
4288 operand immIOffset16()
4289 %{
4290 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4291 match(ConI);
4292
4293 op_cost(0);
4294 format %{ %}
4295 interface(CONST_INTER);
4296 %}
4297
4298 operand immLOffset()
4299 %{
4300 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4301 match(ConL);
4302
4303 op_cost(0);
4304 format %{ %}
4305 interface(CONST_INTER);
4306 %}
4307
4308 operand immLoffset1()
4309 %{
4310 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4311 match(ConL);
4312
4313 op_cost(0);
4314 format %{ %}
4315 interface(CONST_INTER);
4316 %}
4317
4318 operand immLoffset2()
4319 %{
4320 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4321 match(ConL);
4322
4323 op_cost(0);
4324 format %{ %}
4325 interface(CONST_INTER);
4326 %}
4327
4328 operand immLoffset4()
4329 %{
4330 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4331 match(ConL);
4332
4333 op_cost(0);
4334 format %{ %}
4335 interface(CONST_INTER);
4336 %}
4337
4338 operand immLoffset8()
4339 %{
4340 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4341 match(ConL);
4342
4343 op_cost(0);
4344 format %{ %}
4345 interface(CONST_INTER);
4346 %}
4347
4348 operand immLoffset16()
4349 %{
4350 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4351 match(ConL);
4352
4353 op_cost(0);
4354 format %{ %}
4355 interface(CONST_INTER);
4356 %}
4357
4358 // 5 bit signed long integer
4359 operand immL5()
4360 %{
4361 predicate(Assembler::is_simm(n->get_long(), 5));
4362 match(ConL);
4363
4364 op_cost(0);
4365 format %{ %}
4366 interface(CONST_INTER);
4367 %}
4368
4369 // 7 bit unsigned long integer
4370 operand immLU7()
4371 %{
4372 predicate(Assembler::is_uimm(n->get_long(), 7));
4373 match(ConL);
4374
4375 op_cost(0);
4376 format %{ %}
4377 interface(CONST_INTER);
4378 %}
4379
4380 // 8 bit signed value.
4381 operand immI8()
4382 %{
4383 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4384 match(ConI);
4385
4386 op_cost(0);
4387 format %{ %}
4388 interface(CONST_INTER);
4389 %}
4390
4391 // 8 bit signed value (simm8), or #simm8 LSL 8.
4392 operand immIDupV()
4393 %{
4394 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4395 match(ConI);
4396
4397 op_cost(0);
4398 format %{ %}
4399 interface(CONST_INTER);
4400 %}
4401
4402 // 8 bit signed value (simm8), or #simm8 LSL 8.
4403 operand immLDupV()
4404 %{
4405 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4406 match(ConL);
4407
4408 op_cost(0);
4409 format %{ %}
4410 interface(CONST_INTER);
4411 %}
4412
4413 // 8 bit signed value (simm8), or #simm8 LSL 8.
4414 operand immHDupV()
4415 %{
4416 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4417 match(ConH);
4418
4419 op_cost(0);
4420 format %{ %}
4421 interface(CONST_INTER);
4422 %}
4423
4424 // 8 bit integer valid for vector add sub immediate
4425 operand immBAddSubV()
4426 %{
4427 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4428 match(ConI);
4429
4430 op_cost(0);
4431 format %{ %}
4432 interface(CONST_INTER);
4433 %}
4434
4435 // 32 bit integer valid for add sub immediate
4436 operand immIAddSub()
4437 %{
4438 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4439 match(ConI);
4440 op_cost(0);
4441 format %{ %}
4442 interface(CONST_INTER);
4443 %}
4444
4445 // 32 bit integer valid for vector add sub immediate
4446 operand immIAddSubV()
4447 %{
4448 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4449 match(ConI);
4450
4451 op_cost(0);
4452 format %{ %}
4453 interface(CONST_INTER);
4454 %}
4455
4456 // 32 bit unsigned integer valid for logical immediate
4457
4458 operand immBLog()
4459 %{
4460 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4461 match(ConI);
4462
4463 op_cost(0);
4464 format %{ %}
4465 interface(CONST_INTER);
4466 %}
4467
4468 operand immSLog()
4469 %{
4470 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4471 match(ConI);
4472
4473 op_cost(0);
4474 format %{ %}
4475 interface(CONST_INTER);
4476 %}
4477
4478 operand immILog()
4479 %{
4480 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4481 match(ConI);
4482
4483 op_cost(0);
4484 format %{ %}
4485 interface(CONST_INTER);
4486 %}
4487
4488 // Integer operands 64 bit
4489 // 64 bit immediate
4490 operand immL()
4491 %{
4492 match(ConL);
4493
4494 op_cost(0);
4495 format %{ %}
4496 interface(CONST_INTER);
4497 %}
4498
4499 // 64 bit zero
4500 operand immL0()
4501 %{
4502 predicate(n->get_long() == 0);
4503 match(ConL);
4504
4505 op_cost(0);
4506 format %{ %}
4507 interface(CONST_INTER);
4508 %}
4509
4510 // 64 bit unit decrement
4511 operand immL_M1()
4512 %{
4513 predicate(n->get_long() == -1);
4514 match(ConL);
4515
4516 op_cost(0);
4517 format %{ %}
4518 interface(CONST_INTER);
4519 %}
4520
4521 // 64 bit integer valid for add sub immediate
4522 operand immLAddSub()
4523 %{
4524 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4525 match(ConL);
4526 op_cost(0);
4527 format %{ %}
4528 interface(CONST_INTER);
4529 %}
4530
4531 // 64 bit integer valid for addv subv immediate
4532 operand immLAddSubV()
4533 %{
4534 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4535 match(ConL);
4536
4537 op_cost(0);
4538 format %{ %}
4539 interface(CONST_INTER);
4540 %}
4541
4542 // 64 bit integer valid for logical immediate
4543 operand immLLog()
4544 %{
4545 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4546 match(ConL);
4547 op_cost(0);
4548 format %{ %}
4549 interface(CONST_INTER);
4550 %}
4551
4552 // Long Immediate: low 32-bit mask
4553 operand immL_32bits()
4554 %{
4555 predicate(n->get_long() == 0xFFFFFFFFL);
4556 match(ConL);
4557 op_cost(0);
4558 format %{ %}
4559 interface(CONST_INTER);
4560 %}
4561
4562 // Pointer operands
4563 // Pointer Immediate
4564 operand immP()
4565 %{
4566 match(ConP);
4567
4568 op_cost(0);
4569 format %{ %}
4570 interface(CONST_INTER);
4571 %}
4572
4573 // nullptr Pointer Immediate
4574 operand immP0()
4575 %{
4576 predicate(n->get_ptr() == 0);
4577 match(ConP);
4578
4579 op_cost(0);
4580 format %{ %}
4581 interface(CONST_INTER);
4582 %}
4583
4584 // Pointer Immediate One
4585 // this is used in object initialization (initial object header)
4586 operand immP_1()
4587 %{
4588 predicate(n->get_ptr() == 1);
4589 match(ConP);
4590
4591 op_cost(0);
4592 format %{ %}
4593 interface(CONST_INTER);
4594 %}
4595
4596 // Float and Double operands
4597 // Double Immediate
4598 operand immD()
4599 %{
4600 match(ConD);
4601 op_cost(0);
4602 format %{ %}
4603 interface(CONST_INTER);
4604 %}
4605
4606 // Double Immediate: +0.0d
4607 operand immD0()
4608 %{
4609 predicate(jlong_cast(n->getd()) == 0);
4610 match(ConD);
4611
4612 op_cost(0);
4613 format %{ %}
4614 interface(CONST_INTER);
4615 %}
4616
4617 // constant 'double +0.0'.
4618 operand immDPacked()
4619 %{
4620 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4621 match(ConD);
4622 op_cost(0);
4623 format %{ %}
4624 interface(CONST_INTER);
4625 %}
4626
4627 // Float Immediate
4628 operand immF()
4629 %{
4630 match(ConF);
4631 op_cost(0);
4632 format %{ %}
4633 interface(CONST_INTER);
4634 %}
4635
4636 // Float Immediate: +0.0f.
4637 operand immF0()
4638 %{
4639 predicate(jint_cast(n->getf()) == 0);
4640 match(ConF);
4641
4642 op_cost(0);
4643 format %{ %}
4644 interface(CONST_INTER);
4645 %}
4646
4647 // Half Float (FP16) Immediate
4648 operand immH()
4649 %{
4650 match(ConH);
4651 op_cost(0);
4652 format %{ %}
4653 interface(CONST_INTER);
4654 %}
4655
4656 //
4657 operand immFPacked()
4658 %{
4659 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4660 match(ConF);
4661 op_cost(0);
4662 format %{ %}
4663 interface(CONST_INTER);
4664 %}
4665
4666 // Narrow pointer operands
4667 // Narrow Pointer Immediate
4668 operand immN()
4669 %{
4670 match(ConN);
4671
4672 op_cost(0);
4673 format %{ %}
4674 interface(CONST_INTER);
4675 %}
4676
4677 // Narrow nullptr Pointer Immediate
4678 operand immN0()
4679 %{
4680 predicate(n->get_narrowcon() == 0);
4681 match(ConN);
4682
4683 op_cost(0);
4684 format %{ %}
4685 interface(CONST_INTER);
4686 %}
4687
4688 operand immNKlass()
4689 %{
4690 match(ConNKlass);
4691
4692 op_cost(0);
4693 format %{ %}
4694 interface(CONST_INTER);
4695 %}
4696
4697 // Integer 32 bit Register Operands
4698 // Integer 32 bitRegister (excludes SP)
4699 operand iRegI()
4700 %{
4701 constraint(ALLOC_IN_RC(any_reg32));
4702 match(RegI);
4703 match(iRegINoSp);
4704 op_cost(0);
4705 format %{ %}
4706 interface(REG_INTER);
4707 %}
4708
4709 // Integer 32 bit Register not Special
4710 operand iRegINoSp()
4711 %{
4712 constraint(ALLOC_IN_RC(no_special_reg32));
4713 match(RegI);
4714 op_cost(0);
4715 format %{ %}
4716 interface(REG_INTER);
4717 %}
4718
4719 // Integer 64 bit Register Operands
4720 // Integer 64 bit Register (includes SP)
4721 operand iRegL()
4722 %{
4723 constraint(ALLOC_IN_RC(any_reg));
4724 match(RegL);
4725 match(iRegLNoSp);
4726 op_cost(0);
4727 format %{ %}
4728 interface(REG_INTER);
4729 %}
4730
4731 // Integer 64 bit Register not Special
4732 operand iRegLNoSp()
4733 %{
4734 constraint(ALLOC_IN_RC(no_special_reg));
4735 match(RegL);
4736 match(iRegL_R0);
4737 format %{ %}
4738 interface(REG_INTER);
4739 %}
4740
4741 // Pointer Register Operands
4742 // Pointer Register
4743 operand iRegP()
4744 %{
4745 constraint(ALLOC_IN_RC(ptr_reg));
4746 match(RegP);
4747 match(iRegPNoSp);
4748 match(iRegP_R0);
4749 //match(iRegP_R2);
4750 //match(iRegP_R4);
4751 match(iRegP_R5);
4752 match(thread_RegP);
4753 op_cost(0);
4754 format %{ %}
4755 interface(REG_INTER);
4756 %}
4757
4758 // Pointer 64 bit Register not Special
4759 operand iRegPNoSp()
4760 %{
4761 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4762 match(RegP);
4763 // match(iRegP);
4764 // match(iRegP_R0);
4765 // match(iRegP_R2);
4766 // match(iRegP_R4);
4767 // match(iRegP_R5);
4768 // match(thread_RegP);
4769 op_cost(0);
4770 format %{ %}
4771 interface(REG_INTER);
4772 %}
4773
4774 // This operand is not allowed to use rfp even if
4775 // rfp is not used to hold the frame pointer.
4776 operand iRegPNoSpNoRfp()
4777 %{
4778 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4779 match(RegP);
4780 match(iRegPNoSp);
4781 op_cost(0);
4782 format %{ %}
4783 interface(REG_INTER);
4784 %}
4785
4786 // Pointer 64 bit Register R0 only
4787 operand iRegP_R0()
4788 %{
4789 constraint(ALLOC_IN_RC(r0_reg));
4790 match(RegP);
4791 // match(iRegP);
4792 match(iRegPNoSp);
4793 op_cost(0);
4794 format %{ %}
4795 interface(REG_INTER);
4796 %}
4797
4798 // Pointer 64 bit Register R1 only
4799 operand iRegP_R1()
4800 %{
4801 constraint(ALLOC_IN_RC(r1_reg));
4802 match(RegP);
4803 // match(iRegP);
4804 match(iRegPNoSp);
4805 op_cost(0);
4806 format %{ %}
4807 interface(REG_INTER);
4808 %}
4809
4810 // Pointer 64 bit Register R2 only
4811 operand iRegP_R2()
4812 %{
4813 constraint(ALLOC_IN_RC(r2_reg));
4814 match(RegP);
4815 // match(iRegP);
4816 match(iRegPNoSp);
4817 op_cost(0);
4818 format %{ %}
4819 interface(REG_INTER);
4820 %}
4821
4822 // Pointer 64 bit Register R3 only
4823 operand iRegP_R3()
4824 %{
4825 constraint(ALLOC_IN_RC(r3_reg));
4826 match(RegP);
4827 // match(iRegP);
4828 match(iRegPNoSp);
4829 op_cost(0);
4830 format %{ %}
4831 interface(REG_INTER);
4832 %}
4833
4834 // Pointer 64 bit Register R4 only
4835 operand iRegP_R4()
4836 %{
4837 constraint(ALLOC_IN_RC(r4_reg));
4838 match(RegP);
4839 // match(iRegP);
4840 match(iRegPNoSp);
4841 op_cost(0);
4842 format %{ %}
4843 interface(REG_INTER);
4844 %}
4845
4846 // Pointer 64 bit Register R5 only
4847 operand iRegP_R5()
4848 %{
4849 constraint(ALLOC_IN_RC(r5_reg));
4850 match(RegP);
4851 // match(iRegP);
4852 match(iRegPNoSp);
4853 op_cost(0);
4854 format %{ %}
4855 interface(REG_INTER);
4856 %}
4857
4858 // Pointer 64 bit Register R10 only
4859 operand iRegP_R10()
4860 %{
4861 constraint(ALLOC_IN_RC(r10_reg));
4862 match(RegP);
4863 // match(iRegP);
4864 match(iRegPNoSp);
4865 op_cost(0);
4866 format %{ %}
4867 interface(REG_INTER);
4868 %}
4869
4870 // Long 64 bit Register R0 only
4871 operand iRegL_R0()
4872 %{
4873 constraint(ALLOC_IN_RC(r0_reg));
4874 match(RegL);
4875 match(iRegLNoSp);
4876 op_cost(0);
4877 format %{ %}
4878 interface(REG_INTER);
4879 %}
4880
4881 // Long 64 bit Register R11 only
4882 operand iRegL_R11()
4883 %{
4884 constraint(ALLOC_IN_RC(r11_reg));
4885 match(RegL);
4886 match(iRegLNoSp);
4887 op_cost(0);
4888 format %{ %}
4889 interface(REG_INTER);
4890 %}
4891
4892 // Register R0 only
4893 operand iRegI_R0()
4894 %{
4895 constraint(ALLOC_IN_RC(int_r0_reg));
4896 match(RegI);
4897 match(iRegINoSp);
4898 op_cost(0);
4899 format %{ %}
4900 interface(REG_INTER);
4901 %}
4902
4903 // Register R2 only
4904 operand iRegI_R2()
4905 %{
4906 constraint(ALLOC_IN_RC(int_r2_reg));
4907 match(RegI);
4908 match(iRegINoSp);
4909 op_cost(0);
4910 format %{ %}
4911 interface(REG_INTER);
4912 %}
4913
4914 // Register R3 only
4915 operand iRegI_R3()
4916 %{
4917 constraint(ALLOC_IN_RC(int_r3_reg));
4918 match(RegI);
4919 match(iRegINoSp);
4920 op_cost(0);
4921 format %{ %}
4922 interface(REG_INTER);
4923 %}
4924
4925
4926 // Register R4 only
4927 operand iRegI_R4()
4928 %{
4929 constraint(ALLOC_IN_RC(int_r4_reg));
4930 match(RegI);
4931 match(iRegINoSp);
4932 op_cost(0);
4933 format %{ %}
4934 interface(REG_INTER);
4935 %}
4936
4937
4938 // Pointer Register Operands
4939 // Narrow Pointer Register
4940 operand iRegN()
4941 %{
4942 constraint(ALLOC_IN_RC(any_reg32));
4943 match(RegN);
4944 match(iRegNNoSp);
4945 op_cost(0);
4946 format %{ %}
4947 interface(REG_INTER);
4948 %}
4949
4950 // Integer 64 bit Register not Special
4951 operand iRegNNoSp()
4952 %{
4953 constraint(ALLOC_IN_RC(no_special_reg32));
4954 match(RegN);
4955 op_cost(0);
4956 format %{ %}
4957 interface(REG_INTER);
4958 %}
4959
4960 // Float Register
4961 // Float register operands
4962 operand vRegF()
4963 %{
4964 constraint(ALLOC_IN_RC(float_reg));
4965 match(RegF);
4966
4967 op_cost(0);
4968 format %{ %}
4969 interface(REG_INTER);
4970 %}
4971
4972 // Double Register
4973 // Double register operands
4974 operand vRegD()
4975 %{
4976 constraint(ALLOC_IN_RC(double_reg));
4977 match(RegD);
4978
4979 op_cost(0);
4980 format %{ %}
4981 interface(REG_INTER);
4982 %}
4983
4984 // Generic vector class. This will be used for
4985 // all vector operands, including NEON and SVE.
4986 operand vReg()
4987 %{
4988 constraint(ALLOC_IN_RC(dynamic));
4989 match(VecA);
4990 match(VecD);
4991 match(VecX);
4992
4993 op_cost(0);
4994 format %{ %}
4995 interface(REG_INTER);
4996 %}
4997
4998 operand vReg_V10()
4999 %{
5000 constraint(ALLOC_IN_RC(v10_veca_reg));
5001 match(vReg);
5002
5003 op_cost(0);
5004 format %{ %}
5005 interface(REG_INTER);
5006 %}
5007
5008 operand vReg_V11()
5009 %{
5010 constraint(ALLOC_IN_RC(v11_veca_reg));
5011 match(vReg);
5012
5013 op_cost(0);
5014 format %{ %}
5015 interface(REG_INTER);
5016 %}
5017
5018 operand vReg_V12()
5019 %{
5020 constraint(ALLOC_IN_RC(v12_veca_reg));
5021 match(vReg);
5022
5023 op_cost(0);
5024 format %{ %}
5025 interface(REG_INTER);
5026 %}
5027
5028 operand vReg_V13()
5029 %{
5030 constraint(ALLOC_IN_RC(v13_veca_reg));
5031 match(vReg);
5032
5033 op_cost(0);
5034 format %{ %}
5035 interface(REG_INTER);
5036 %}
5037
5038 operand vReg_V17()
5039 %{
5040 constraint(ALLOC_IN_RC(v17_veca_reg));
5041 match(vReg);
5042
5043 op_cost(0);
5044 format %{ %}
5045 interface(REG_INTER);
5046 %}
5047
5048 operand vReg_V18()
5049 %{
5050 constraint(ALLOC_IN_RC(v18_veca_reg));
5051 match(vReg);
5052
5053 op_cost(0);
5054 format %{ %}
5055 interface(REG_INTER);
5056 %}
5057
5058 operand vReg_V23()
5059 %{
5060 constraint(ALLOC_IN_RC(v23_veca_reg));
5061 match(vReg);
5062
5063 op_cost(0);
5064 format %{ %}
5065 interface(REG_INTER);
5066 %}
5067
5068 operand vReg_V24()
5069 %{
5070 constraint(ALLOC_IN_RC(v24_veca_reg));
5071 match(vReg);
5072
5073 op_cost(0);
5074 format %{ %}
5075 interface(REG_INTER);
5076 %}
5077
5078 operand vecA()
5079 %{
5080 constraint(ALLOC_IN_RC(vectora_reg));
5081 match(VecA);
5082
5083 op_cost(0);
5084 format %{ %}
5085 interface(REG_INTER);
5086 %}
5087
5088 operand vecD()
5089 %{
5090 constraint(ALLOC_IN_RC(vectord_reg));
5091 match(VecD);
5092
5093 op_cost(0);
5094 format %{ %}
5095 interface(REG_INTER);
5096 %}
5097
5098 operand vecX()
5099 %{
5100 constraint(ALLOC_IN_RC(vectorx_reg));
5101 match(VecX);
5102
5103 op_cost(0);
5104 format %{ %}
5105 interface(REG_INTER);
5106 %}
5107
5108 operand vRegD_V0()
5109 %{
5110 constraint(ALLOC_IN_RC(v0_reg));
5111 match(RegD);
5112 op_cost(0);
5113 format %{ %}
5114 interface(REG_INTER);
5115 %}
5116
5117 operand vRegD_V1()
5118 %{
5119 constraint(ALLOC_IN_RC(v1_reg));
5120 match(RegD);
5121 op_cost(0);
5122 format %{ %}
5123 interface(REG_INTER);
5124 %}
5125
5126 operand vRegD_V2()
5127 %{
5128 constraint(ALLOC_IN_RC(v2_reg));
5129 match(RegD);
5130 op_cost(0);
5131 format %{ %}
5132 interface(REG_INTER);
5133 %}
5134
5135 operand vRegD_V3()
5136 %{
5137 constraint(ALLOC_IN_RC(v3_reg));
5138 match(RegD);
5139 op_cost(0);
5140 format %{ %}
5141 interface(REG_INTER);
5142 %}
5143
5144 operand vRegD_V4()
5145 %{
5146 constraint(ALLOC_IN_RC(v4_reg));
5147 match(RegD);
5148 op_cost(0);
5149 format %{ %}
5150 interface(REG_INTER);
5151 %}
5152
5153 operand vRegD_V5()
5154 %{
5155 constraint(ALLOC_IN_RC(v5_reg));
5156 match(RegD);
5157 op_cost(0);
5158 format %{ %}
5159 interface(REG_INTER);
5160 %}
5161
5162 operand vRegD_V6()
5163 %{
5164 constraint(ALLOC_IN_RC(v6_reg));
5165 match(RegD);
5166 op_cost(0);
5167 format %{ %}
5168 interface(REG_INTER);
5169 %}
5170
5171 operand vRegD_V7()
5172 %{
5173 constraint(ALLOC_IN_RC(v7_reg));
5174 match(RegD);
5175 op_cost(0);
5176 format %{ %}
5177 interface(REG_INTER);
5178 %}
5179
5180 operand vRegD_V12()
5181 %{
5182 constraint(ALLOC_IN_RC(v12_reg));
5183 match(RegD);
5184 op_cost(0);
5185 format %{ %}
5186 interface(REG_INTER);
5187 %}
5188
5189 operand vRegD_V13()
5190 %{
5191 constraint(ALLOC_IN_RC(v13_reg));
5192 match(RegD);
5193 op_cost(0);
5194 format %{ %}
5195 interface(REG_INTER);
5196 %}
5197
5198 operand pReg()
5199 %{
5200 constraint(ALLOC_IN_RC(pr_reg));
5201 match(RegVectMask);
5202 match(pRegGov);
5203 op_cost(0);
5204 format %{ %}
5205 interface(REG_INTER);
5206 %}
5207
5208 operand pRegGov()
5209 %{
5210 constraint(ALLOC_IN_RC(gov_pr));
5211 match(RegVectMask);
5212 match(pReg);
5213 op_cost(0);
5214 format %{ %}
5215 interface(REG_INTER);
5216 %}
5217
5218 operand pRegGov_P0()
5219 %{
5220 constraint(ALLOC_IN_RC(p0_reg));
5221 match(RegVectMask);
5222 op_cost(0);
5223 format %{ %}
5224 interface(REG_INTER);
5225 %}
5226
5227 operand pRegGov_P1()
5228 %{
5229 constraint(ALLOC_IN_RC(p1_reg));
5230 match(RegVectMask);
5231 op_cost(0);
5232 format %{ %}
5233 interface(REG_INTER);
5234 %}
5235
5236 // Flags register, used as output of signed compare instructions
5237
5238 // note that on AArch64 we also use this register as the output for
5239 // for floating point compare instructions (CmpF CmpD). this ensures
5240 // that ordered inequality tests use GT, GE, LT or LE none of which
5241 // pass through cases where the result is unordered i.e. one or both
5242 // inputs to the compare is a NaN. this means that the ideal code can
5243 // replace e.g. a GT with an LE and not end up capturing the NaN case
5244 // (where the comparison should always fail). EQ and NE tests are
5245 // always generated in ideal code so that unordered folds into the NE
5246 // case, matching the behaviour of AArch64 NE.
5247 //
5248 // This differs from x86 where the outputs of FP compares use a
5249 // special FP flags registers and where compares based on this
5250 // register are distinguished into ordered inequalities (cmpOpUCF) and
5251 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5252 // to explicitly handle the unordered case in branches. x86 also has
5253 // to include extra CMoveX rules to accept a cmpOpUCF input.
5254
5255 operand rFlagsReg()
5256 %{
5257 constraint(ALLOC_IN_RC(int_flags));
5258 match(RegFlags);
5259
5260 op_cost(0);
5261 format %{ "RFLAGS" %}
5262 interface(REG_INTER);
5263 %}
5264
5265 // Flags register, used as output of unsigned compare instructions
5266 operand rFlagsRegU()
5267 %{
5268 constraint(ALLOC_IN_RC(int_flags));
5269 match(RegFlags);
5270
5271 op_cost(0);
5272 format %{ "RFLAGSU" %}
5273 interface(REG_INTER);
5274 %}
5275
5276 // Special Registers
5277
5278 // Method Register
5279 operand inline_cache_RegP(iRegP reg)
5280 %{
5281 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5282 match(reg);
5283 match(iRegPNoSp);
5284 op_cost(0);
5285 format %{ %}
5286 interface(REG_INTER);
5287 %}
5288
5289 // Thread Register
5290 operand thread_RegP(iRegP reg)
5291 %{
5292 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5293 match(reg);
5294 op_cost(0);
5295 format %{ %}
5296 interface(REG_INTER);
5297 %}
5298
5299 //----------Memory Operands----------------------------------------------------
5300
5301 operand indirect(iRegP reg)
5302 %{
5303 constraint(ALLOC_IN_RC(ptr_reg));
5304 match(reg);
5305 op_cost(0);
5306 format %{ "[$reg]" %}
5307 interface(MEMORY_INTER) %{
5308 base($reg);
5309 index(0xffffffff);
5310 scale(0x0);
5311 disp(0x0);
5312 %}
5313 %}
5314
5315 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5316 %{
5317 constraint(ALLOC_IN_RC(ptr_reg));
5318 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5319 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5320 op_cost(0);
5321 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5322 interface(MEMORY_INTER) %{
5323 base($reg);
5324 index($ireg);
5325 scale($scale);
5326 disp(0x0);
5327 %}
5328 %}
5329
5330 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5331 %{
5332 constraint(ALLOC_IN_RC(ptr_reg));
5333 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5334 match(AddP reg (LShiftL lreg scale));
5335 op_cost(0);
5336 format %{ "$reg, $lreg lsl($scale)" %}
5337 interface(MEMORY_INTER) %{
5338 base($reg);
5339 index($lreg);
5340 scale($scale);
5341 disp(0x0);
5342 %}
5343 %}
5344
5345 operand indIndexI2L(iRegP reg, iRegI ireg)
5346 %{
5347 constraint(ALLOC_IN_RC(ptr_reg));
5348 match(AddP reg (ConvI2L ireg));
5349 op_cost(0);
5350 format %{ "$reg, $ireg, 0, I2L" %}
5351 interface(MEMORY_INTER) %{
5352 base($reg);
5353 index($ireg);
5354 scale(0x0);
5355 disp(0x0);
5356 %}
5357 %}
5358
5359 operand indIndex(iRegP reg, iRegL lreg)
5360 %{
5361 constraint(ALLOC_IN_RC(ptr_reg));
5362 match(AddP reg lreg);
5363 op_cost(0);
5364 format %{ "$reg, $lreg" %}
5365 interface(MEMORY_INTER) %{
5366 base($reg);
5367 index($lreg);
5368 scale(0x0);
5369 disp(0x0);
5370 %}
5371 %}
5372
5373 operand indOffI1(iRegP reg, immIOffset1 off)
5374 %{
5375 constraint(ALLOC_IN_RC(ptr_reg));
5376 match(AddP reg off);
5377 op_cost(0);
5378 format %{ "[$reg, $off]" %}
5379 interface(MEMORY_INTER) %{
5380 base($reg);
5381 index(0xffffffff);
5382 scale(0x0);
5383 disp($off);
5384 %}
5385 %}
5386
5387 operand indOffI2(iRegP reg, immIOffset2 off)
5388 %{
5389 constraint(ALLOC_IN_RC(ptr_reg));
5390 match(AddP reg off);
5391 op_cost(0);
5392 format %{ "[$reg, $off]" %}
5393 interface(MEMORY_INTER) %{
5394 base($reg);
5395 index(0xffffffff);
5396 scale(0x0);
5397 disp($off);
5398 %}
5399 %}
5400
5401 operand indOffI4(iRegP reg, immIOffset4 off)
5402 %{
5403 constraint(ALLOC_IN_RC(ptr_reg));
5404 match(AddP reg off);
5405 op_cost(0);
5406 format %{ "[$reg, $off]" %}
5407 interface(MEMORY_INTER) %{
5408 base($reg);
5409 index(0xffffffff);
5410 scale(0x0);
5411 disp($off);
5412 %}
5413 %}
5414
5415 operand indOffI8(iRegP reg, immIOffset8 off)
5416 %{
5417 constraint(ALLOC_IN_RC(ptr_reg));
5418 match(AddP reg off);
5419 op_cost(0);
5420 format %{ "[$reg, $off]" %}
5421 interface(MEMORY_INTER) %{
5422 base($reg);
5423 index(0xffffffff);
5424 scale(0x0);
5425 disp($off);
5426 %}
5427 %}
5428
5429 operand indOffI16(iRegP reg, immIOffset16 off)
5430 %{
5431 constraint(ALLOC_IN_RC(ptr_reg));
5432 match(AddP reg off);
5433 op_cost(0);
5434 format %{ "[$reg, $off]" %}
5435 interface(MEMORY_INTER) %{
5436 base($reg);
5437 index(0xffffffff);
5438 scale(0x0);
5439 disp($off);
5440 %}
5441 %}
5442
5443 operand indOffL1(iRegP reg, immLoffset1 off)
5444 %{
5445 constraint(ALLOC_IN_RC(ptr_reg));
5446 match(AddP reg off);
5447 op_cost(0);
5448 format %{ "[$reg, $off]" %}
5449 interface(MEMORY_INTER) %{
5450 base($reg);
5451 index(0xffffffff);
5452 scale(0x0);
5453 disp($off);
5454 %}
5455 %}
5456
5457 operand indOffL2(iRegP reg, immLoffset2 off)
5458 %{
5459 constraint(ALLOC_IN_RC(ptr_reg));
5460 match(AddP reg off);
5461 op_cost(0);
5462 format %{ "[$reg, $off]" %}
5463 interface(MEMORY_INTER) %{
5464 base($reg);
5465 index(0xffffffff);
5466 scale(0x0);
5467 disp($off);
5468 %}
5469 %}
5470
5471 operand indOffL4(iRegP reg, immLoffset4 off)
5472 %{
5473 constraint(ALLOC_IN_RC(ptr_reg));
5474 match(AddP reg off);
5475 op_cost(0);
5476 format %{ "[$reg, $off]" %}
5477 interface(MEMORY_INTER) %{
5478 base($reg);
5479 index(0xffffffff);
5480 scale(0x0);
5481 disp($off);
5482 %}
5483 %}
5484
5485 operand indOffL8(iRegP reg, immLoffset8 off)
5486 %{
5487 constraint(ALLOC_IN_RC(ptr_reg));
5488 match(AddP reg off);
5489 op_cost(0);
5490 format %{ "[$reg, $off]" %}
5491 interface(MEMORY_INTER) %{
5492 base($reg);
5493 index(0xffffffff);
5494 scale(0x0);
5495 disp($off);
5496 %}
5497 %}
5498
5499 operand indOffL16(iRegP reg, immLoffset16 off)
5500 %{
5501 constraint(ALLOC_IN_RC(ptr_reg));
5502 match(AddP reg off);
5503 op_cost(0);
5504 format %{ "[$reg, $off]" %}
5505 interface(MEMORY_INTER) %{
5506 base($reg);
5507 index(0xffffffff);
5508 scale(0x0);
5509 disp($off);
5510 %}
5511 %}
5512
5513 operand indirectX2P(iRegL reg)
5514 %{
5515 constraint(ALLOC_IN_RC(ptr_reg));
5516 match(CastX2P reg);
5517 op_cost(0);
5518 format %{ "[$reg]\t# long -> ptr" %}
5519 interface(MEMORY_INTER) %{
5520 base($reg);
5521 index(0xffffffff);
5522 scale(0x0);
5523 disp(0x0);
5524 %}
5525 %}
5526
5527 operand indOffX2P(iRegL reg, immLOffset off)
5528 %{
5529 constraint(ALLOC_IN_RC(ptr_reg));
5530 match(AddP (CastX2P reg) off);
5531 op_cost(0);
5532 format %{ "[$reg, $off]\t# long -> ptr" %}
5533 interface(MEMORY_INTER) %{
5534 base($reg);
5535 index(0xffffffff);
5536 scale(0x0);
5537 disp($off);
5538 %}
5539 %}
5540
5541 operand indirectN(iRegN reg)
5542 %{
5543 predicate(CompressedOops::shift() == 0);
5544 constraint(ALLOC_IN_RC(ptr_reg));
5545 match(DecodeN reg);
5546 op_cost(0);
5547 format %{ "[$reg]\t# narrow" %}
5548 interface(MEMORY_INTER) %{
5549 base($reg);
5550 index(0xffffffff);
5551 scale(0x0);
5552 disp(0x0);
5553 %}
5554 %}
5555
5556 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5557 %{
5558 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5559 constraint(ALLOC_IN_RC(ptr_reg));
5560 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5561 op_cost(0);
5562 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5563 interface(MEMORY_INTER) %{
5564 base($reg);
5565 index($ireg);
5566 scale($scale);
5567 disp(0x0);
5568 %}
5569 %}
5570
5571 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5572 %{
5573 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5574 constraint(ALLOC_IN_RC(ptr_reg));
5575 match(AddP (DecodeN reg) (LShiftL lreg scale));
5576 op_cost(0);
5577 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5578 interface(MEMORY_INTER) %{
5579 base($reg);
5580 index($lreg);
5581 scale($scale);
5582 disp(0x0);
5583 %}
5584 %}
5585
5586 operand indIndexI2LN(iRegN reg, iRegI ireg)
5587 %{
5588 predicate(CompressedOops::shift() == 0);
5589 constraint(ALLOC_IN_RC(ptr_reg));
5590 match(AddP (DecodeN reg) (ConvI2L ireg));
5591 op_cost(0);
5592 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5593 interface(MEMORY_INTER) %{
5594 base($reg);
5595 index($ireg);
5596 scale(0x0);
5597 disp(0x0);
5598 %}
5599 %}
5600
5601 operand indIndexN(iRegN reg, iRegL lreg)
5602 %{
5603 predicate(CompressedOops::shift() == 0);
5604 constraint(ALLOC_IN_RC(ptr_reg));
5605 match(AddP (DecodeN reg) lreg);
5606 op_cost(0);
5607 format %{ "$reg, $lreg\t# narrow" %}
5608 interface(MEMORY_INTER) %{
5609 base($reg);
5610 index($lreg);
5611 scale(0x0);
5612 disp(0x0);
5613 %}
5614 %}
5615
5616 operand indOffIN(iRegN reg, immIOffset off)
5617 %{
5618 predicate(CompressedOops::shift() == 0);
5619 constraint(ALLOC_IN_RC(ptr_reg));
5620 match(AddP (DecodeN reg) off);
5621 op_cost(0);
5622 format %{ "[$reg, $off]\t# narrow" %}
5623 interface(MEMORY_INTER) %{
5624 base($reg);
5625 index(0xffffffff);
5626 scale(0x0);
5627 disp($off);
5628 %}
5629 %}
5630
5631 operand indOffLN(iRegN reg, immLOffset off)
5632 %{
5633 predicate(CompressedOops::shift() == 0);
5634 constraint(ALLOC_IN_RC(ptr_reg));
5635 match(AddP (DecodeN reg) off);
5636 op_cost(0);
5637 format %{ "[$reg, $off]\t# narrow" %}
5638 interface(MEMORY_INTER) %{
5639 base($reg);
5640 index(0xffffffff);
5641 scale(0x0);
5642 disp($off);
5643 %}
5644 %}
5645
5646
5647 //----------Special Memory Operands--------------------------------------------
5648 // Stack Slot Operand - This operand is used for loading and storing temporary
5649 // values on the stack where a match requires a value to
5650 // flow through memory.
5651 operand stackSlotP(sRegP reg)
5652 %{
5653 constraint(ALLOC_IN_RC(stack_slots));
5654 op_cost(100);
5655 // No match rule because this operand is only generated in matching
5656 // match(RegP);
5657 format %{ "[$reg]" %}
5658 interface(MEMORY_INTER) %{
5659 base(0x1e); // RSP
5660 index(0x0); // No Index
5661 scale(0x0); // No Scale
5662 disp($reg); // Stack Offset
5663 %}
5664 %}
5665
5666 operand stackSlotI(sRegI reg)
5667 %{
5668 constraint(ALLOC_IN_RC(stack_slots));
5669 // No match rule because this operand is only generated in matching
5670 // match(RegI);
5671 format %{ "[$reg]" %}
5672 interface(MEMORY_INTER) %{
5673 base(0x1e); // RSP
5674 index(0x0); // No Index
5675 scale(0x0); // No Scale
5676 disp($reg); // Stack Offset
5677 %}
5678 %}
5679
5680 operand stackSlotF(sRegF reg)
5681 %{
5682 constraint(ALLOC_IN_RC(stack_slots));
5683 // No match rule because this operand is only generated in matching
5684 // match(RegF);
5685 format %{ "[$reg]" %}
5686 interface(MEMORY_INTER) %{
5687 base(0x1e); // RSP
5688 index(0x0); // No Index
5689 scale(0x0); // No Scale
5690 disp($reg); // Stack Offset
5691 %}
5692 %}
5693
5694 operand stackSlotD(sRegD reg)
5695 %{
5696 constraint(ALLOC_IN_RC(stack_slots));
5697 // No match rule because this operand is only generated in matching
5698 // match(RegD);
5699 format %{ "[$reg]" %}
5700 interface(MEMORY_INTER) %{
5701 base(0x1e); // RSP
5702 index(0x0); // No Index
5703 scale(0x0); // No Scale
5704 disp($reg); // Stack Offset
5705 %}
5706 %}
5707
5708 operand stackSlotL(sRegL reg)
5709 %{
5710 constraint(ALLOC_IN_RC(stack_slots));
5711 // No match rule because this operand is only generated in matching
5712 // match(RegL);
5713 format %{ "[$reg]" %}
5714 interface(MEMORY_INTER) %{
5715 base(0x1e); // RSP
5716 index(0x0); // No Index
5717 scale(0x0); // No Scale
5718 disp($reg); // Stack Offset
5719 %}
5720 %}
5721
5722 // Operands for expressing Control Flow
5723 // NOTE: Label is a predefined operand which should not be redefined in
5724 // the AD file. It is generically handled within the ADLC.
5725
5726 //----------Conditional Branch Operands----------------------------------------
5727 // Comparison Op - This is the operation of the comparison, and is limited to
5728 // the following set of codes:
5729 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5730 //
5731 // Other attributes of the comparison, such as unsignedness, are specified
5732 // by the comparison instruction that sets a condition code flags register.
5733 // That result is represented by a flags operand whose subtype is appropriate
5734 // to the unsignedness (etc.) of the comparison.
5735 //
5736 // Later, the instruction which matches both the Comparison Op (a Bool) and
5737 // the flags (produced by the Cmp) specifies the coding of the comparison op
5738 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5739
5740 // used for signed integral comparisons and fp comparisons
5741
5742 operand cmpOp()
5743 %{
5744 match(Bool);
5745
5746 format %{ "" %}
5747 interface(COND_INTER) %{
5748 equal(0x0, "eq");
5749 not_equal(0x1, "ne");
5750 less(0xb, "lt");
5751 greater_equal(0xa, "ge");
5752 less_equal(0xd, "le");
5753 greater(0xc, "gt");
5754 overflow(0x6, "vs");
5755 no_overflow(0x7, "vc");
5756 %}
5757 %}
5758
5759 // used for unsigned integral comparisons
5760
5761 operand cmpOpU()
5762 %{
5763 match(Bool);
5764
5765 format %{ "" %}
5766 interface(COND_INTER) %{
5767 equal(0x0, "eq");
5768 not_equal(0x1, "ne");
5769 less(0x3, "lo");
5770 greater_equal(0x2, "hs");
5771 less_equal(0x9, "ls");
5772 greater(0x8, "hi");
5773 overflow(0x6, "vs");
5774 no_overflow(0x7, "vc");
5775 %}
5776 %}
5777
5778 // used for certain integral comparisons which can be
5779 // converted to cbxx or tbxx instructions
5780
5781 operand cmpOpEqNe()
5782 %{
5783 match(Bool);
5784 op_cost(0);
5785 predicate(n->as_Bool()->_test._test == BoolTest::ne
5786 || n->as_Bool()->_test._test == BoolTest::eq);
5787
5788 format %{ "" %}
5789 interface(COND_INTER) %{
5790 equal(0x0, "eq");
5791 not_equal(0x1, "ne");
5792 less(0xb, "lt");
5793 greater_equal(0xa, "ge");
5794 less_equal(0xd, "le");
5795 greater(0xc, "gt");
5796 overflow(0x6, "vs");
5797 no_overflow(0x7, "vc");
5798 %}
5799 %}
5800
5801 // used for certain integral comparisons which can be
5802 // converted to cbxx or tbxx instructions
5803
5804 operand cmpOpLtGe()
5805 %{
5806 match(Bool);
5807 op_cost(0);
5808
5809 predicate(n->as_Bool()->_test._test == BoolTest::lt
5810 || n->as_Bool()->_test._test == BoolTest::ge);
5811
5812 format %{ "" %}
5813 interface(COND_INTER) %{
5814 equal(0x0, "eq");
5815 not_equal(0x1, "ne");
5816 less(0xb, "lt");
5817 greater_equal(0xa, "ge");
5818 less_equal(0xd, "le");
5819 greater(0xc, "gt");
5820 overflow(0x6, "vs");
5821 no_overflow(0x7, "vc");
5822 %}
5823 %}
5824
5825 // used for certain unsigned integral comparisons which can be
5826 // converted to cbxx or tbxx instructions
5827
5828 operand cmpOpUEqNeLeGt()
5829 %{
5830 match(Bool);
5831 op_cost(0);
5832
5833 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5834 n->as_Bool()->_test._test == BoolTest::ne ||
5835 n->as_Bool()->_test._test == BoolTest::le ||
5836 n->as_Bool()->_test._test == BoolTest::gt);
5837
5838 format %{ "" %}
5839 interface(COND_INTER) %{
5840 equal(0x0, "eq");
5841 not_equal(0x1, "ne");
5842 less(0x3, "lo");
5843 greater_equal(0x2, "hs");
5844 less_equal(0x9, "ls");
5845 greater(0x8, "hi");
5846 overflow(0x6, "vs");
5847 no_overflow(0x7, "vc");
5848 %}
5849 %}
5850
5851 // Special operand allowing long args to int ops to be truncated for free
5852
5853 operand iRegL2I(iRegL reg) %{
5854
5855 op_cost(0);
5856
5857 match(ConvL2I reg);
5858
5859 format %{ "l2i($reg)" %}
5860
5861 interface(REG_INTER)
5862 %}
5863
5864 operand iRegL2P(iRegL reg) %{
5865
5866 op_cost(0);
5867
5868 match(CastX2P reg);
5869
5870 format %{ "l2p($reg)" %}
5871
5872 interface(REG_INTER)
5873 %}
5874
5875 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5876 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5877 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5878 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5879
5880 //----------OPERAND CLASSES----------------------------------------------------
5881 // Operand Classes are groups of operands that are used as to simplify
5882 // instruction definitions by not requiring the AD writer to specify
5883 // separate instructions for every form of operand when the
5884 // instruction accepts multiple operand types with the same basic
5885 // encoding and format. The classic case of this is memory operands.
5886
5887 // memory is used to define read/write location for load/store
5888 // instruction defs. we can turn a memory op into an Address
5889
5890 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5891 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5892
5893 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5894 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5895
5896 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5897 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5898
5899 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5900 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5901
5902 // All of the memory operands. For the pipeline description.
5903 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5904 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5905 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5906
5907
5908 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5909 // operations. it allows the src to be either an iRegI or a (ConvL2I
5910 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5911 // can be elided because the 32-bit instruction will just employ the
5912 // lower 32 bits anyway.
5913 //
5914 // n.b. this does not elide all L2I conversions. if the truncated
5915 // value is consumed by more than one operation then the ConvL2I
5916 // cannot be bundled into the consuming nodes so an l2i gets planted
5917 // (actually a movw $dst $src) and the downstream instructions consume
5918 // the result of the l2i as an iRegI input. That's a shame since the
5919 // movw is actually redundant but its not too costly.
5920
5921 opclass iRegIorL2I(iRegI, iRegL2I);
5922 opclass iRegPorL2P(iRegP, iRegL2P);
5923
5924 //----------PIPELINE-----------------------------------------------------------
5925 // Rules which define the behavior of the target architectures pipeline.
5926
5927 // For specific pipelines, eg A53, define the stages of that pipeline
5928 //pipe_desc(ISS, EX1, EX2, WR);
5929 #define ISS S0
5930 #define EX1 S1
5931 #define EX2 S2
5932 #define WR S3
5933
5934 // Integer ALU reg operation
5935 pipeline %{
5936
5937 attributes %{
5938 // ARM instructions are of fixed length
5939 fixed_size_instructions; // Fixed size instructions TODO does
5940 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5941 // ARM instructions come in 32-bit word units
5942 instruction_unit_size = 4; // An instruction is 4 bytes long
5943 instruction_fetch_unit_size = 64; // The processor fetches one line
5944 instruction_fetch_units = 1; // of 64 bytes
5945 %}
5946
5947 // We don't use an actual pipeline model so don't care about resources
5948 // or description. we do use pipeline classes to introduce fixed
5949 // latencies
5950
5951 //----------RESOURCES----------------------------------------------------------
5952 // Resources are the functional units available to the machine
5953
5954 resources( INS0, INS1, INS01 = INS0 | INS1,
5955 ALU0, ALU1, ALU = ALU0 | ALU1,
5956 MAC,
5957 DIV,
5958 BRANCH,
5959 LDST,
5960 NEON_FP);
5961
5962 //----------PIPELINE DESCRIPTION-----------------------------------------------
5963 // Pipeline Description specifies the stages in the machine's pipeline
5964
5965 // Define the pipeline as a generic 6 stage pipeline
5966 pipe_desc(S0, S1, S2, S3, S4, S5);
5967
5968 //----------PIPELINE CLASSES---------------------------------------------------
5969 // Pipeline Classes describe the stages in which input and output are
5970 // referenced by the hardware pipeline.
5971
5972 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5973 %{
5974 single_instruction;
5975 src1 : S1(read);
5976 src2 : S2(read);
5977 dst : S5(write);
5978 INS01 : ISS;
5979 NEON_FP : S5;
5980 %}
5981
5982 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5983 %{
5984 single_instruction;
5985 src1 : S1(read);
5986 src2 : S2(read);
5987 dst : S5(write);
5988 INS01 : ISS;
5989 NEON_FP : S5;
5990 %}
5991
5992 pipe_class fp_uop_s(vRegF dst, vRegF src)
5993 %{
5994 single_instruction;
5995 src : S1(read);
5996 dst : S5(write);
5997 INS01 : ISS;
5998 NEON_FP : S5;
5999 %}
6000
6001 pipe_class fp_uop_d(vRegD dst, vRegD src)
6002 %{
6003 single_instruction;
6004 src : S1(read);
6005 dst : S5(write);
6006 INS01 : ISS;
6007 NEON_FP : S5;
6008 %}
6009
6010 pipe_class fp_d2f(vRegF dst, vRegD src)
6011 %{
6012 single_instruction;
6013 src : S1(read);
6014 dst : S5(write);
6015 INS01 : ISS;
6016 NEON_FP : S5;
6017 %}
6018
6019 pipe_class fp_f2d(vRegD dst, vRegF src)
6020 %{
6021 single_instruction;
6022 src : S1(read);
6023 dst : S5(write);
6024 INS01 : ISS;
6025 NEON_FP : S5;
6026 %}
6027
6028 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6029 %{
6030 single_instruction;
6031 src : S1(read);
6032 dst : S5(write);
6033 INS01 : ISS;
6034 NEON_FP : S5;
6035 %}
6036
6037 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6038 %{
6039 single_instruction;
6040 src : S1(read);
6041 dst : S5(write);
6042 INS01 : ISS;
6043 NEON_FP : S5;
6044 %}
6045
6046 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6047 %{
6048 single_instruction;
6049 src : S1(read);
6050 dst : S5(write);
6051 INS01 : ISS;
6052 NEON_FP : S5;
6053 %}
6054
6055 pipe_class fp_l2f(vRegF dst, iRegL src)
6056 %{
6057 single_instruction;
6058 src : S1(read);
6059 dst : S5(write);
6060 INS01 : ISS;
6061 NEON_FP : S5;
6062 %}
6063
6064 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6065 %{
6066 single_instruction;
6067 src : S1(read);
6068 dst : S5(write);
6069 INS01 : ISS;
6070 NEON_FP : S5;
6071 %}
6072
6073 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6074 %{
6075 single_instruction;
6076 src : S1(read);
6077 dst : S5(write);
6078 INS01 : ISS;
6079 NEON_FP : S5;
6080 %}
6081
6082 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6083 %{
6084 single_instruction;
6085 src : S1(read);
6086 dst : S5(write);
6087 INS01 : ISS;
6088 NEON_FP : S5;
6089 %}
6090
6091 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6092 %{
6093 single_instruction;
6094 src : S1(read);
6095 dst : S5(write);
6096 INS01 : ISS;
6097 NEON_FP : S5;
6098 %}
6099
6100 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6101 %{
6102 single_instruction;
6103 src1 : S1(read);
6104 src2 : S2(read);
6105 dst : S5(write);
6106 INS0 : ISS;
6107 NEON_FP : S5;
6108 %}
6109
6110 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6111 %{
6112 single_instruction;
6113 src1 : S1(read);
6114 src2 : S2(read);
6115 dst : S5(write);
6116 INS0 : ISS;
6117 NEON_FP : S5;
6118 %}
6119
6120 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6121 %{
6122 single_instruction;
6123 cr : S1(read);
6124 src1 : S1(read);
6125 src2 : S1(read);
6126 dst : S3(write);
6127 INS01 : ISS;
6128 NEON_FP : S3;
6129 %}
6130
6131 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6132 %{
6133 single_instruction;
6134 cr : S1(read);
6135 src1 : S1(read);
6136 src2 : S1(read);
6137 dst : S3(write);
6138 INS01 : ISS;
6139 NEON_FP : S3;
6140 %}
6141
6142 pipe_class fp_imm_s(vRegF dst)
6143 %{
6144 single_instruction;
6145 dst : S3(write);
6146 INS01 : ISS;
6147 NEON_FP : S3;
6148 %}
6149
6150 pipe_class fp_imm_d(vRegD dst)
6151 %{
6152 single_instruction;
6153 dst : S3(write);
6154 INS01 : ISS;
6155 NEON_FP : S3;
6156 %}
6157
6158 pipe_class fp_load_constant_s(vRegF dst)
6159 %{
6160 single_instruction;
6161 dst : S4(write);
6162 INS01 : ISS;
6163 NEON_FP : S4;
6164 %}
6165
6166 pipe_class fp_load_constant_d(vRegD dst)
6167 %{
6168 single_instruction;
6169 dst : S4(write);
6170 INS01 : ISS;
6171 NEON_FP : S4;
6172 %}
6173
6174 //------- Integer ALU operations --------------------------
6175
6176 // Integer ALU reg-reg operation
6177 // Operands needed in EX1, result generated in EX2
6178 // Eg. ADD x0, x1, x2
6179 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6180 %{
6181 single_instruction;
6182 dst : EX2(write);
6183 src1 : EX1(read);
6184 src2 : EX1(read);
6185 INS01 : ISS; // Dual issue as instruction 0 or 1
6186 ALU : EX2;
6187 %}
6188
6189 // Integer ALU reg-reg operation with constant shift
6190 // Shifted register must be available in LATE_ISS instead of EX1
6191 // Eg. ADD x0, x1, x2, LSL #2
6192 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6193 %{
6194 single_instruction;
6195 dst : EX2(write);
6196 src1 : EX1(read);
6197 src2 : ISS(read);
6198 INS01 : ISS;
6199 ALU : EX2;
6200 %}
6201
6202 // Integer ALU reg operation with constant shift
6203 // Eg. LSL x0, x1, #shift
6204 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6205 %{
6206 single_instruction;
6207 dst : EX2(write);
6208 src1 : ISS(read);
6209 INS01 : ISS;
6210 ALU : EX2;
6211 %}
6212
6213 // Integer ALU reg-reg operation with variable shift
6214 // Both operands must be available in LATE_ISS instead of EX1
6215 // Result is available in EX1 instead of EX2
6216 // Eg. LSLV x0, x1, x2
6217 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6218 %{
6219 single_instruction;
6220 dst : EX1(write);
6221 src1 : ISS(read);
6222 src2 : ISS(read);
6223 INS01 : ISS;
6224 ALU : EX1;
6225 %}
6226
6227 // Integer ALU reg-reg operation with extract
6228 // As for _vshift above, but result generated in EX2
6229 // Eg. EXTR x0, x1, x2, #N
6230 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6231 %{
6232 single_instruction;
6233 dst : EX2(write);
6234 src1 : ISS(read);
6235 src2 : ISS(read);
6236 INS1 : ISS; // Can only dual issue as Instruction 1
6237 ALU : EX1;
6238 %}
6239
6240 // Integer ALU reg operation
6241 // Eg. NEG x0, x1
6242 pipe_class ialu_reg(iRegI dst, iRegI src)
6243 %{
6244 single_instruction;
6245 dst : EX2(write);
6246 src : EX1(read);
6247 INS01 : ISS;
6248 ALU : EX2;
6249 %}
6250
6251 // Integer ALU reg mmediate operation
6252 // Eg. ADD x0, x1, #N
6253 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6254 %{
6255 single_instruction;
6256 dst : EX2(write);
6257 src1 : EX1(read);
6258 INS01 : ISS;
6259 ALU : EX2;
6260 %}
6261
6262 // Integer ALU immediate operation (no source operands)
6263 // Eg. MOV x0, #N
6264 pipe_class ialu_imm(iRegI dst)
6265 %{
6266 single_instruction;
6267 dst : EX1(write);
6268 INS01 : ISS;
6269 ALU : EX1;
6270 %}
6271
6272 //------- Compare operation -------------------------------
6273
6274 // Compare reg-reg
6275 // Eg. CMP x0, x1
6276 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6277 %{
6278 single_instruction;
6279 // fixed_latency(16);
6280 cr : EX2(write);
6281 op1 : EX1(read);
6282 op2 : EX1(read);
6283 INS01 : ISS;
6284 ALU : EX2;
6285 %}
6286
6287 // Compare reg-reg
6288 // Eg. CMP x0, #N
6289 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6290 %{
6291 single_instruction;
6292 // fixed_latency(16);
6293 cr : EX2(write);
6294 op1 : EX1(read);
6295 INS01 : ISS;
6296 ALU : EX2;
6297 %}
6298
6299 //------- Conditional instructions ------------------------
6300
6301 // Conditional no operands
6302 // Eg. CSINC x0, zr, zr, <cond>
6303 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6304 %{
6305 single_instruction;
6306 cr : EX1(read);
6307 dst : EX2(write);
6308 INS01 : ISS;
6309 ALU : EX2;
6310 %}
6311
6312 // Conditional 2 operand
6313 // EG. CSEL X0, X1, X2, <cond>
6314 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6315 %{
6316 single_instruction;
6317 cr : EX1(read);
6318 src1 : EX1(read);
6319 src2 : EX1(read);
6320 dst : EX2(write);
6321 INS01 : ISS;
6322 ALU : EX2;
6323 %}
6324
6325 // Conditional 2 operand
6326 // EG. CSEL X0, X1, X2, <cond>
6327 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6328 %{
6329 single_instruction;
6330 cr : EX1(read);
6331 src : EX1(read);
6332 dst : EX2(write);
6333 INS01 : ISS;
6334 ALU : EX2;
6335 %}
6336
6337 //------- Multiply pipeline operations --------------------
6338
6339 // Multiply reg-reg
6340 // Eg. MUL w0, w1, w2
6341 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6342 %{
6343 single_instruction;
6344 dst : WR(write);
6345 src1 : ISS(read);
6346 src2 : ISS(read);
6347 INS01 : ISS;
6348 MAC : WR;
6349 %}
6350
6351 // Multiply accumulate
6352 // Eg. MADD w0, w1, w2, w3
6353 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6354 %{
6355 single_instruction;
6356 dst : WR(write);
6357 src1 : ISS(read);
6358 src2 : ISS(read);
6359 src3 : ISS(read);
6360 INS01 : ISS;
6361 MAC : WR;
6362 %}
6363
6364 // Eg. MUL w0, w1, w2
6365 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6366 %{
6367 single_instruction;
6368 fixed_latency(3); // Maximum latency for 64 bit mul
6369 dst : WR(write);
6370 src1 : ISS(read);
6371 src2 : ISS(read);
6372 INS01 : ISS;
6373 MAC : WR;
6374 %}
6375
6376 // Multiply accumulate
6377 // Eg. MADD w0, w1, w2, w3
6378 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6379 %{
6380 single_instruction;
6381 fixed_latency(3); // Maximum latency for 64 bit mul
6382 dst : WR(write);
6383 src1 : ISS(read);
6384 src2 : ISS(read);
6385 src3 : ISS(read);
6386 INS01 : ISS;
6387 MAC : WR;
6388 %}
6389
6390 //------- Divide pipeline operations --------------------
6391
6392 // Eg. SDIV w0, w1, w2
6393 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6394 %{
6395 single_instruction;
6396 fixed_latency(8); // Maximum latency for 32 bit divide
6397 dst : WR(write);
6398 src1 : ISS(read);
6399 src2 : ISS(read);
6400 INS0 : ISS; // Can only dual issue as instruction 0
6401 DIV : WR;
6402 %}
6403
6404 // Eg. SDIV x0, x1, x2
6405 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6406 %{
6407 single_instruction;
6408 fixed_latency(16); // Maximum latency for 64 bit divide
6409 dst : WR(write);
6410 src1 : ISS(read);
6411 src2 : ISS(read);
6412 INS0 : ISS; // Can only dual issue as instruction 0
6413 DIV : WR;
6414 %}
6415
6416 //------- Load pipeline operations ------------------------
6417
6418 // Load - prefetch
6419 // Eg. PFRM <mem>
6420 pipe_class iload_prefetch(memory mem)
6421 %{
6422 single_instruction;
6423 mem : ISS(read);
6424 INS01 : ISS;
6425 LDST : WR;
6426 %}
6427
6428 // Load - reg, mem
6429 // Eg. LDR x0, <mem>
6430 pipe_class iload_reg_mem(iRegI dst, memory mem)
6431 %{
6432 single_instruction;
6433 dst : WR(write);
6434 mem : ISS(read);
6435 INS01 : ISS;
6436 LDST : WR;
6437 %}
6438
6439 // Load - reg, reg
6440 // Eg. LDR x0, [sp, x1]
6441 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6442 %{
6443 single_instruction;
6444 dst : WR(write);
6445 src : ISS(read);
6446 INS01 : ISS;
6447 LDST : WR;
6448 %}
6449
6450 //------- Store pipeline operations -----------------------
6451
6452 // Store - zr, mem
6453 // Eg. STR zr, <mem>
6454 pipe_class istore_mem(memory mem)
6455 %{
6456 single_instruction;
6457 mem : ISS(read);
6458 INS01 : ISS;
6459 LDST : WR;
6460 %}
6461
6462 // Store - reg, mem
6463 // Eg. STR x0, <mem>
6464 pipe_class istore_reg_mem(iRegI src, memory mem)
6465 %{
6466 single_instruction;
6467 mem : ISS(read);
6468 src : EX2(read);
6469 INS01 : ISS;
6470 LDST : WR;
6471 %}
6472
6473 // Store - reg, reg
6474 // Eg. STR x0, [sp, x1]
6475 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6476 %{
6477 single_instruction;
6478 dst : ISS(read);
6479 src : EX2(read);
6480 INS01 : ISS;
6481 LDST : WR;
6482 %}
6483
6484 //------- Store pipeline operations -----------------------
6485
6486 // Branch
6487 pipe_class pipe_branch()
6488 %{
6489 single_instruction;
6490 INS01 : ISS;
6491 BRANCH : EX1;
6492 %}
6493
6494 // Conditional branch
6495 pipe_class pipe_branch_cond(rFlagsReg cr)
6496 %{
6497 single_instruction;
6498 cr : EX1(read);
6499 INS01 : ISS;
6500 BRANCH : EX1;
6501 %}
6502
6503 // Compare & Branch
6504 // EG. CBZ/CBNZ
6505 pipe_class pipe_cmp_branch(iRegI op1)
6506 %{
6507 single_instruction;
6508 op1 : EX1(read);
6509 INS01 : ISS;
6510 BRANCH : EX1;
6511 %}
6512
6513 //------- Synchronisation operations ----------------------
6514
6515 // Any operation requiring serialization.
6516 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6517 pipe_class pipe_serial()
6518 %{
6519 single_instruction;
6520 force_serialization;
6521 fixed_latency(16);
6522 INS01 : ISS(2); // Cannot dual issue with any other instruction
6523 LDST : WR;
6524 %}
6525
6526 // Generic big/slow expanded idiom - also serialized
6527 pipe_class pipe_slow()
6528 %{
6529 instruction_count(10);
6530 multiple_bundles;
6531 force_serialization;
6532 fixed_latency(16);
6533 INS01 : ISS(2); // Cannot dual issue with any other instruction
6534 LDST : WR;
6535 %}
6536
6537 // Empty pipeline class
6538 pipe_class pipe_class_empty()
6539 %{
6540 single_instruction;
6541 fixed_latency(0);
6542 %}
6543
6544 // Default pipeline class.
6545 pipe_class pipe_class_default()
6546 %{
6547 single_instruction;
6548 fixed_latency(2);
6549 %}
6550
6551 // Pipeline class for compares.
6552 pipe_class pipe_class_compare()
6553 %{
6554 single_instruction;
6555 fixed_latency(16);
6556 %}
6557
6558 // Pipeline class for memory operations.
6559 pipe_class pipe_class_memory()
6560 %{
6561 single_instruction;
6562 fixed_latency(16);
6563 %}
6564
6565 // Pipeline class for call.
6566 pipe_class pipe_class_call()
6567 %{
6568 single_instruction;
6569 fixed_latency(100);
6570 %}
6571
6572 // Define the class for the Nop node.
6573 define %{
6574 MachNop = pipe_class_empty;
6575 %}
6576
6577 %}
6578 //----------INSTRUCTIONS-------------------------------------------------------
6579 //
6580 // match -- States which machine-independent subtree may be replaced
6581 // by this instruction.
6582 // ins_cost -- The estimated cost of this instruction is used by instruction
6583 // selection to identify a minimum cost tree of machine
6584 // instructions that matches a tree of machine-independent
6585 // instructions.
6586 // format -- A string providing the disassembly for this instruction.
6587 // The value of an instruction's operand may be inserted
6588 // by referring to it with a '$' prefix.
6589 // opcode -- Three instruction opcodes may be provided. These are referred
6590 // to within an encode class as $primary, $secondary, and $tertiary
6591 // rrspectively. The primary opcode is commonly used to
6592 // indicate the type of machine instruction, while secondary
6593 // and tertiary are often used for prefix options or addressing
6594 // modes.
6595 // ins_encode -- A list of encode classes with parameters. The encode class
6596 // name must have been defined in an 'enc_class' specification
6597 // in the encode section of the architecture description.
6598
6599 // ============================================================================
6600 // Memory (Load/Store) Instructions
6601
6602 // Load Instructions
6603
6604 // Load Byte (8 bit signed)
6605 instruct loadB(iRegINoSp dst, memory1 mem)
6606 %{
6607 match(Set dst (LoadB mem));
6608 predicate(!needs_acquiring_load(n));
6609
6610 ins_cost(4 * INSN_COST);
6611 format %{ "ldrsbw $dst, $mem\t# byte" %}
6612
6613 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6614
6615 ins_pipe(iload_reg_mem);
6616 %}
6617
6618 // Load Byte (8 bit signed) into long
6619 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6620 %{
6621 match(Set dst (ConvI2L (LoadB mem)));
6622 predicate(!needs_acquiring_load(n->in(1)));
6623
6624 ins_cost(4 * INSN_COST);
6625 format %{ "ldrsb $dst, $mem\t# byte" %}
6626
6627 ins_encode(aarch64_enc_ldrsb(dst, mem));
6628
6629 ins_pipe(iload_reg_mem);
6630 %}
6631
6632 // Load Byte (8 bit unsigned)
6633 instruct loadUB(iRegINoSp dst, memory1 mem)
6634 %{
6635 match(Set dst (LoadUB mem));
6636 predicate(!needs_acquiring_load(n));
6637
6638 ins_cost(4 * INSN_COST);
6639 format %{ "ldrbw $dst, $mem\t# byte" %}
6640
6641 ins_encode(aarch64_enc_ldrb(dst, mem));
6642
6643 ins_pipe(iload_reg_mem);
6644 %}
6645
6646 // Load Byte (8 bit unsigned) into long
6647 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6648 %{
6649 match(Set dst (ConvI2L (LoadUB mem)));
6650 predicate(!needs_acquiring_load(n->in(1)));
6651
6652 ins_cost(4 * INSN_COST);
6653 format %{ "ldrb $dst, $mem\t# byte" %}
6654
6655 ins_encode(aarch64_enc_ldrb(dst, mem));
6656
6657 ins_pipe(iload_reg_mem);
6658 %}
6659
6660 // Load Short (16 bit signed)
6661 instruct loadS(iRegINoSp dst, memory2 mem)
6662 %{
6663 match(Set dst (LoadS mem));
6664 predicate(!needs_acquiring_load(n));
6665
6666 ins_cost(4 * INSN_COST);
6667 format %{ "ldrshw $dst, $mem\t# short" %}
6668
6669 ins_encode(aarch64_enc_ldrshw(dst, mem));
6670
6671 ins_pipe(iload_reg_mem);
6672 %}
6673
6674 // Load Short (16 bit signed) into long
6675 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6676 %{
6677 match(Set dst (ConvI2L (LoadS mem)));
6678 predicate(!needs_acquiring_load(n->in(1)));
6679
6680 ins_cost(4 * INSN_COST);
6681 format %{ "ldrsh $dst, $mem\t# short" %}
6682
6683 ins_encode(aarch64_enc_ldrsh(dst, mem));
6684
6685 ins_pipe(iload_reg_mem);
6686 %}
6687
6688 // Load Char (16 bit unsigned)
6689 instruct loadUS(iRegINoSp dst, memory2 mem)
6690 %{
6691 match(Set dst (LoadUS mem));
6692 predicate(!needs_acquiring_load(n));
6693
6694 ins_cost(4 * INSN_COST);
6695 format %{ "ldrh $dst, $mem\t# short" %}
6696
6697 ins_encode(aarch64_enc_ldrh(dst, mem));
6698
6699 ins_pipe(iload_reg_mem);
6700 %}
6701
6702 // Load Short/Char (16 bit unsigned) into long
6703 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6704 %{
6705 match(Set dst (ConvI2L (LoadUS mem)));
6706 predicate(!needs_acquiring_load(n->in(1)));
6707
6708 ins_cost(4 * INSN_COST);
6709 format %{ "ldrh $dst, $mem\t# short" %}
6710
6711 ins_encode(aarch64_enc_ldrh(dst, mem));
6712
6713 ins_pipe(iload_reg_mem);
6714 %}
6715
6716 // Load Integer (32 bit signed)
6717 instruct loadI(iRegINoSp dst, memory4 mem)
6718 %{
6719 match(Set dst (LoadI mem));
6720 predicate(!needs_acquiring_load(n));
6721
6722 ins_cost(4 * INSN_COST);
6723 format %{ "ldrw $dst, $mem\t# int" %}
6724
6725 ins_encode(aarch64_enc_ldrw(dst, mem));
6726
6727 ins_pipe(iload_reg_mem);
6728 %}
6729
6730 // Load Integer (32 bit signed) into long
6731 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6732 %{
6733 match(Set dst (ConvI2L (LoadI mem)));
6734 predicate(!needs_acquiring_load(n->in(1)));
6735
6736 ins_cost(4 * INSN_COST);
6737 format %{ "ldrsw $dst, $mem\t# int" %}
6738
6739 ins_encode(aarch64_enc_ldrsw(dst, mem));
6740
6741 ins_pipe(iload_reg_mem);
6742 %}
6743
6744 // Load Integer (32 bit unsigned) into long
6745 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6746 %{
6747 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6748 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6749
6750 ins_cost(4 * INSN_COST);
6751 format %{ "ldrw $dst, $mem\t# int" %}
6752
6753 ins_encode(aarch64_enc_ldrw(dst, mem));
6754
6755 ins_pipe(iload_reg_mem);
6756 %}
6757
6758 // Load Long (64 bit signed)
6759 instruct loadL(iRegLNoSp dst, memory8 mem)
6760 %{
6761 match(Set dst (LoadL mem));
6762 predicate(!needs_acquiring_load(n));
6763
6764 ins_cost(4 * INSN_COST);
6765 format %{ "ldr $dst, $mem\t# int" %}
6766
6767 ins_encode(aarch64_enc_ldr(dst, mem));
6768
6769 ins_pipe(iload_reg_mem);
6770 %}
6771
6772 // Load Range
6773 instruct loadRange(iRegINoSp dst, memory4 mem)
6774 %{
6775 match(Set dst (LoadRange mem));
6776
6777 ins_cost(4 * INSN_COST);
6778 format %{ "ldrw $dst, $mem\t# range" %}
6779
6780 ins_encode(aarch64_enc_ldrw(dst, mem));
6781
6782 ins_pipe(iload_reg_mem);
6783 %}
6784
6785 // Load Pointer
6786 instruct loadP(iRegPNoSp dst, memory8 mem)
6787 %{
6788 match(Set dst (LoadP mem));
6789 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6790
6791 ins_cost(4 * INSN_COST);
6792 format %{ "ldr $dst, $mem\t# ptr" %}
6793
6794 ins_encode(aarch64_enc_ldr(dst, mem));
6795
6796 ins_pipe(iload_reg_mem);
6797 %}
6798
6799 // Load Compressed Pointer
6800 instruct loadN(iRegNNoSp dst, memory4 mem)
6801 %{
6802 match(Set dst (LoadN mem));
6803 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6804
6805 ins_cost(4 * INSN_COST);
6806 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6807
6808 ins_encode(aarch64_enc_ldrw(dst, mem));
6809
6810 ins_pipe(iload_reg_mem);
6811 %}
6812
6813 // Load Klass Pointer
6814 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6815 %{
6816 match(Set dst (LoadKlass mem));
6817 predicate(!needs_acquiring_load(n));
6818
6819 ins_cost(4 * INSN_COST);
6820 format %{ "ldr $dst, $mem\t# class" %}
6821
6822 ins_encode(aarch64_enc_ldr(dst, mem));
6823
6824 ins_pipe(iload_reg_mem);
6825 %}
6826
6827 // Load Narrow Klass Pointer
6828 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6829 %{
6830 match(Set dst (LoadNKlass mem));
6831 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6832
6833 ins_cost(4 * INSN_COST);
6834 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6835
6836 ins_encode(aarch64_enc_ldrw(dst, mem));
6837
6838 ins_pipe(iload_reg_mem);
6839 %}
6840
6841 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6842 %{
6843 match(Set dst (LoadNKlass mem));
6844 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6845
6846 ins_cost(4 * INSN_COST);
6847 format %{
6848 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6849 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6850 %}
6851 ins_encode %{
6852 // inlined aarch64_enc_ldrw
6853 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6854 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6855 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6856 %}
6857 ins_pipe(iload_reg_mem);
6858 %}
6859
6860 // Load Float
6861 instruct loadF(vRegF dst, memory4 mem)
6862 %{
6863 match(Set dst (LoadF mem));
6864 predicate(!needs_acquiring_load(n));
6865
6866 ins_cost(4 * INSN_COST);
6867 format %{ "ldrs $dst, $mem\t# float" %}
6868
6869 ins_encode( aarch64_enc_ldrs(dst, mem) );
6870
6871 ins_pipe(pipe_class_memory);
6872 %}
6873
6874 // Load Double
6875 instruct loadD(vRegD dst, memory8 mem)
6876 %{
6877 match(Set dst (LoadD mem));
6878 predicate(!needs_acquiring_load(n));
6879
6880 ins_cost(4 * INSN_COST);
6881 format %{ "ldrd $dst, $mem\t# double" %}
6882
6883 ins_encode( aarch64_enc_ldrd(dst, mem) );
6884
6885 ins_pipe(pipe_class_memory);
6886 %}
6887
6888
6889 // Load Int Constant
6890 instruct loadConI(iRegINoSp dst, immI src)
6891 %{
6892 match(Set dst src);
6893
6894 ins_cost(INSN_COST);
6895 format %{ "mov $dst, $src\t# int" %}
6896
6897 ins_encode( aarch64_enc_movw_imm(dst, src) );
6898
6899 ins_pipe(ialu_imm);
6900 %}
6901
6902 // Load Long Constant
6903 instruct loadConL(iRegLNoSp dst, immL src)
6904 %{
6905 match(Set dst src);
6906
6907 ins_cost(INSN_COST);
6908 format %{ "mov $dst, $src\t# long" %}
6909
6910 ins_encode( aarch64_enc_mov_imm(dst, src) );
6911
6912 ins_pipe(ialu_imm);
6913 %}
6914
6915 // Load Pointer Constant
6916
6917 instruct loadConP(iRegPNoSp dst, immP con)
6918 %{
6919 match(Set dst con);
6920
6921 ins_cost(INSN_COST * 4);
6922 format %{
6923 "mov $dst, $con\t# ptr\n\t"
6924 %}
6925
6926 ins_encode(aarch64_enc_mov_p(dst, con));
6927
6928 ins_pipe(ialu_imm);
6929 %}
6930
6931 // Load Null Pointer Constant
6932
6933 instruct loadConP0(iRegPNoSp dst, immP0 con)
6934 %{
6935 match(Set dst con);
6936
6937 ins_cost(INSN_COST);
6938 format %{ "mov $dst, $con\t# nullptr ptr" %}
6939
6940 ins_encode(aarch64_enc_mov_p0(dst, con));
6941
6942 ins_pipe(ialu_imm);
6943 %}
6944
6945 // Load Pointer Constant One
6946
6947 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6948 %{
6949 match(Set dst con);
6950
6951 ins_cost(INSN_COST);
6952 format %{ "mov $dst, $con\t# nullptr ptr" %}
6953
6954 ins_encode(aarch64_enc_mov_p1(dst, con));
6955
6956 ins_pipe(ialu_imm);
6957 %}
6958
6959 // Load Narrow Pointer Constant
6960
6961 instruct loadConN(iRegNNoSp dst, immN con)
6962 %{
6963 match(Set dst con);
6964
6965 ins_cost(INSN_COST * 4);
6966 format %{ "mov $dst, $con\t# compressed ptr" %}
6967
6968 ins_encode(aarch64_enc_mov_n(dst, con));
6969
6970 ins_pipe(ialu_imm);
6971 %}
6972
6973 // Load Narrow Null Pointer Constant
6974
6975 instruct loadConN0(iRegNNoSp dst, immN0 con)
6976 %{
6977 match(Set dst con);
6978
6979 ins_cost(INSN_COST);
6980 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6981
6982 ins_encode(aarch64_enc_mov_n0(dst, con));
6983
6984 ins_pipe(ialu_imm);
6985 %}
6986
6987 // Load Narrow Klass Constant
6988
6989 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6990 %{
6991 match(Set dst con);
6992
6993 ins_cost(INSN_COST);
6994 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6995
6996 ins_encode(aarch64_enc_mov_nk(dst, con));
6997
6998 ins_pipe(ialu_imm);
6999 %}
7000
7001 // Load Packed Float Constant
7002
7003 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7004 match(Set dst con);
7005 ins_cost(INSN_COST * 4);
7006 format %{ "fmovs $dst, $con"%}
7007 ins_encode %{
7008 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7009 %}
7010
7011 ins_pipe(fp_imm_s);
7012 %}
7013
7014 // Load Float Constant
7015
7016 instruct loadConF(vRegF dst, immF con) %{
7017 match(Set dst con);
7018
7019 ins_cost(INSN_COST * 4);
7020
7021 format %{
7022 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7023 %}
7024
7025 ins_encode %{
7026 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7027 %}
7028
7029 ins_pipe(fp_load_constant_s);
7030 %}
7031
7032 // Load Packed Double Constant
7033
7034 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7035 match(Set dst con);
7036 ins_cost(INSN_COST);
7037 format %{ "fmovd $dst, $con"%}
7038 ins_encode %{
7039 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7040 %}
7041
7042 ins_pipe(fp_imm_d);
7043 %}
7044
7045 // Load Double Constant
7046
7047 instruct loadConD(vRegD dst, immD con) %{
7048 match(Set dst con);
7049
7050 ins_cost(INSN_COST * 5);
7051 format %{
7052 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7053 %}
7054
7055 ins_encode %{
7056 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7057 %}
7058
7059 ins_pipe(fp_load_constant_d);
7060 %}
7061
7062 // Load Half Float Constant
7063 instruct loadConH(vRegF dst, immH con) %{
7064 match(Set dst con);
7065 format %{ "mov rscratch1, $con\n\t"
7066 "fmov $dst, rscratch1"
7067 %}
7068 ins_encode %{
7069 __ movw(rscratch1, (uint32_t)$con$$constant);
7070 __ fmovs($dst$$FloatRegister, rscratch1);
7071 %}
7072 ins_pipe(pipe_class_default);
7073 %}
7074
7075 // Store Instructions
7076
7077 // Store Byte
7078 instruct storeB(iRegIorL2I src, memory1 mem)
7079 %{
7080 match(Set mem (StoreB mem src));
7081 predicate(!needs_releasing_store(n));
7082
7083 ins_cost(INSN_COST);
7084 format %{ "strb $src, $mem\t# byte" %}
7085
7086 ins_encode(aarch64_enc_strb(src, mem));
7087
7088 ins_pipe(istore_reg_mem);
7089 %}
7090
7091
7092 instruct storeimmB0(immI0 zero, memory1 mem)
7093 %{
7094 match(Set mem (StoreB mem zero));
7095 predicate(!needs_releasing_store(n));
7096
7097 ins_cost(INSN_COST);
7098 format %{ "strb rscractch2, $mem\t# byte" %}
7099
7100 ins_encode(aarch64_enc_strb0(mem));
7101
7102 ins_pipe(istore_mem);
7103 %}
7104
7105 // Store Char/Short
7106 instruct storeC(iRegIorL2I src, memory2 mem)
7107 %{
7108 match(Set mem (StoreC mem src));
7109 predicate(!needs_releasing_store(n));
7110
7111 ins_cost(INSN_COST);
7112 format %{ "strh $src, $mem\t# short" %}
7113
7114 ins_encode(aarch64_enc_strh(src, mem));
7115
7116 ins_pipe(istore_reg_mem);
7117 %}
7118
7119 instruct storeimmC0(immI0 zero, memory2 mem)
7120 %{
7121 match(Set mem (StoreC mem zero));
7122 predicate(!needs_releasing_store(n));
7123
7124 ins_cost(INSN_COST);
7125 format %{ "strh zr, $mem\t# short" %}
7126
7127 ins_encode(aarch64_enc_strh0(mem));
7128
7129 ins_pipe(istore_mem);
7130 %}
7131
7132 // Store Integer
7133
7134 instruct storeI(iRegIorL2I src, memory4 mem)
7135 %{
7136 match(Set mem(StoreI mem src));
7137 predicate(!needs_releasing_store(n));
7138
7139 ins_cost(INSN_COST);
7140 format %{ "strw $src, $mem\t# int" %}
7141
7142 ins_encode(aarch64_enc_strw(src, mem));
7143
7144 ins_pipe(istore_reg_mem);
7145 %}
7146
7147 instruct storeimmI0(immI0 zero, memory4 mem)
7148 %{
7149 match(Set mem(StoreI mem zero));
7150 predicate(!needs_releasing_store(n));
7151
7152 ins_cost(INSN_COST);
7153 format %{ "strw zr, $mem\t# int" %}
7154
7155 ins_encode(aarch64_enc_strw0(mem));
7156
7157 ins_pipe(istore_mem);
7158 %}
7159
7160 // Store Long (64 bit signed)
7161 instruct storeL(iRegL src, memory8 mem)
7162 %{
7163 match(Set mem (StoreL mem src));
7164 predicate(!needs_releasing_store(n));
7165
7166 ins_cost(INSN_COST);
7167 format %{ "str $src, $mem\t# int" %}
7168
7169 ins_encode(aarch64_enc_str(src, mem));
7170
7171 ins_pipe(istore_reg_mem);
7172 %}
7173
7174 // Store Long (64 bit signed)
7175 instruct storeimmL0(immL0 zero, memory8 mem)
7176 %{
7177 match(Set mem (StoreL mem zero));
7178 predicate(!needs_releasing_store(n));
7179
7180 ins_cost(INSN_COST);
7181 format %{ "str zr, $mem\t# int" %}
7182
7183 ins_encode(aarch64_enc_str0(mem));
7184
7185 ins_pipe(istore_mem);
7186 %}
7187
7188 // Store Pointer
7189 instruct storeP(iRegP src, memory8 mem)
7190 %{
7191 match(Set mem (StoreP mem src));
7192 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7193
7194 ins_cost(INSN_COST);
7195 format %{ "str $src, $mem\t# ptr" %}
7196
7197 ins_encode(aarch64_enc_str(src, mem));
7198
7199 ins_pipe(istore_reg_mem);
7200 %}
7201
7202 // Store Pointer
7203 instruct storeimmP0(immP0 zero, memory8 mem)
7204 %{
7205 match(Set mem (StoreP mem zero));
7206 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7207
7208 ins_cost(INSN_COST);
7209 format %{ "str zr, $mem\t# ptr" %}
7210
7211 ins_encode(aarch64_enc_str0(mem));
7212
7213 ins_pipe(istore_mem);
7214 %}
7215
7216 // Store Compressed Pointer
7217 instruct storeN(iRegN src, memory4 mem)
7218 %{
7219 match(Set mem (StoreN mem src));
7220 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7221
7222 ins_cost(INSN_COST);
7223 format %{ "strw $src, $mem\t# compressed ptr" %}
7224
7225 ins_encode(aarch64_enc_strw(src, mem));
7226
7227 ins_pipe(istore_reg_mem);
7228 %}
7229
7230 instruct storeImmN0(immN0 zero, memory4 mem)
7231 %{
7232 match(Set mem (StoreN mem zero));
7233 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7234
7235 ins_cost(INSN_COST);
7236 format %{ "strw zr, $mem\t# compressed ptr" %}
7237
7238 ins_encode(aarch64_enc_strw0(mem));
7239
7240 ins_pipe(istore_mem);
7241 %}
7242
7243 // Store Float
7244 instruct storeF(vRegF src, memory4 mem)
7245 %{
7246 match(Set mem (StoreF mem src));
7247 predicate(!needs_releasing_store(n));
7248
7249 ins_cost(INSN_COST);
7250 format %{ "strs $src, $mem\t# float" %}
7251
7252 ins_encode( aarch64_enc_strs(src, mem) );
7253
7254 ins_pipe(pipe_class_memory);
7255 %}
7256
7257 // TODO
7258 // implement storeImmF0 and storeFImmPacked
7259
7260 // Store Double
7261 instruct storeD(vRegD src, memory8 mem)
7262 %{
7263 match(Set mem (StoreD mem src));
7264 predicate(!needs_releasing_store(n));
7265
7266 ins_cost(INSN_COST);
7267 format %{ "strd $src, $mem\t# double" %}
7268
7269 ins_encode( aarch64_enc_strd(src, mem) );
7270
7271 ins_pipe(pipe_class_memory);
7272 %}
7273
7274 // Store Compressed Klass Pointer
7275 instruct storeNKlass(iRegN src, memory4 mem)
7276 %{
7277 predicate(!needs_releasing_store(n));
7278 match(Set mem (StoreNKlass mem src));
7279
7280 ins_cost(INSN_COST);
7281 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7282
7283 ins_encode(aarch64_enc_strw(src, mem));
7284
7285 ins_pipe(istore_reg_mem);
7286 %}
7287
7288 // TODO
7289 // implement storeImmD0 and storeDImmPacked
7290
7291 // prefetch instructions
7292 // Must be safe to execute with invalid address (cannot fault).
7293
7294 instruct prefetchalloc( memory8 mem ) %{
7295 match(PrefetchAllocation mem);
7296
7297 ins_cost(INSN_COST);
7298 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7299
7300 ins_encode( aarch64_enc_prefetchw(mem) );
7301
7302 ins_pipe(iload_prefetch);
7303 %}
7304
7305 // ---------------- volatile loads and stores ----------------
7306
7307 // Load Byte (8 bit signed)
7308 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7309 %{
7310 match(Set dst (LoadB mem));
7311
7312 ins_cost(VOLATILE_REF_COST);
7313 format %{ "ldarsb $dst, $mem\t# byte" %}
7314
7315 ins_encode(aarch64_enc_ldarsb(dst, mem));
7316
7317 ins_pipe(pipe_serial);
7318 %}
7319
7320 // Load Byte (8 bit signed) into long
7321 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7322 %{
7323 match(Set dst (ConvI2L (LoadB mem)));
7324
7325 ins_cost(VOLATILE_REF_COST);
7326 format %{ "ldarsb $dst, $mem\t# byte" %}
7327
7328 ins_encode(aarch64_enc_ldarsb(dst, mem));
7329
7330 ins_pipe(pipe_serial);
7331 %}
7332
7333 // Load Byte (8 bit unsigned)
7334 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7335 %{
7336 match(Set dst (LoadUB mem));
7337
7338 ins_cost(VOLATILE_REF_COST);
7339 format %{ "ldarb $dst, $mem\t# byte" %}
7340
7341 ins_encode(aarch64_enc_ldarb(dst, mem));
7342
7343 ins_pipe(pipe_serial);
7344 %}
7345
7346 // Load Byte (8 bit unsigned) into long
7347 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7348 %{
7349 match(Set dst (ConvI2L (LoadUB mem)));
7350
7351 ins_cost(VOLATILE_REF_COST);
7352 format %{ "ldarb $dst, $mem\t# byte" %}
7353
7354 ins_encode(aarch64_enc_ldarb(dst, mem));
7355
7356 ins_pipe(pipe_serial);
7357 %}
7358
7359 // Load Short (16 bit signed)
7360 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7361 %{
7362 match(Set dst (LoadS mem));
7363
7364 ins_cost(VOLATILE_REF_COST);
7365 format %{ "ldarshw $dst, $mem\t# short" %}
7366
7367 ins_encode(aarch64_enc_ldarshw(dst, mem));
7368
7369 ins_pipe(pipe_serial);
7370 %}
7371
7372 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7373 %{
7374 match(Set dst (LoadUS mem));
7375
7376 ins_cost(VOLATILE_REF_COST);
7377 format %{ "ldarhw $dst, $mem\t# short" %}
7378
7379 ins_encode(aarch64_enc_ldarhw(dst, mem));
7380
7381 ins_pipe(pipe_serial);
7382 %}
7383
7384 // Load Short/Char (16 bit unsigned) into long
7385 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7386 %{
7387 match(Set dst (ConvI2L (LoadUS mem)));
7388
7389 ins_cost(VOLATILE_REF_COST);
7390 format %{ "ldarh $dst, $mem\t# short" %}
7391
7392 ins_encode(aarch64_enc_ldarh(dst, mem));
7393
7394 ins_pipe(pipe_serial);
7395 %}
7396
7397 // Load Short/Char (16 bit signed) into long
7398 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7399 %{
7400 match(Set dst (ConvI2L (LoadS mem)));
7401
7402 ins_cost(VOLATILE_REF_COST);
7403 format %{ "ldarh $dst, $mem\t# short" %}
7404
7405 ins_encode(aarch64_enc_ldarsh(dst, mem));
7406
7407 ins_pipe(pipe_serial);
7408 %}
7409
7410 // Load Integer (32 bit signed)
7411 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7412 %{
7413 match(Set dst (LoadI mem));
7414
7415 ins_cost(VOLATILE_REF_COST);
7416 format %{ "ldarw $dst, $mem\t# int" %}
7417
7418 ins_encode(aarch64_enc_ldarw(dst, mem));
7419
7420 ins_pipe(pipe_serial);
7421 %}
7422
7423 // Load Integer (32 bit unsigned) into long
7424 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7425 %{
7426 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7427
7428 ins_cost(VOLATILE_REF_COST);
7429 format %{ "ldarw $dst, $mem\t# int" %}
7430
7431 ins_encode(aarch64_enc_ldarw(dst, mem));
7432
7433 ins_pipe(pipe_serial);
7434 %}
7435
7436 // Load Long (64 bit signed)
7437 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7438 %{
7439 match(Set dst (LoadL mem));
7440
7441 ins_cost(VOLATILE_REF_COST);
7442 format %{ "ldar $dst, $mem\t# int" %}
7443
7444 ins_encode(aarch64_enc_ldar(dst, mem));
7445
7446 ins_pipe(pipe_serial);
7447 %}
7448
7449 // Load Pointer
7450 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7451 %{
7452 match(Set dst (LoadP mem));
7453 predicate(n->as_Load()->barrier_data() == 0);
7454
7455 ins_cost(VOLATILE_REF_COST);
7456 format %{ "ldar $dst, $mem\t# ptr" %}
7457
7458 ins_encode(aarch64_enc_ldar(dst, mem));
7459
7460 ins_pipe(pipe_serial);
7461 %}
7462
7463 // Load Compressed Pointer
7464 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7465 %{
7466 match(Set dst (LoadN mem));
7467 predicate(n->as_Load()->barrier_data() == 0);
7468
7469 ins_cost(VOLATILE_REF_COST);
7470 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7471
7472 ins_encode(aarch64_enc_ldarw(dst, mem));
7473
7474 ins_pipe(pipe_serial);
7475 %}
7476
7477 // Load Float
7478 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7479 %{
7480 match(Set dst (LoadF mem));
7481
7482 ins_cost(VOLATILE_REF_COST);
7483 format %{ "ldars $dst, $mem\t# float" %}
7484
7485 ins_encode( aarch64_enc_fldars(dst, mem) );
7486
7487 ins_pipe(pipe_serial);
7488 %}
7489
7490 // Load Double
7491 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7492 %{
7493 match(Set dst (LoadD mem));
7494
7495 ins_cost(VOLATILE_REF_COST);
7496 format %{ "ldard $dst, $mem\t# double" %}
7497
7498 ins_encode( aarch64_enc_fldard(dst, mem) );
7499
7500 ins_pipe(pipe_serial);
7501 %}
7502
7503 // Store Byte
7504 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7505 %{
7506 match(Set mem (StoreB mem src));
7507
7508 ins_cost(VOLATILE_REF_COST);
7509 format %{ "stlrb $src, $mem\t# byte" %}
7510
7511 ins_encode(aarch64_enc_stlrb(src, mem));
7512
7513 ins_pipe(pipe_class_memory);
7514 %}
7515
7516 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7517 %{
7518 match(Set mem (StoreB mem zero));
7519
7520 ins_cost(VOLATILE_REF_COST);
7521 format %{ "stlrb zr, $mem\t# byte" %}
7522
7523 ins_encode(aarch64_enc_stlrb0(mem));
7524
7525 ins_pipe(pipe_class_memory);
7526 %}
7527
7528 // Store Char/Short
7529 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7530 %{
7531 match(Set mem (StoreC mem src));
7532
7533 ins_cost(VOLATILE_REF_COST);
7534 format %{ "stlrh $src, $mem\t# short" %}
7535
7536 ins_encode(aarch64_enc_stlrh(src, mem));
7537
7538 ins_pipe(pipe_class_memory);
7539 %}
7540
7541 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7542 %{
7543 match(Set mem (StoreC mem zero));
7544
7545 ins_cost(VOLATILE_REF_COST);
7546 format %{ "stlrh zr, $mem\t# short" %}
7547
7548 ins_encode(aarch64_enc_stlrh0(mem));
7549
7550 ins_pipe(pipe_class_memory);
7551 %}
7552
7553 // Store Integer
7554
7555 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7556 %{
7557 match(Set mem(StoreI mem src));
7558
7559 ins_cost(VOLATILE_REF_COST);
7560 format %{ "stlrw $src, $mem\t# int" %}
7561
7562 ins_encode(aarch64_enc_stlrw(src, mem));
7563
7564 ins_pipe(pipe_class_memory);
7565 %}
7566
7567 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7568 %{
7569 match(Set mem(StoreI mem zero));
7570
7571 ins_cost(VOLATILE_REF_COST);
7572 format %{ "stlrw zr, $mem\t# int" %}
7573
7574 ins_encode(aarch64_enc_stlrw0(mem));
7575
7576 ins_pipe(pipe_class_memory);
7577 %}
7578
7579 // Store Long (64 bit signed)
7580 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7581 %{
7582 match(Set mem (StoreL mem src));
7583
7584 ins_cost(VOLATILE_REF_COST);
7585 format %{ "stlr $src, $mem\t# int" %}
7586
7587 ins_encode(aarch64_enc_stlr(src, mem));
7588
7589 ins_pipe(pipe_class_memory);
7590 %}
7591
7592 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7593 %{
7594 match(Set mem (StoreL mem zero));
7595
7596 ins_cost(VOLATILE_REF_COST);
7597 format %{ "stlr zr, $mem\t# int" %}
7598
7599 ins_encode(aarch64_enc_stlr0(mem));
7600
7601 ins_pipe(pipe_class_memory);
7602 %}
7603
7604 // Store Pointer
7605 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7606 %{
7607 match(Set mem (StoreP mem src));
7608 predicate(n->as_Store()->barrier_data() == 0);
7609
7610 ins_cost(VOLATILE_REF_COST);
7611 format %{ "stlr $src, $mem\t# ptr" %}
7612
7613 ins_encode(aarch64_enc_stlr(src, mem));
7614
7615 ins_pipe(pipe_class_memory);
7616 %}
7617
7618 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7619 %{
7620 match(Set mem (StoreP mem zero));
7621 predicate(n->as_Store()->barrier_data() == 0);
7622
7623 ins_cost(VOLATILE_REF_COST);
7624 format %{ "stlr zr, $mem\t# ptr" %}
7625
7626 ins_encode(aarch64_enc_stlr0(mem));
7627
7628 ins_pipe(pipe_class_memory);
7629 %}
7630
7631 // Store Compressed Pointer
7632 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7633 %{
7634 match(Set mem (StoreN mem src));
7635 predicate(n->as_Store()->barrier_data() == 0);
7636
7637 ins_cost(VOLATILE_REF_COST);
7638 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7639
7640 ins_encode(aarch64_enc_stlrw(src, mem));
7641
7642 ins_pipe(pipe_class_memory);
7643 %}
7644
7645 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7646 %{
7647 match(Set mem (StoreN mem zero));
7648 predicate(n->as_Store()->barrier_data() == 0);
7649
7650 ins_cost(VOLATILE_REF_COST);
7651 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7652
7653 ins_encode(aarch64_enc_stlrw0(mem));
7654
7655 ins_pipe(pipe_class_memory);
7656 %}
7657
7658 // Store Float
7659 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7660 %{
7661 match(Set mem (StoreF mem src));
7662
7663 ins_cost(VOLATILE_REF_COST);
7664 format %{ "stlrs $src, $mem\t# float" %}
7665
7666 ins_encode( aarch64_enc_fstlrs(src, mem) );
7667
7668 ins_pipe(pipe_class_memory);
7669 %}
7670
7671 // TODO
7672 // implement storeImmF0 and storeFImmPacked
7673
7674 // Store Double
7675 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7676 %{
7677 match(Set mem (StoreD mem src));
7678
7679 ins_cost(VOLATILE_REF_COST);
7680 format %{ "stlrd $src, $mem\t# double" %}
7681
7682 ins_encode( aarch64_enc_fstlrd(src, mem) );
7683
7684 ins_pipe(pipe_class_memory);
7685 %}
7686
7687 // ---------------- end of volatile loads and stores ----------------
7688
7689 instruct cacheWB(indirect addr)
7690 %{
7691 predicate(VM_Version::supports_data_cache_line_flush());
7692 match(CacheWB addr);
7693
7694 ins_cost(100);
7695 format %{"cache wb $addr" %}
7696 ins_encode %{
7697 assert($addr->index_position() < 0, "should be");
7698 assert($addr$$disp == 0, "should be");
7699 __ cache_wb(Address($addr$$base$$Register, 0));
7700 %}
7701 ins_pipe(pipe_slow); // XXX
7702 %}
7703
7704 instruct cacheWBPreSync()
7705 %{
7706 predicate(VM_Version::supports_data_cache_line_flush());
7707 match(CacheWBPreSync);
7708
7709 ins_cost(100);
7710 format %{"cache wb presync" %}
7711 ins_encode %{
7712 __ cache_wbsync(true);
7713 %}
7714 ins_pipe(pipe_slow); // XXX
7715 %}
7716
7717 instruct cacheWBPostSync()
7718 %{
7719 predicate(VM_Version::supports_data_cache_line_flush());
7720 match(CacheWBPostSync);
7721
7722 ins_cost(100);
7723 format %{"cache wb postsync" %}
7724 ins_encode %{
7725 __ cache_wbsync(false);
7726 %}
7727 ins_pipe(pipe_slow); // XXX
7728 %}
7729
7730 // ============================================================================
7731 // BSWAP Instructions
7732
7733 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7734 match(Set dst (ReverseBytesI src));
7735
7736 ins_cost(INSN_COST);
7737 format %{ "revw $dst, $src" %}
7738
7739 ins_encode %{
7740 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7741 %}
7742
7743 ins_pipe(ialu_reg);
7744 %}
7745
7746 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7747 match(Set dst (ReverseBytesL src));
7748
7749 ins_cost(INSN_COST);
7750 format %{ "rev $dst, $src" %}
7751
7752 ins_encode %{
7753 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7754 %}
7755
7756 ins_pipe(ialu_reg);
7757 %}
7758
7759 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7760 match(Set dst (ReverseBytesUS src));
7761
7762 ins_cost(INSN_COST);
7763 format %{ "rev16w $dst, $src" %}
7764
7765 ins_encode %{
7766 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7767 %}
7768
7769 ins_pipe(ialu_reg);
7770 %}
7771
7772 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7773 match(Set dst (ReverseBytesS src));
7774
7775 ins_cost(INSN_COST);
7776 format %{ "rev16w $dst, $src\n\t"
7777 "sbfmw $dst, $dst, #0, #15" %}
7778
7779 ins_encode %{
7780 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7781 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7782 %}
7783
7784 ins_pipe(ialu_reg);
7785 %}
7786
7787 // ============================================================================
7788 // Zero Count Instructions
7789
7790 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7791 match(Set dst (CountLeadingZerosI src));
7792
7793 ins_cost(INSN_COST);
7794 format %{ "clzw $dst, $src" %}
7795 ins_encode %{
7796 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7797 %}
7798
7799 ins_pipe(ialu_reg);
7800 %}
7801
7802 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7803 match(Set dst (CountLeadingZerosL src));
7804
7805 ins_cost(INSN_COST);
7806 format %{ "clz $dst, $src" %}
7807 ins_encode %{
7808 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7809 %}
7810
7811 ins_pipe(ialu_reg);
7812 %}
7813
7814 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7815 match(Set dst (CountTrailingZerosI src));
7816
7817 ins_cost(INSN_COST * 2);
7818 format %{ "rbitw $dst, $src\n\t"
7819 "clzw $dst, $dst" %}
7820 ins_encode %{
7821 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7822 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7823 %}
7824
7825 ins_pipe(ialu_reg);
7826 %}
7827
7828 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7829 match(Set dst (CountTrailingZerosL src));
7830
7831 ins_cost(INSN_COST * 2);
7832 format %{ "rbit $dst, $src\n\t"
7833 "clz $dst, $dst" %}
7834 ins_encode %{
7835 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7836 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7837 %}
7838
7839 ins_pipe(ialu_reg);
7840 %}
7841
7842 //---------- Population Count Instructions -------------------------------------
7843 //
7844
7845 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7846 match(Set dst (PopCountI src));
7847 effect(TEMP tmp);
7848 ins_cost(INSN_COST * 13);
7849
7850 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7851 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7852 "addv $tmp, $tmp\t# vector (8B)\n\t"
7853 "mov $dst, $tmp\t# vector (1D)" %}
7854 ins_encode %{
7855 __ fmovs($tmp$$FloatRegister, $src$$Register);
7856 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7857 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7858 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7859 %}
7860
7861 ins_pipe(pipe_class_default);
7862 %}
7863
7864 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7865 match(Set dst (PopCountI (LoadI mem)));
7866 effect(TEMP tmp);
7867 ins_cost(INSN_COST * 13);
7868
7869 format %{ "ldrs $tmp, $mem\n\t"
7870 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7871 "addv $tmp, $tmp\t# vector (8B)\n\t"
7872 "mov $dst, $tmp\t# vector (1D)" %}
7873 ins_encode %{
7874 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7875 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7876 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7877 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7878 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7879 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7880 %}
7881
7882 ins_pipe(pipe_class_default);
7883 %}
7884
7885 // Note: Long.bitCount(long) returns an int.
7886 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7887 match(Set dst (PopCountL src));
7888 effect(TEMP tmp);
7889 ins_cost(INSN_COST * 13);
7890
7891 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7892 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7893 "addv $tmp, $tmp\t# vector (8B)\n\t"
7894 "mov $dst, $tmp\t# vector (1D)" %}
7895 ins_encode %{
7896 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7897 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7898 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7899 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7900 %}
7901
7902 ins_pipe(pipe_class_default);
7903 %}
7904
7905 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7906 match(Set dst (PopCountL (LoadL mem)));
7907 effect(TEMP tmp);
7908 ins_cost(INSN_COST * 13);
7909
7910 format %{ "ldrd $tmp, $mem\n\t"
7911 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7912 "addv $tmp, $tmp\t# vector (8B)\n\t"
7913 "mov $dst, $tmp\t# vector (1D)" %}
7914 ins_encode %{
7915 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7916 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7917 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7918 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7919 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7920 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7921 %}
7922
7923 ins_pipe(pipe_class_default);
7924 %}
7925
7926 // ============================================================================
7927 // VerifyVectorAlignment Instruction
7928
7929 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7930 match(Set addr (VerifyVectorAlignment addr mask));
7931 effect(KILL cr);
7932 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7933 ins_encode %{
7934 Label Lskip;
7935 // check if masked bits of addr are zero
7936 __ tst($addr$$Register, $mask$$constant);
7937 __ br(Assembler::EQ, Lskip);
7938 __ stop("verify_vector_alignment found a misaligned vector memory access");
7939 __ bind(Lskip);
7940 %}
7941 ins_pipe(pipe_slow);
7942 %}
7943
7944 // ============================================================================
7945 // MemBar Instruction
7946
7947 instruct load_fence() %{
7948 match(LoadFence);
7949 ins_cost(VOLATILE_REF_COST);
7950
7951 format %{ "load_fence" %}
7952
7953 ins_encode %{
7954 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7955 %}
7956 ins_pipe(pipe_serial);
7957 %}
7958
7959 instruct unnecessary_membar_acquire() %{
7960 predicate(unnecessary_acquire(n));
7961 match(MemBarAcquire);
7962 ins_cost(0);
7963
7964 format %{ "membar_acquire (elided)" %}
7965
7966 ins_encode %{
7967 __ block_comment("membar_acquire (elided)");
7968 %}
7969
7970 ins_pipe(pipe_class_empty);
7971 %}
7972
7973 instruct membar_acquire() %{
7974 match(MemBarAcquire);
7975 ins_cost(VOLATILE_REF_COST);
7976
7977 format %{ "membar_acquire\n\t"
7978 "dmb ishld" %}
7979
7980 ins_encode %{
7981 __ block_comment("membar_acquire");
7982 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7983 %}
7984
7985 ins_pipe(pipe_serial);
7986 %}
7987
7988
7989 instruct membar_acquire_lock() %{
7990 match(MemBarAcquireLock);
7991 ins_cost(VOLATILE_REF_COST);
7992
7993 format %{ "membar_acquire_lock (elided)" %}
7994
7995 ins_encode %{
7996 __ block_comment("membar_acquire_lock (elided)");
7997 %}
7998
7999 ins_pipe(pipe_serial);
8000 %}
8001
8002 instruct store_fence() %{
8003 match(StoreFence);
8004 ins_cost(VOLATILE_REF_COST);
8005
8006 format %{ "store_fence" %}
8007
8008 ins_encode %{
8009 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8010 %}
8011 ins_pipe(pipe_serial);
8012 %}
8013
8014 instruct unnecessary_membar_release() %{
8015 predicate(unnecessary_release(n));
8016 match(MemBarRelease);
8017 ins_cost(0);
8018
8019 format %{ "membar_release (elided)" %}
8020
8021 ins_encode %{
8022 __ block_comment("membar_release (elided)");
8023 %}
8024 ins_pipe(pipe_serial);
8025 %}
8026
8027 instruct membar_release() %{
8028 match(MemBarRelease);
8029 ins_cost(VOLATILE_REF_COST);
8030
8031 format %{ "membar_release\n\t"
8032 "dmb ishst\n\tdmb ishld" %}
8033
8034 ins_encode %{
8035 __ block_comment("membar_release");
8036 // These will be merged if AlwaysMergeDMB is enabled.
8037 __ membar(Assembler::StoreStore);
8038 __ membar(Assembler::LoadStore);
8039 %}
8040 ins_pipe(pipe_serial);
8041 %}
8042
8043 instruct membar_storestore() %{
8044 match(MemBarStoreStore);
8045 match(StoreStoreFence);
8046 ins_cost(VOLATILE_REF_COST);
8047
8048 format %{ "MEMBAR-store-store" %}
8049
8050 ins_encode %{
8051 __ membar(Assembler::StoreStore);
8052 %}
8053 ins_pipe(pipe_serial);
8054 %}
8055
8056 instruct membar_release_lock() %{
8057 match(MemBarReleaseLock);
8058 ins_cost(VOLATILE_REF_COST);
8059
8060 format %{ "membar_release_lock (elided)" %}
8061
8062 ins_encode %{
8063 __ block_comment("membar_release_lock (elided)");
8064 %}
8065
8066 ins_pipe(pipe_serial);
8067 %}
8068
8069 instruct unnecessary_membar_volatile() %{
8070 predicate(unnecessary_volatile(n));
8071 match(MemBarVolatile);
8072 ins_cost(0);
8073
8074 format %{ "membar_volatile (elided)" %}
8075
8076 ins_encode %{
8077 __ block_comment("membar_volatile (elided)");
8078 %}
8079
8080 ins_pipe(pipe_serial);
8081 %}
8082
8083 instruct membar_volatile() %{
8084 match(MemBarVolatile);
8085 ins_cost(VOLATILE_REF_COST*100);
8086
8087 format %{ "membar_volatile\n\t"
8088 "dmb ish"%}
8089
8090 ins_encode %{
8091 __ block_comment("membar_volatile");
8092 __ membar(Assembler::StoreLoad);
8093 %}
8094
8095 ins_pipe(pipe_serial);
8096 %}
8097
8098 // ============================================================================
8099 // Cast/Convert Instructions
8100
8101 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8102 match(Set dst (CastX2P src));
8103
8104 ins_cost(INSN_COST);
8105 format %{ "mov $dst, $src\t# long -> ptr" %}
8106
8107 ins_encode %{
8108 if ($dst$$reg != $src$$reg) {
8109 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8110 }
8111 %}
8112
8113 ins_pipe(ialu_reg);
8114 %}
8115
8116 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8117 match(Set dst (CastP2X src));
8118
8119 ins_cost(INSN_COST);
8120 format %{ "mov $dst, $src\t# ptr -> long" %}
8121
8122 ins_encode %{
8123 if ($dst$$reg != $src$$reg) {
8124 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8125 }
8126 %}
8127
8128 ins_pipe(ialu_reg);
8129 %}
8130
8131 // Convert oop into int for vectors alignment masking
8132 instruct convP2I(iRegINoSp dst, iRegP src) %{
8133 match(Set dst (ConvL2I (CastP2X src)));
8134
8135 ins_cost(INSN_COST);
8136 format %{ "movw $dst, $src\t# ptr -> int" %}
8137 ins_encode %{
8138 __ movw($dst$$Register, $src$$Register);
8139 %}
8140
8141 ins_pipe(ialu_reg);
8142 %}
8143
8144 // Convert compressed oop into int for vectors alignment masking
8145 // in case of 32bit oops (heap < 4Gb).
8146 instruct convN2I(iRegINoSp dst, iRegN src)
8147 %{
8148 predicate(CompressedOops::shift() == 0);
8149 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8150
8151 ins_cost(INSN_COST);
8152 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8153 ins_encode %{
8154 __ movw($dst$$Register, $src$$Register);
8155 %}
8156
8157 ins_pipe(ialu_reg);
8158 %}
8159
8160
8161 // Convert oop pointer into compressed form
8162 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8163 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8164 match(Set dst (EncodeP src));
8165 effect(KILL cr);
8166 ins_cost(INSN_COST * 3);
8167 format %{ "encode_heap_oop $dst, $src" %}
8168 ins_encode %{
8169 Register s = $src$$Register;
8170 Register d = $dst$$Register;
8171 __ encode_heap_oop(d, s);
8172 %}
8173 ins_pipe(ialu_reg);
8174 %}
8175
8176 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8177 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8178 match(Set dst (EncodeP src));
8179 ins_cost(INSN_COST * 3);
8180 format %{ "encode_heap_oop_not_null $dst, $src" %}
8181 ins_encode %{
8182 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8183 %}
8184 ins_pipe(ialu_reg);
8185 %}
8186
8187 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8188 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8189 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8190 match(Set dst (DecodeN src));
8191 ins_cost(INSN_COST * 3);
8192 format %{ "decode_heap_oop $dst, $src" %}
8193 ins_encode %{
8194 Register s = $src$$Register;
8195 Register d = $dst$$Register;
8196 __ decode_heap_oop(d, s);
8197 %}
8198 ins_pipe(ialu_reg);
8199 %}
8200
8201 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8202 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8203 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8204 match(Set dst (DecodeN src));
8205 ins_cost(INSN_COST * 3);
8206 format %{ "decode_heap_oop_not_null $dst, $src" %}
8207 ins_encode %{
8208 Register s = $src$$Register;
8209 Register d = $dst$$Register;
8210 __ decode_heap_oop_not_null(d, s);
8211 %}
8212 ins_pipe(ialu_reg);
8213 %}
8214
8215 // n.b. AArch64 implementations of encode_klass_not_null and
8216 // decode_klass_not_null do not modify the flags register so, unlike
8217 // Intel, we don't kill CR as a side effect here
8218
8219 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8220 match(Set dst (EncodePKlass src));
8221
8222 ins_cost(INSN_COST * 3);
8223 format %{ "encode_klass_not_null $dst,$src" %}
8224
8225 ins_encode %{
8226 Register src_reg = as_Register($src$$reg);
8227 Register dst_reg = as_Register($dst$$reg);
8228 __ encode_klass_not_null(dst_reg, src_reg);
8229 %}
8230
8231 ins_pipe(ialu_reg);
8232 %}
8233
8234 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8235 match(Set dst (DecodeNKlass src));
8236
8237 ins_cost(INSN_COST * 3);
8238 format %{ "decode_klass_not_null $dst,$src" %}
8239
8240 ins_encode %{
8241 Register src_reg = as_Register($src$$reg);
8242 Register dst_reg = as_Register($dst$$reg);
8243 if (dst_reg != src_reg) {
8244 __ decode_klass_not_null(dst_reg, src_reg);
8245 } else {
8246 __ decode_klass_not_null(dst_reg);
8247 }
8248 %}
8249
8250 ins_pipe(ialu_reg);
8251 %}
8252
8253 instruct checkCastPP(iRegPNoSp dst)
8254 %{
8255 match(Set dst (CheckCastPP dst));
8256
8257 size(0);
8258 format %{ "# checkcastPP of $dst" %}
8259 ins_encode(/* empty encoding */);
8260 ins_pipe(pipe_class_empty);
8261 %}
8262
8263 instruct castPP(iRegPNoSp dst)
8264 %{
8265 match(Set dst (CastPP dst));
8266
8267 size(0);
8268 format %{ "# castPP of $dst" %}
8269 ins_encode(/* empty encoding */);
8270 ins_pipe(pipe_class_empty);
8271 %}
8272
8273 instruct castII(iRegI dst)
8274 %{
8275 predicate(VerifyConstraintCasts == 0);
8276 match(Set dst (CastII dst));
8277
8278 size(0);
8279 format %{ "# castII of $dst" %}
8280 ins_encode(/* empty encoding */);
8281 ins_cost(0);
8282 ins_pipe(pipe_class_empty);
8283 %}
8284
8285 instruct castII_checked(iRegI dst, rFlagsReg cr)
8286 %{
8287 predicate(VerifyConstraintCasts > 0);
8288 match(Set dst (CastII dst));
8289 effect(KILL cr);
8290
8291 format %{ "# castII_checked of $dst" %}
8292 ins_encode %{
8293 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8294 %}
8295 ins_pipe(pipe_slow);
8296 %}
8297
8298 instruct castLL(iRegL dst)
8299 %{
8300 predicate(VerifyConstraintCasts == 0);
8301 match(Set dst (CastLL dst));
8302
8303 size(0);
8304 format %{ "# castLL of $dst" %}
8305 ins_encode(/* empty encoding */);
8306 ins_cost(0);
8307 ins_pipe(pipe_class_empty);
8308 %}
8309
8310 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8311 %{
8312 predicate(VerifyConstraintCasts > 0);
8313 match(Set dst (CastLL dst));
8314 effect(KILL cr);
8315
8316 format %{ "# castLL_checked of $dst" %}
8317 ins_encode %{
8318 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8319 %}
8320 ins_pipe(pipe_slow);
8321 %}
8322
8323 instruct castHH(vRegF dst)
8324 %{
8325 match(Set dst (CastHH dst));
8326 size(0);
8327 format %{ "# castHH of $dst" %}
8328 ins_encode(/* empty encoding */);
8329 ins_cost(0);
8330 ins_pipe(pipe_class_empty);
8331 %}
8332
8333 instruct castFF(vRegF dst)
8334 %{
8335 match(Set dst (CastFF dst));
8336
8337 size(0);
8338 format %{ "# castFF of $dst" %}
8339 ins_encode(/* empty encoding */);
8340 ins_cost(0);
8341 ins_pipe(pipe_class_empty);
8342 %}
8343
8344 instruct castDD(vRegD dst)
8345 %{
8346 match(Set dst (CastDD dst));
8347
8348 size(0);
8349 format %{ "# castDD of $dst" %}
8350 ins_encode(/* empty encoding */);
8351 ins_cost(0);
8352 ins_pipe(pipe_class_empty);
8353 %}
8354
8355 instruct castVV(vReg dst)
8356 %{
8357 match(Set dst (CastVV dst));
8358
8359 size(0);
8360 format %{ "# castVV of $dst" %}
8361 ins_encode(/* empty encoding */);
8362 ins_cost(0);
8363 ins_pipe(pipe_class_empty);
8364 %}
8365
8366 instruct castVVMask(pRegGov dst)
8367 %{
8368 match(Set dst (CastVV dst));
8369
8370 size(0);
8371 format %{ "# castVV of $dst" %}
8372 ins_encode(/* empty encoding */);
8373 ins_cost(0);
8374 ins_pipe(pipe_class_empty);
8375 %}
8376
8377 // ============================================================================
8378 // Atomic operation instructions
8379 //
8380
8381 // standard CompareAndSwapX when we are using barriers
8382 // these have higher priority than the rules selected by a predicate
8383
8384 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8385 // can't match them
8386
8387 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8388
8389 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8390 ins_cost(2 * VOLATILE_REF_COST);
8391
8392 effect(KILL cr);
8393
8394 format %{
8395 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8396 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8397 %}
8398
8399 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8400 aarch64_enc_cset_eq(res));
8401
8402 ins_pipe(pipe_slow);
8403 %}
8404
8405 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8406
8407 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8408 ins_cost(2 * VOLATILE_REF_COST);
8409
8410 effect(KILL cr);
8411
8412 format %{
8413 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8414 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8415 %}
8416
8417 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8418 aarch64_enc_cset_eq(res));
8419
8420 ins_pipe(pipe_slow);
8421 %}
8422
8423 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8424
8425 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8426 ins_cost(2 * VOLATILE_REF_COST);
8427
8428 effect(KILL cr);
8429
8430 format %{
8431 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8432 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8433 %}
8434
8435 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8436 aarch64_enc_cset_eq(res));
8437
8438 ins_pipe(pipe_slow);
8439 %}
8440
8441 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8442
8443 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8444 ins_cost(2 * VOLATILE_REF_COST);
8445
8446 effect(KILL cr);
8447
8448 format %{
8449 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8450 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8451 %}
8452
8453 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8454 aarch64_enc_cset_eq(res));
8455
8456 ins_pipe(pipe_slow);
8457 %}
8458
8459 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8460
8461 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8462 predicate(n->as_LoadStore()->barrier_data() == 0);
8463 ins_cost(2 * VOLATILE_REF_COST);
8464
8465 effect(KILL cr);
8466
8467 format %{
8468 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8469 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8470 %}
8471
8472 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8473 aarch64_enc_cset_eq(res));
8474
8475 ins_pipe(pipe_slow);
8476 %}
8477
8478 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8479
8480 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8481 predicate(n->as_LoadStore()->barrier_data() == 0);
8482 ins_cost(2 * VOLATILE_REF_COST);
8483
8484 effect(KILL cr);
8485
8486 format %{
8487 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8488 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8489 %}
8490
8491 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8492 aarch64_enc_cset_eq(res));
8493
8494 ins_pipe(pipe_slow);
8495 %}
8496
8497 // alternative CompareAndSwapX when we are eliding barriers
8498
8499 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8500
8501 predicate(needs_acquiring_load_exclusive(n));
8502 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8503 ins_cost(VOLATILE_REF_COST);
8504
8505 effect(KILL cr);
8506
8507 format %{
8508 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8509 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8510 %}
8511
8512 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8513 aarch64_enc_cset_eq(res));
8514
8515 ins_pipe(pipe_slow);
8516 %}
8517
8518 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8519
8520 predicate(needs_acquiring_load_exclusive(n));
8521 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8522 ins_cost(VOLATILE_REF_COST);
8523
8524 effect(KILL cr);
8525
8526 format %{
8527 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8528 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8529 %}
8530
8531 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8532 aarch64_enc_cset_eq(res));
8533
8534 ins_pipe(pipe_slow);
8535 %}
8536
8537 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8538
8539 predicate(needs_acquiring_load_exclusive(n));
8540 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8541 ins_cost(VOLATILE_REF_COST);
8542
8543 effect(KILL cr);
8544
8545 format %{
8546 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8547 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8548 %}
8549
8550 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8551 aarch64_enc_cset_eq(res));
8552
8553 ins_pipe(pipe_slow);
8554 %}
8555
8556 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8557
8558 predicate(needs_acquiring_load_exclusive(n));
8559 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8560 ins_cost(VOLATILE_REF_COST);
8561
8562 effect(KILL cr);
8563
8564 format %{
8565 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8566 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8567 %}
8568
8569 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8570 aarch64_enc_cset_eq(res));
8571
8572 ins_pipe(pipe_slow);
8573 %}
8574
8575 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8576
8577 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8578 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8579 ins_cost(VOLATILE_REF_COST);
8580
8581 effect(KILL cr);
8582
8583 format %{
8584 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8585 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8586 %}
8587
8588 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8589 aarch64_enc_cset_eq(res));
8590
8591 ins_pipe(pipe_slow);
8592 %}
8593
8594 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8595
8596 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8597 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8598 ins_cost(VOLATILE_REF_COST);
8599
8600 effect(KILL cr);
8601
8602 format %{
8603 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8604 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8605 %}
8606
8607 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8608 aarch64_enc_cset_eq(res));
8609
8610 ins_pipe(pipe_slow);
8611 %}
8612
8613
8614 // ---------------------------------------------------------------------
8615
8616 // BEGIN This section of the file is automatically generated. Do not edit --------------
8617
8618 // Sundry CAS operations. Note that release is always true,
8619 // regardless of the memory ordering of the CAS. This is because we
8620 // need the volatile case to be sequentially consistent but there is
8621 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8622 // can't check the type of memory ordering here, so we always emit a
8623 // STLXR.
8624
8625 // This section is generated from cas.m4
8626
8627
8628 // This pattern is generated automatically from cas.m4.
8629 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8630 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8631 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8632 ins_cost(2 * VOLATILE_REF_COST);
8633 effect(TEMP_DEF res, KILL cr);
8634 format %{
8635 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8636 %}
8637 ins_encode %{
8638 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8639 Assembler::byte, /*acquire*/ false, /*release*/ true,
8640 /*weak*/ false, $res$$Register);
8641 __ sxtbw($res$$Register, $res$$Register);
8642 %}
8643 ins_pipe(pipe_slow);
8644 %}
8645
8646 // This pattern is generated automatically from cas.m4.
8647 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8648 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8649 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8650 ins_cost(2 * VOLATILE_REF_COST);
8651 effect(TEMP_DEF res, KILL cr);
8652 format %{
8653 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8654 %}
8655 ins_encode %{
8656 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8657 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8658 /*weak*/ false, $res$$Register);
8659 __ sxthw($res$$Register, $res$$Register);
8660 %}
8661 ins_pipe(pipe_slow);
8662 %}
8663
8664 // This pattern is generated automatically from cas.m4.
8665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8666 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8667 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8668 ins_cost(2 * VOLATILE_REF_COST);
8669 effect(TEMP_DEF res, KILL cr);
8670 format %{
8671 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8672 %}
8673 ins_encode %{
8674 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8675 Assembler::word, /*acquire*/ false, /*release*/ true,
8676 /*weak*/ false, $res$$Register);
8677 %}
8678 ins_pipe(pipe_slow);
8679 %}
8680
8681 // This pattern is generated automatically from cas.m4.
8682 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8683 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8684 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8685 ins_cost(2 * VOLATILE_REF_COST);
8686 effect(TEMP_DEF res, KILL cr);
8687 format %{
8688 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8689 %}
8690 ins_encode %{
8691 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8692 Assembler::xword, /*acquire*/ false, /*release*/ true,
8693 /*weak*/ false, $res$$Register);
8694 %}
8695 ins_pipe(pipe_slow);
8696 %}
8697
8698 // This pattern is generated automatically from cas.m4.
8699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8700 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8701 predicate(n->as_LoadStore()->barrier_data() == 0);
8702 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8703 ins_cost(2 * VOLATILE_REF_COST);
8704 effect(TEMP_DEF res, KILL cr);
8705 format %{
8706 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8707 %}
8708 ins_encode %{
8709 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8710 Assembler::word, /*acquire*/ false, /*release*/ true,
8711 /*weak*/ false, $res$$Register);
8712 %}
8713 ins_pipe(pipe_slow);
8714 %}
8715
8716 // This pattern is generated automatically from cas.m4.
8717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8718 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8719 predicate(n->as_LoadStore()->barrier_data() == 0);
8720 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8721 ins_cost(2 * VOLATILE_REF_COST);
8722 effect(TEMP_DEF res, KILL cr);
8723 format %{
8724 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8725 %}
8726 ins_encode %{
8727 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8728 Assembler::xword, /*acquire*/ false, /*release*/ true,
8729 /*weak*/ false, $res$$Register);
8730 %}
8731 ins_pipe(pipe_slow);
8732 %}
8733
8734 // This pattern is generated automatically from cas.m4.
8735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8736 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8737 predicate(needs_acquiring_load_exclusive(n));
8738 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8739 ins_cost(VOLATILE_REF_COST);
8740 effect(TEMP_DEF res, KILL cr);
8741 format %{
8742 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8743 %}
8744 ins_encode %{
8745 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8746 Assembler::byte, /*acquire*/ true, /*release*/ true,
8747 /*weak*/ false, $res$$Register);
8748 __ sxtbw($res$$Register, $res$$Register);
8749 %}
8750 ins_pipe(pipe_slow);
8751 %}
8752
8753 // This pattern is generated automatically from cas.m4.
8754 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8755 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8756 predicate(needs_acquiring_load_exclusive(n));
8757 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8758 ins_cost(VOLATILE_REF_COST);
8759 effect(TEMP_DEF res, KILL cr);
8760 format %{
8761 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8762 %}
8763 ins_encode %{
8764 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8765 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8766 /*weak*/ false, $res$$Register);
8767 __ sxthw($res$$Register, $res$$Register);
8768 %}
8769 ins_pipe(pipe_slow);
8770 %}
8771
8772 // This pattern is generated automatically from cas.m4.
8773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8774 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8775 predicate(needs_acquiring_load_exclusive(n));
8776 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8777 ins_cost(VOLATILE_REF_COST);
8778 effect(TEMP_DEF res, KILL cr);
8779 format %{
8780 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8781 %}
8782 ins_encode %{
8783 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8784 Assembler::word, /*acquire*/ true, /*release*/ true,
8785 /*weak*/ false, $res$$Register);
8786 %}
8787 ins_pipe(pipe_slow);
8788 %}
8789
8790 // This pattern is generated automatically from cas.m4.
8791 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8792 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8793 predicate(needs_acquiring_load_exclusive(n));
8794 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8795 ins_cost(VOLATILE_REF_COST);
8796 effect(TEMP_DEF res, KILL cr);
8797 format %{
8798 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8799 %}
8800 ins_encode %{
8801 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8802 Assembler::xword, /*acquire*/ true, /*release*/ true,
8803 /*weak*/ false, $res$$Register);
8804 %}
8805 ins_pipe(pipe_slow);
8806 %}
8807
8808 // This pattern is generated automatically from cas.m4.
8809 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8810 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8811 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8812 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8813 ins_cost(VOLATILE_REF_COST);
8814 effect(TEMP_DEF res, KILL cr);
8815 format %{
8816 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8817 %}
8818 ins_encode %{
8819 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8820 Assembler::word, /*acquire*/ true, /*release*/ true,
8821 /*weak*/ false, $res$$Register);
8822 %}
8823 ins_pipe(pipe_slow);
8824 %}
8825
8826 // This pattern is generated automatically from cas.m4.
8827 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8828 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8829 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8830 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8831 ins_cost(VOLATILE_REF_COST);
8832 effect(TEMP_DEF res, KILL cr);
8833 format %{
8834 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8835 %}
8836 ins_encode %{
8837 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8838 Assembler::xword, /*acquire*/ true, /*release*/ true,
8839 /*weak*/ false, $res$$Register);
8840 %}
8841 ins_pipe(pipe_slow);
8842 %}
8843
8844 // This pattern is generated automatically from cas.m4.
8845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8846 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8847 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8848 ins_cost(2 * VOLATILE_REF_COST);
8849 effect(KILL cr);
8850 format %{
8851 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8852 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8853 %}
8854 ins_encode %{
8855 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8856 Assembler::byte, /*acquire*/ false, /*release*/ true,
8857 /*weak*/ true, noreg);
8858 __ csetw($res$$Register, Assembler::EQ);
8859 %}
8860 ins_pipe(pipe_slow);
8861 %}
8862
8863 // This pattern is generated automatically from cas.m4.
8864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8865 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8866 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8867 ins_cost(2 * VOLATILE_REF_COST);
8868 effect(KILL cr);
8869 format %{
8870 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8871 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8872 %}
8873 ins_encode %{
8874 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8875 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8876 /*weak*/ true, noreg);
8877 __ csetw($res$$Register, Assembler::EQ);
8878 %}
8879 ins_pipe(pipe_slow);
8880 %}
8881
8882 // This pattern is generated automatically from cas.m4.
8883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8884 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8885 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8886 ins_cost(2 * VOLATILE_REF_COST);
8887 effect(KILL cr);
8888 format %{
8889 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8890 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8891 %}
8892 ins_encode %{
8893 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8894 Assembler::word, /*acquire*/ false, /*release*/ true,
8895 /*weak*/ true, noreg);
8896 __ csetw($res$$Register, Assembler::EQ);
8897 %}
8898 ins_pipe(pipe_slow);
8899 %}
8900
8901 // This pattern is generated automatically from cas.m4.
8902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8903 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8904 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8905 ins_cost(2 * VOLATILE_REF_COST);
8906 effect(KILL cr);
8907 format %{
8908 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8909 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8910 %}
8911 ins_encode %{
8912 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8913 Assembler::xword, /*acquire*/ false, /*release*/ true,
8914 /*weak*/ true, noreg);
8915 __ csetw($res$$Register, Assembler::EQ);
8916 %}
8917 ins_pipe(pipe_slow);
8918 %}
8919
8920 // This pattern is generated automatically from cas.m4.
8921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8922 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8923 predicate(n->as_LoadStore()->barrier_data() == 0);
8924 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8925 ins_cost(2 * VOLATILE_REF_COST);
8926 effect(KILL cr);
8927 format %{
8928 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8929 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8930 %}
8931 ins_encode %{
8932 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8933 Assembler::word, /*acquire*/ false, /*release*/ true,
8934 /*weak*/ true, noreg);
8935 __ csetw($res$$Register, Assembler::EQ);
8936 %}
8937 ins_pipe(pipe_slow);
8938 %}
8939
8940 // This pattern is generated automatically from cas.m4.
8941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8942 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8943 predicate(n->as_LoadStore()->barrier_data() == 0);
8944 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8945 ins_cost(2 * VOLATILE_REF_COST);
8946 effect(KILL cr);
8947 format %{
8948 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8949 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8950 %}
8951 ins_encode %{
8952 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8953 Assembler::xword, /*acquire*/ false, /*release*/ true,
8954 /*weak*/ true, noreg);
8955 __ csetw($res$$Register, Assembler::EQ);
8956 %}
8957 ins_pipe(pipe_slow);
8958 %}
8959
8960 // This pattern is generated automatically from cas.m4.
8961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8962 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8963 predicate(needs_acquiring_load_exclusive(n));
8964 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8965 ins_cost(VOLATILE_REF_COST);
8966 effect(KILL cr);
8967 format %{
8968 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8969 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8970 %}
8971 ins_encode %{
8972 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8973 Assembler::byte, /*acquire*/ true, /*release*/ true,
8974 /*weak*/ true, noreg);
8975 __ csetw($res$$Register, Assembler::EQ);
8976 %}
8977 ins_pipe(pipe_slow);
8978 %}
8979
8980 // This pattern is generated automatically from cas.m4.
8981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8982 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8983 predicate(needs_acquiring_load_exclusive(n));
8984 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8985 ins_cost(VOLATILE_REF_COST);
8986 effect(KILL cr);
8987 format %{
8988 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8989 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8990 %}
8991 ins_encode %{
8992 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8993 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8994 /*weak*/ true, noreg);
8995 __ csetw($res$$Register, Assembler::EQ);
8996 %}
8997 ins_pipe(pipe_slow);
8998 %}
8999
9000 // This pattern is generated automatically from cas.m4.
9001 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9002 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9003 predicate(needs_acquiring_load_exclusive(n));
9004 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9005 ins_cost(VOLATILE_REF_COST);
9006 effect(KILL cr);
9007 format %{
9008 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9009 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9010 %}
9011 ins_encode %{
9012 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9013 Assembler::word, /*acquire*/ true, /*release*/ true,
9014 /*weak*/ true, noreg);
9015 __ csetw($res$$Register, Assembler::EQ);
9016 %}
9017 ins_pipe(pipe_slow);
9018 %}
9019
9020 // This pattern is generated automatically from cas.m4.
9021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9022 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9023 predicate(needs_acquiring_load_exclusive(n));
9024 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9025 ins_cost(VOLATILE_REF_COST);
9026 effect(KILL cr);
9027 format %{
9028 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9029 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9030 %}
9031 ins_encode %{
9032 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9033 Assembler::xword, /*acquire*/ true, /*release*/ true,
9034 /*weak*/ true, noreg);
9035 __ csetw($res$$Register, Assembler::EQ);
9036 %}
9037 ins_pipe(pipe_slow);
9038 %}
9039
9040 // This pattern is generated automatically from cas.m4.
9041 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9042 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9043 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9044 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9045 ins_cost(VOLATILE_REF_COST);
9046 effect(KILL cr);
9047 format %{
9048 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9049 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9050 %}
9051 ins_encode %{
9052 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9053 Assembler::word, /*acquire*/ true, /*release*/ true,
9054 /*weak*/ true, noreg);
9055 __ csetw($res$$Register, Assembler::EQ);
9056 %}
9057 ins_pipe(pipe_slow);
9058 %}
9059
9060 // This pattern is generated automatically from cas.m4.
9061 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9062 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9063 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9064 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9065 ins_cost(VOLATILE_REF_COST);
9066 effect(KILL cr);
9067 format %{
9068 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9069 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9070 %}
9071 ins_encode %{
9072 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9073 Assembler::xword, /*acquire*/ true, /*release*/ true,
9074 /*weak*/ true, noreg);
9075 __ csetw($res$$Register, Assembler::EQ);
9076 %}
9077 ins_pipe(pipe_slow);
9078 %}
9079
9080 // END This section of the file is automatically generated. Do not edit --------------
9081 // ---------------------------------------------------------------------
9082
9083 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9084 match(Set prev (GetAndSetI mem newv));
9085 ins_cost(2 * VOLATILE_REF_COST);
9086 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9087 ins_encode %{
9088 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9089 %}
9090 ins_pipe(pipe_serial);
9091 %}
9092
9093 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9094 match(Set prev (GetAndSetL mem newv));
9095 ins_cost(2 * VOLATILE_REF_COST);
9096 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9097 ins_encode %{
9098 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9099 %}
9100 ins_pipe(pipe_serial);
9101 %}
9102
9103 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9104 predicate(n->as_LoadStore()->barrier_data() == 0);
9105 match(Set prev (GetAndSetN mem newv));
9106 ins_cost(2 * VOLATILE_REF_COST);
9107 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9108 ins_encode %{
9109 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9110 %}
9111 ins_pipe(pipe_serial);
9112 %}
9113
9114 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9115 predicate(n->as_LoadStore()->barrier_data() == 0);
9116 match(Set prev (GetAndSetP mem newv));
9117 ins_cost(2 * VOLATILE_REF_COST);
9118 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9119 ins_encode %{
9120 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9121 %}
9122 ins_pipe(pipe_serial);
9123 %}
9124
9125 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9126 predicate(needs_acquiring_load_exclusive(n));
9127 match(Set prev (GetAndSetI mem newv));
9128 ins_cost(VOLATILE_REF_COST);
9129 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9130 ins_encode %{
9131 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9132 %}
9133 ins_pipe(pipe_serial);
9134 %}
9135
9136 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9137 predicate(needs_acquiring_load_exclusive(n));
9138 match(Set prev (GetAndSetL mem newv));
9139 ins_cost(VOLATILE_REF_COST);
9140 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9141 ins_encode %{
9142 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9143 %}
9144 ins_pipe(pipe_serial);
9145 %}
9146
9147 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9148 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9149 match(Set prev (GetAndSetN mem newv));
9150 ins_cost(VOLATILE_REF_COST);
9151 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9152 ins_encode %{
9153 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9154 %}
9155 ins_pipe(pipe_serial);
9156 %}
9157
9158 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9159 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9160 match(Set prev (GetAndSetP mem newv));
9161 ins_cost(VOLATILE_REF_COST);
9162 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9163 ins_encode %{
9164 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9165 %}
9166 ins_pipe(pipe_serial);
9167 %}
9168
9169
9170 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9171 match(Set newval (GetAndAddL mem incr));
9172 ins_cost(2 * VOLATILE_REF_COST + 1);
9173 format %{ "get_and_addL $newval, [$mem], $incr" %}
9174 ins_encode %{
9175 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9176 %}
9177 ins_pipe(pipe_serial);
9178 %}
9179
9180 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9181 predicate(n->as_LoadStore()->result_not_used());
9182 match(Set dummy (GetAndAddL mem incr));
9183 ins_cost(2 * VOLATILE_REF_COST);
9184 format %{ "get_and_addL [$mem], $incr" %}
9185 ins_encode %{
9186 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9187 %}
9188 ins_pipe(pipe_serial);
9189 %}
9190
9191 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9192 match(Set newval (GetAndAddL mem incr));
9193 ins_cost(2 * VOLATILE_REF_COST + 1);
9194 format %{ "get_and_addL $newval, [$mem], $incr" %}
9195 ins_encode %{
9196 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9197 %}
9198 ins_pipe(pipe_serial);
9199 %}
9200
9201 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9202 predicate(n->as_LoadStore()->result_not_used());
9203 match(Set dummy (GetAndAddL mem incr));
9204 ins_cost(2 * VOLATILE_REF_COST);
9205 format %{ "get_and_addL [$mem], $incr" %}
9206 ins_encode %{
9207 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9208 %}
9209 ins_pipe(pipe_serial);
9210 %}
9211
9212 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9213 match(Set newval (GetAndAddI mem incr));
9214 ins_cost(2 * VOLATILE_REF_COST + 1);
9215 format %{ "get_and_addI $newval, [$mem], $incr" %}
9216 ins_encode %{
9217 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9218 %}
9219 ins_pipe(pipe_serial);
9220 %}
9221
9222 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9223 predicate(n->as_LoadStore()->result_not_used());
9224 match(Set dummy (GetAndAddI mem incr));
9225 ins_cost(2 * VOLATILE_REF_COST);
9226 format %{ "get_and_addI [$mem], $incr" %}
9227 ins_encode %{
9228 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9229 %}
9230 ins_pipe(pipe_serial);
9231 %}
9232
9233 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9234 match(Set newval (GetAndAddI mem incr));
9235 ins_cost(2 * VOLATILE_REF_COST + 1);
9236 format %{ "get_and_addI $newval, [$mem], $incr" %}
9237 ins_encode %{
9238 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9239 %}
9240 ins_pipe(pipe_serial);
9241 %}
9242
9243 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9244 predicate(n->as_LoadStore()->result_not_used());
9245 match(Set dummy (GetAndAddI mem incr));
9246 ins_cost(2 * VOLATILE_REF_COST);
9247 format %{ "get_and_addI [$mem], $incr" %}
9248 ins_encode %{
9249 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9250 %}
9251 ins_pipe(pipe_serial);
9252 %}
9253
9254 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9255 predicate(needs_acquiring_load_exclusive(n));
9256 match(Set newval (GetAndAddL mem incr));
9257 ins_cost(VOLATILE_REF_COST + 1);
9258 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9259 ins_encode %{
9260 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9261 %}
9262 ins_pipe(pipe_serial);
9263 %}
9264
9265 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9266 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9267 match(Set dummy (GetAndAddL mem incr));
9268 ins_cost(VOLATILE_REF_COST);
9269 format %{ "get_and_addL_acq [$mem], $incr" %}
9270 ins_encode %{
9271 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9272 %}
9273 ins_pipe(pipe_serial);
9274 %}
9275
9276 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9277 predicate(needs_acquiring_load_exclusive(n));
9278 match(Set newval (GetAndAddL mem incr));
9279 ins_cost(VOLATILE_REF_COST + 1);
9280 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9281 ins_encode %{
9282 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9283 %}
9284 ins_pipe(pipe_serial);
9285 %}
9286
9287 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9288 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9289 match(Set dummy (GetAndAddL mem incr));
9290 ins_cost(VOLATILE_REF_COST);
9291 format %{ "get_and_addL_acq [$mem], $incr" %}
9292 ins_encode %{
9293 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9294 %}
9295 ins_pipe(pipe_serial);
9296 %}
9297
9298 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9299 predicate(needs_acquiring_load_exclusive(n));
9300 match(Set newval (GetAndAddI mem incr));
9301 ins_cost(VOLATILE_REF_COST + 1);
9302 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9303 ins_encode %{
9304 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9305 %}
9306 ins_pipe(pipe_serial);
9307 %}
9308
9309 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9310 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9311 match(Set dummy (GetAndAddI mem incr));
9312 ins_cost(VOLATILE_REF_COST);
9313 format %{ "get_and_addI_acq [$mem], $incr" %}
9314 ins_encode %{
9315 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9316 %}
9317 ins_pipe(pipe_serial);
9318 %}
9319
9320 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9321 predicate(needs_acquiring_load_exclusive(n));
9322 match(Set newval (GetAndAddI mem incr));
9323 ins_cost(VOLATILE_REF_COST + 1);
9324 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9325 ins_encode %{
9326 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9327 %}
9328 ins_pipe(pipe_serial);
9329 %}
9330
9331 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9332 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9333 match(Set dummy (GetAndAddI mem incr));
9334 ins_cost(VOLATILE_REF_COST);
9335 format %{ "get_and_addI_acq [$mem], $incr" %}
9336 ins_encode %{
9337 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9338 %}
9339 ins_pipe(pipe_serial);
9340 %}
9341
9342 // Manifest a CmpU result in an integer register.
9343 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9344 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9345 %{
9346 match(Set dst (CmpU3 src1 src2));
9347 effect(KILL flags);
9348
9349 ins_cost(INSN_COST * 3);
9350 format %{
9351 "cmpw $src1, $src2\n\t"
9352 "csetw $dst, ne\n\t"
9353 "cnegw $dst, lo\t# CmpU3(reg)"
9354 %}
9355 ins_encode %{
9356 __ cmpw($src1$$Register, $src2$$Register);
9357 __ csetw($dst$$Register, Assembler::NE);
9358 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9359 %}
9360
9361 ins_pipe(pipe_class_default);
9362 %}
9363
9364 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9365 %{
9366 match(Set dst (CmpU3 src1 src2));
9367 effect(KILL flags);
9368
9369 ins_cost(INSN_COST * 3);
9370 format %{
9371 "subsw zr, $src1, $src2\n\t"
9372 "csetw $dst, ne\n\t"
9373 "cnegw $dst, lo\t# CmpU3(imm)"
9374 %}
9375 ins_encode %{
9376 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9377 __ csetw($dst$$Register, Assembler::NE);
9378 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9379 %}
9380
9381 ins_pipe(pipe_class_default);
9382 %}
9383
9384 // Manifest a CmpUL result in an integer register.
9385 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9386 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9387 %{
9388 match(Set dst (CmpUL3 src1 src2));
9389 effect(KILL flags);
9390
9391 ins_cost(INSN_COST * 3);
9392 format %{
9393 "cmp $src1, $src2\n\t"
9394 "csetw $dst, ne\n\t"
9395 "cnegw $dst, lo\t# CmpUL3(reg)"
9396 %}
9397 ins_encode %{
9398 __ cmp($src1$$Register, $src2$$Register);
9399 __ csetw($dst$$Register, Assembler::NE);
9400 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9401 %}
9402
9403 ins_pipe(pipe_class_default);
9404 %}
9405
9406 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9407 %{
9408 match(Set dst (CmpUL3 src1 src2));
9409 effect(KILL flags);
9410
9411 ins_cost(INSN_COST * 3);
9412 format %{
9413 "subs zr, $src1, $src2\n\t"
9414 "csetw $dst, ne\n\t"
9415 "cnegw $dst, lo\t# CmpUL3(imm)"
9416 %}
9417 ins_encode %{
9418 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9419 __ csetw($dst$$Register, Assembler::NE);
9420 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9421 %}
9422
9423 ins_pipe(pipe_class_default);
9424 %}
9425
9426 // Manifest a CmpL result in an integer register.
9427 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9428 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9429 %{
9430 match(Set dst (CmpL3 src1 src2));
9431 effect(KILL flags);
9432
9433 ins_cost(INSN_COST * 3);
9434 format %{
9435 "cmp $src1, $src2\n\t"
9436 "csetw $dst, ne\n\t"
9437 "cnegw $dst, lt\t# CmpL3(reg)"
9438 %}
9439 ins_encode %{
9440 __ cmp($src1$$Register, $src2$$Register);
9441 __ csetw($dst$$Register, Assembler::NE);
9442 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9443 %}
9444
9445 ins_pipe(pipe_class_default);
9446 %}
9447
9448 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9449 %{
9450 match(Set dst (CmpL3 src1 src2));
9451 effect(KILL flags);
9452
9453 ins_cost(INSN_COST * 3);
9454 format %{
9455 "subs zr, $src1, $src2\n\t"
9456 "csetw $dst, ne\n\t"
9457 "cnegw $dst, lt\t# CmpL3(imm)"
9458 %}
9459 ins_encode %{
9460 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9461 __ csetw($dst$$Register, Assembler::NE);
9462 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9463 %}
9464
9465 ins_pipe(pipe_class_default);
9466 %}
9467
9468 // ============================================================================
9469 // Conditional Move Instructions
9470
9471 // n.b. we have identical rules for both a signed compare op (cmpOp)
9472 // and an unsigned compare op (cmpOpU). it would be nice if we could
9473 // define an op class which merged both inputs and use it to type the
9474 // argument to a single rule. unfortunatelyt his fails because the
9475 // opclass does not live up to the COND_INTER interface of its
9476 // component operands. When the generic code tries to negate the
9477 // operand it ends up running the generci Machoper::negate method
9478 // which throws a ShouldNotHappen. So, we have to provide two flavours
9479 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9480
9481 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9482 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9483
9484 ins_cost(INSN_COST * 2);
9485 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9486
9487 ins_encode %{
9488 __ cselw(as_Register($dst$$reg),
9489 as_Register($src2$$reg),
9490 as_Register($src1$$reg),
9491 (Assembler::Condition)$cmp$$cmpcode);
9492 %}
9493
9494 ins_pipe(icond_reg_reg);
9495 %}
9496
9497 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9498 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9499
9500 ins_cost(INSN_COST * 2);
9501 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9502
9503 ins_encode %{
9504 __ cselw(as_Register($dst$$reg),
9505 as_Register($src2$$reg),
9506 as_Register($src1$$reg),
9507 (Assembler::Condition)$cmp$$cmpcode);
9508 %}
9509
9510 ins_pipe(icond_reg_reg);
9511 %}
9512
9513 // special cases where one arg is zero
9514
9515 // n.b. this is selected in preference to the rule above because it
9516 // avoids loading constant 0 into a source register
9517
9518 // TODO
9519 // we ought only to be able to cull one of these variants as the ideal
9520 // transforms ought always to order the zero consistently (to left/right?)
9521
9522 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9523 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9524
9525 ins_cost(INSN_COST * 2);
9526 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9527
9528 ins_encode %{
9529 __ cselw(as_Register($dst$$reg),
9530 as_Register($src$$reg),
9531 zr,
9532 (Assembler::Condition)$cmp$$cmpcode);
9533 %}
9534
9535 ins_pipe(icond_reg);
9536 %}
9537
9538 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9539 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9540
9541 ins_cost(INSN_COST * 2);
9542 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9543
9544 ins_encode %{
9545 __ cselw(as_Register($dst$$reg),
9546 as_Register($src$$reg),
9547 zr,
9548 (Assembler::Condition)$cmp$$cmpcode);
9549 %}
9550
9551 ins_pipe(icond_reg);
9552 %}
9553
9554 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9555 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9556
9557 ins_cost(INSN_COST * 2);
9558 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9559
9560 ins_encode %{
9561 __ cselw(as_Register($dst$$reg),
9562 zr,
9563 as_Register($src$$reg),
9564 (Assembler::Condition)$cmp$$cmpcode);
9565 %}
9566
9567 ins_pipe(icond_reg);
9568 %}
9569
9570 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9571 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9572
9573 ins_cost(INSN_COST * 2);
9574 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9575
9576 ins_encode %{
9577 __ cselw(as_Register($dst$$reg),
9578 zr,
9579 as_Register($src$$reg),
9580 (Assembler::Condition)$cmp$$cmpcode);
9581 %}
9582
9583 ins_pipe(icond_reg);
9584 %}
9585
9586 // special case for creating a boolean 0 or 1
9587
9588 // n.b. this is selected in preference to the rule above because it
9589 // avoids loading constants 0 and 1 into a source register
9590
9591 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9592 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9593
9594 ins_cost(INSN_COST * 2);
9595 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9596
9597 ins_encode %{
9598 // equivalently
9599 // cset(as_Register($dst$$reg),
9600 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9601 __ csincw(as_Register($dst$$reg),
9602 zr,
9603 zr,
9604 (Assembler::Condition)$cmp$$cmpcode);
9605 %}
9606
9607 ins_pipe(icond_none);
9608 %}
9609
9610 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9611 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9612
9613 ins_cost(INSN_COST * 2);
9614 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9615
9616 ins_encode %{
9617 // equivalently
9618 // cset(as_Register($dst$$reg),
9619 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9620 __ csincw(as_Register($dst$$reg),
9621 zr,
9622 zr,
9623 (Assembler::Condition)$cmp$$cmpcode);
9624 %}
9625
9626 ins_pipe(icond_none);
9627 %}
9628
9629 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9630 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9631
9632 ins_cost(INSN_COST * 2);
9633 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9634
9635 ins_encode %{
9636 __ csel(as_Register($dst$$reg),
9637 as_Register($src2$$reg),
9638 as_Register($src1$$reg),
9639 (Assembler::Condition)$cmp$$cmpcode);
9640 %}
9641
9642 ins_pipe(icond_reg_reg);
9643 %}
9644
9645 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9646 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9647
9648 ins_cost(INSN_COST * 2);
9649 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9650
9651 ins_encode %{
9652 __ csel(as_Register($dst$$reg),
9653 as_Register($src2$$reg),
9654 as_Register($src1$$reg),
9655 (Assembler::Condition)$cmp$$cmpcode);
9656 %}
9657
9658 ins_pipe(icond_reg_reg);
9659 %}
9660
9661 // special cases where one arg is zero
9662
9663 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9664 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9665
9666 ins_cost(INSN_COST * 2);
9667 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9668
9669 ins_encode %{
9670 __ csel(as_Register($dst$$reg),
9671 zr,
9672 as_Register($src$$reg),
9673 (Assembler::Condition)$cmp$$cmpcode);
9674 %}
9675
9676 ins_pipe(icond_reg);
9677 %}
9678
9679 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9680 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9681
9682 ins_cost(INSN_COST * 2);
9683 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9684
9685 ins_encode %{
9686 __ csel(as_Register($dst$$reg),
9687 zr,
9688 as_Register($src$$reg),
9689 (Assembler::Condition)$cmp$$cmpcode);
9690 %}
9691
9692 ins_pipe(icond_reg);
9693 %}
9694
9695 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9696 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9697
9698 ins_cost(INSN_COST * 2);
9699 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9700
9701 ins_encode %{
9702 __ csel(as_Register($dst$$reg),
9703 as_Register($src$$reg),
9704 zr,
9705 (Assembler::Condition)$cmp$$cmpcode);
9706 %}
9707
9708 ins_pipe(icond_reg);
9709 %}
9710
9711 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9712 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9713
9714 ins_cost(INSN_COST * 2);
9715 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9716
9717 ins_encode %{
9718 __ csel(as_Register($dst$$reg),
9719 as_Register($src$$reg),
9720 zr,
9721 (Assembler::Condition)$cmp$$cmpcode);
9722 %}
9723
9724 ins_pipe(icond_reg);
9725 %}
9726
9727 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9728 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9729
9730 ins_cost(INSN_COST * 2);
9731 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9732
9733 ins_encode %{
9734 __ csel(as_Register($dst$$reg),
9735 as_Register($src2$$reg),
9736 as_Register($src1$$reg),
9737 (Assembler::Condition)$cmp$$cmpcode);
9738 %}
9739
9740 ins_pipe(icond_reg_reg);
9741 %}
9742
9743 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9744 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9745
9746 ins_cost(INSN_COST * 2);
9747 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9748
9749 ins_encode %{
9750 __ csel(as_Register($dst$$reg),
9751 as_Register($src2$$reg),
9752 as_Register($src1$$reg),
9753 (Assembler::Condition)$cmp$$cmpcode);
9754 %}
9755
9756 ins_pipe(icond_reg_reg);
9757 %}
9758
9759 // special cases where one arg is zero
9760
9761 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9762 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9763
9764 ins_cost(INSN_COST * 2);
9765 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9766
9767 ins_encode %{
9768 __ csel(as_Register($dst$$reg),
9769 zr,
9770 as_Register($src$$reg),
9771 (Assembler::Condition)$cmp$$cmpcode);
9772 %}
9773
9774 ins_pipe(icond_reg);
9775 %}
9776
9777 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9778 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9779
9780 ins_cost(INSN_COST * 2);
9781 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9782
9783 ins_encode %{
9784 __ csel(as_Register($dst$$reg),
9785 zr,
9786 as_Register($src$$reg),
9787 (Assembler::Condition)$cmp$$cmpcode);
9788 %}
9789
9790 ins_pipe(icond_reg);
9791 %}
9792
9793 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9794 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9795
9796 ins_cost(INSN_COST * 2);
9797 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9798
9799 ins_encode %{
9800 __ csel(as_Register($dst$$reg),
9801 as_Register($src$$reg),
9802 zr,
9803 (Assembler::Condition)$cmp$$cmpcode);
9804 %}
9805
9806 ins_pipe(icond_reg);
9807 %}
9808
9809 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9810 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9811
9812 ins_cost(INSN_COST * 2);
9813 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9814
9815 ins_encode %{
9816 __ csel(as_Register($dst$$reg),
9817 as_Register($src$$reg),
9818 zr,
9819 (Assembler::Condition)$cmp$$cmpcode);
9820 %}
9821
9822 ins_pipe(icond_reg);
9823 %}
9824
9825 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9826 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9827
9828 ins_cost(INSN_COST * 2);
9829 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9830
9831 ins_encode %{
9832 __ cselw(as_Register($dst$$reg),
9833 as_Register($src2$$reg),
9834 as_Register($src1$$reg),
9835 (Assembler::Condition)$cmp$$cmpcode);
9836 %}
9837
9838 ins_pipe(icond_reg_reg);
9839 %}
9840
9841 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9842 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9843
9844 ins_cost(INSN_COST * 2);
9845 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9846
9847 ins_encode %{
9848 __ cselw(as_Register($dst$$reg),
9849 as_Register($src2$$reg),
9850 as_Register($src1$$reg),
9851 (Assembler::Condition)$cmp$$cmpcode);
9852 %}
9853
9854 ins_pipe(icond_reg_reg);
9855 %}
9856
9857 // special cases where one arg is zero
9858
9859 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9860 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9861
9862 ins_cost(INSN_COST * 2);
9863 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9864
9865 ins_encode %{
9866 __ cselw(as_Register($dst$$reg),
9867 zr,
9868 as_Register($src$$reg),
9869 (Assembler::Condition)$cmp$$cmpcode);
9870 %}
9871
9872 ins_pipe(icond_reg);
9873 %}
9874
9875 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9876 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9877
9878 ins_cost(INSN_COST * 2);
9879 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9880
9881 ins_encode %{
9882 __ cselw(as_Register($dst$$reg),
9883 zr,
9884 as_Register($src$$reg),
9885 (Assembler::Condition)$cmp$$cmpcode);
9886 %}
9887
9888 ins_pipe(icond_reg);
9889 %}
9890
9891 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9892 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9893
9894 ins_cost(INSN_COST * 2);
9895 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9896
9897 ins_encode %{
9898 __ cselw(as_Register($dst$$reg),
9899 as_Register($src$$reg),
9900 zr,
9901 (Assembler::Condition)$cmp$$cmpcode);
9902 %}
9903
9904 ins_pipe(icond_reg);
9905 %}
9906
9907 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9908 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9909
9910 ins_cost(INSN_COST * 2);
9911 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9912
9913 ins_encode %{
9914 __ cselw(as_Register($dst$$reg),
9915 as_Register($src$$reg),
9916 zr,
9917 (Assembler::Condition)$cmp$$cmpcode);
9918 %}
9919
9920 ins_pipe(icond_reg);
9921 %}
9922
9923 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9924 %{
9925 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9926
9927 ins_cost(INSN_COST * 3);
9928
9929 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9930 ins_encode %{
9931 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9932 __ fcsels(as_FloatRegister($dst$$reg),
9933 as_FloatRegister($src2$$reg),
9934 as_FloatRegister($src1$$reg),
9935 cond);
9936 %}
9937
9938 ins_pipe(fp_cond_reg_reg_s);
9939 %}
9940
9941 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9942 %{
9943 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9944
9945 ins_cost(INSN_COST * 3);
9946
9947 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9948 ins_encode %{
9949 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9950 __ fcsels(as_FloatRegister($dst$$reg),
9951 as_FloatRegister($src2$$reg),
9952 as_FloatRegister($src1$$reg),
9953 cond);
9954 %}
9955
9956 ins_pipe(fp_cond_reg_reg_s);
9957 %}
9958
9959 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9960 %{
9961 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9962
9963 ins_cost(INSN_COST * 3);
9964
9965 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9966 ins_encode %{
9967 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9968 __ fcseld(as_FloatRegister($dst$$reg),
9969 as_FloatRegister($src2$$reg),
9970 as_FloatRegister($src1$$reg),
9971 cond);
9972 %}
9973
9974 ins_pipe(fp_cond_reg_reg_d);
9975 %}
9976
9977 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9978 %{
9979 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9980
9981 ins_cost(INSN_COST * 3);
9982
9983 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9984 ins_encode %{
9985 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9986 __ fcseld(as_FloatRegister($dst$$reg),
9987 as_FloatRegister($src2$$reg),
9988 as_FloatRegister($src1$$reg),
9989 cond);
9990 %}
9991
9992 ins_pipe(fp_cond_reg_reg_d);
9993 %}
9994
9995 // ============================================================================
9996 // Arithmetic Instructions
9997 //
9998
9999 // Integer Addition
10000
10001 // TODO
10002 // these currently employ operations which do not set CR and hence are
10003 // not flagged as killing CR but we would like to isolate the cases
10004 // where we want to set flags from those where we don't. need to work
10005 // out how to do that.
10006
10007 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10008 match(Set dst (AddI src1 src2));
10009
10010 ins_cost(INSN_COST);
10011 format %{ "addw $dst, $src1, $src2" %}
10012
10013 ins_encode %{
10014 __ addw(as_Register($dst$$reg),
10015 as_Register($src1$$reg),
10016 as_Register($src2$$reg));
10017 %}
10018
10019 ins_pipe(ialu_reg_reg);
10020 %}
10021
10022 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10023 match(Set dst (AddI src1 src2));
10024
10025 ins_cost(INSN_COST);
10026 format %{ "addw $dst, $src1, $src2" %}
10027
10028 // use opcode to indicate that this is an add not a sub
10029 opcode(0x0);
10030
10031 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10032
10033 ins_pipe(ialu_reg_imm);
10034 %}
10035
10036 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10037 match(Set dst (AddI (ConvL2I src1) src2));
10038
10039 ins_cost(INSN_COST);
10040 format %{ "addw $dst, $src1, $src2" %}
10041
10042 // use opcode to indicate that this is an add not a sub
10043 opcode(0x0);
10044
10045 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10046
10047 ins_pipe(ialu_reg_imm);
10048 %}
10049
10050 // Pointer Addition
10051 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10052 match(Set dst (AddP src1 src2));
10053
10054 ins_cost(INSN_COST);
10055 format %{ "add $dst, $src1, $src2\t# ptr" %}
10056
10057 ins_encode %{
10058 __ add(as_Register($dst$$reg),
10059 as_Register($src1$$reg),
10060 as_Register($src2$$reg));
10061 %}
10062
10063 ins_pipe(ialu_reg_reg);
10064 %}
10065
10066 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10067 match(Set dst (AddP src1 (ConvI2L src2)));
10068
10069 ins_cost(1.9 * INSN_COST);
10070 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10071
10072 ins_encode %{
10073 __ add(as_Register($dst$$reg),
10074 as_Register($src1$$reg),
10075 as_Register($src2$$reg), ext::sxtw);
10076 %}
10077
10078 ins_pipe(ialu_reg_reg);
10079 %}
10080
10081 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10082 match(Set dst (AddP src1 (LShiftL src2 scale)));
10083
10084 ins_cost(1.9 * INSN_COST);
10085 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10086
10087 ins_encode %{
10088 __ lea(as_Register($dst$$reg),
10089 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10090 Address::lsl($scale$$constant)));
10091 %}
10092
10093 ins_pipe(ialu_reg_reg_shift);
10094 %}
10095
10096 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10097 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10098
10099 ins_cost(1.9 * INSN_COST);
10100 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10101
10102 ins_encode %{
10103 __ lea(as_Register($dst$$reg),
10104 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10105 Address::sxtw($scale$$constant)));
10106 %}
10107
10108 ins_pipe(ialu_reg_reg_shift);
10109 %}
10110
10111 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10112 match(Set dst (LShiftL (ConvI2L src) scale));
10113
10114 ins_cost(INSN_COST);
10115 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10116
10117 ins_encode %{
10118 __ sbfiz(as_Register($dst$$reg),
10119 as_Register($src$$reg),
10120 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10121 %}
10122
10123 ins_pipe(ialu_reg_shift);
10124 %}
10125
10126 // Pointer Immediate Addition
10127 // n.b. this needs to be more expensive than using an indirect memory
10128 // operand
10129 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10130 match(Set dst (AddP src1 src2));
10131
10132 ins_cost(INSN_COST);
10133 format %{ "add $dst, $src1, $src2\t# ptr" %}
10134
10135 // use opcode to indicate that this is an add not a sub
10136 opcode(0x0);
10137
10138 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10139
10140 ins_pipe(ialu_reg_imm);
10141 %}
10142
10143 // Long Addition
10144 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10145
10146 match(Set dst (AddL src1 src2));
10147
10148 ins_cost(INSN_COST);
10149 format %{ "add $dst, $src1, $src2" %}
10150
10151 ins_encode %{
10152 __ add(as_Register($dst$$reg),
10153 as_Register($src1$$reg),
10154 as_Register($src2$$reg));
10155 %}
10156
10157 ins_pipe(ialu_reg_reg);
10158 %}
10159
10160 // No constant pool entries requiredLong Immediate Addition.
10161 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10162 match(Set dst (AddL src1 src2));
10163
10164 ins_cost(INSN_COST);
10165 format %{ "add $dst, $src1, $src2" %}
10166
10167 // use opcode to indicate that this is an add not a sub
10168 opcode(0x0);
10169
10170 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10171
10172 ins_pipe(ialu_reg_imm);
10173 %}
10174
10175 // Integer Subtraction
10176 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10177 match(Set dst (SubI src1 src2));
10178
10179 ins_cost(INSN_COST);
10180 format %{ "subw $dst, $src1, $src2" %}
10181
10182 ins_encode %{
10183 __ subw(as_Register($dst$$reg),
10184 as_Register($src1$$reg),
10185 as_Register($src2$$reg));
10186 %}
10187
10188 ins_pipe(ialu_reg_reg);
10189 %}
10190
10191 // Immediate Subtraction
10192 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10193 match(Set dst (SubI src1 src2));
10194
10195 ins_cost(INSN_COST);
10196 format %{ "subw $dst, $src1, $src2" %}
10197
10198 // use opcode to indicate that this is a sub not an add
10199 opcode(0x1);
10200
10201 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10202
10203 ins_pipe(ialu_reg_imm);
10204 %}
10205
10206 // Long Subtraction
10207 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10208
10209 match(Set dst (SubL src1 src2));
10210
10211 ins_cost(INSN_COST);
10212 format %{ "sub $dst, $src1, $src2" %}
10213
10214 ins_encode %{
10215 __ sub(as_Register($dst$$reg),
10216 as_Register($src1$$reg),
10217 as_Register($src2$$reg));
10218 %}
10219
10220 ins_pipe(ialu_reg_reg);
10221 %}
10222
10223 // No constant pool entries requiredLong Immediate Subtraction.
10224 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10225 match(Set dst (SubL src1 src2));
10226
10227 ins_cost(INSN_COST);
10228 format %{ "sub$dst, $src1, $src2" %}
10229
10230 // use opcode to indicate that this is a sub not an add
10231 opcode(0x1);
10232
10233 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10234
10235 ins_pipe(ialu_reg_imm);
10236 %}
10237
10238 // Integer Negation (special case for sub)
10239
10240 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10241 match(Set dst (SubI zero src));
10242
10243 ins_cost(INSN_COST);
10244 format %{ "negw $dst, $src\t# int" %}
10245
10246 ins_encode %{
10247 __ negw(as_Register($dst$$reg),
10248 as_Register($src$$reg));
10249 %}
10250
10251 ins_pipe(ialu_reg);
10252 %}
10253
10254 // Long Negation
10255
10256 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10257 match(Set dst (SubL zero src));
10258
10259 ins_cost(INSN_COST);
10260 format %{ "neg $dst, $src\t# long" %}
10261
10262 ins_encode %{
10263 __ neg(as_Register($dst$$reg),
10264 as_Register($src$$reg));
10265 %}
10266
10267 ins_pipe(ialu_reg);
10268 %}
10269
10270 // Integer Multiply
10271
10272 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10273 match(Set dst (MulI src1 src2));
10274
10275 ins_cost(INSN_COST * 3);
10276 format %{ "mulw $dst, $src1, $src2" %}
10277
10278 ins_encode %{
10279 __ mulw(as_Register($dst$$reg),
10280 as_Register($src1$$reg),
10281 as_Register($src2$$reg));
10282 %}
10283
10284 ins_pipe(imul_reg_reg);
10285 %}
10286
10287 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10288 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10289
10290 ins_cost(INSN_COST * 3);
10291 format %{ "smull $dst, $src1, $src2" %}
10292
10293 ins_encode %{
10294 __ smull(as_Register($dst$$reg),
10295 as_Register($src1$$reg),
10296 as_Register($src2$$reg));
10297 %}
10298
10299 ins_pipe(imul_reg_reg);
10300 %}
10301
10302 // Long Multiply
10303
10304 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10305 match(Set dst (MulL src1 src2));
10306
10307 ins_cost(INSN_COST * 5);
10308 format %{ "mul $dst, $src1, $src2" %}
10309
10310 ins_encode %{
10311 __ mul(as_Register($dst$$reg),
10312 as_Register($src1$$reg),
10313 as_Register($src2$$reg));
10314 %}
10315
10316 ins_pipe(lmul_reg_reg);
10317 %}
10318
10319 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10320 %{
10321 match(Set dst (MulHiL src1 src2));
10322
10323 ins_cost(INSN_COST * 7);
10324 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10325
10326 ins_encode %{
10327 __ smulh(as_Register($dst$$reg),
10328 as_Register($src1$$reg),
10329 as_Register($src2$$reg));
10330 %}
10331
10332 ins_pipe(lmul_reg_reg);
10333 %}
10334
10335 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10336 %{
10337 match(Set dst (UMulHiL src1 src2));
10338
10339 ins_cost(INSN_COST * 7);
10340 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10341
10342 ins_encode %{
10343 __ umulh(as_Register($dst$$reg),
10344 as_Register($src1$$reg),
10345 as_Register($src2$$reg));
10346 %}
10347
10348 ins_pipe(lmul_reg_reg);
10349 %}
10350
10351 // Combined Integer Multiply & Add/Sub
10352
10353 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10354 match(Set dst (AddI src3 (MulI src1 src2)));
10355
10356 ins_cost(INSN_COST * 3);
10357 format %{ "madd $dst, $src1, $src2, $src3" %}
10358
10359 ins_encode %{
10360 __ maddw(as_Register($dst$$reg),
10361 as_Register($src1$$reg),
10362 as_Register($src2$$reg),
10363 as_Register($src3$$reg));
10364 %}
10365
10366 ins_pipe(imac_reg_reg);
10367 %}
10368
10369 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10370 match(Set dst (SubI src3 (MulI src1 src2)));
10371
10372 ins_cost(INSN_COST * 3);
10373 format %{ "msub $dst, $src1, $src2, $src3" %}
10374
10375 ins_encode %{
10376 __ msubw(as_Register($dst$$reg),
10377 as_Register($src1$$reg),
10378 as_Register($src2$$reg),
10379 as_Register($src3$$reg));
10380 %}
10381
10382 ins_pipe(imac_reg_reg);
10383 %}
10384
10385 // Combined Integer Multiply & Neg
10386
10387 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10388 match(Set dst (MulI (SubI zero src1) src2));
10389
10390 ins_cost(INSN_COST * 3);
10391 format %{ "mneg $dst, $src1, $src2" %}
10392
10393 ins_encode %{
10394 __ mnegw(as_Register($dst$$reg),
10395 as_Register($src1$$reg),
10396 as_Register($src2$$reg));
10397 %}
10398
10399 ins_pipe(imac_reg_reg);
10400 %}
10401
10402 // Combined Long Multiply & Add/Sub
10403
10404 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10405 match(Set dst (AddL src3 (MulL src1 src2)));
10406
10407 ins_cost(INSN_COST * 5);
10408 format %{ "madd $dst, $src1, $src2, $src3" %}
10409
10410 ins_encode %{
10411 __ madd(as_Register($dst$$reg),
10412 as_Register($src1$$reg),
10413 as_Register($src2$$reg),
10414 as_Register($src3$$reg));
10415 %}
10416
10417 ins_pipe(lmac_reg_reg);
10418 %}
10419
10420 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10421 match(Set dst (SubL src3 (MulL src1 src2)));
10422
10423 ins_cost(INSN_COST * 5);
10424 format %{ "msub $dst, $src1, $src2, $src3" %}
10425
10426 ins_encode %{
10427 __ msub(as_Register($dst$$reg),
10428 as_Register($src1$$reg),
10429 as_Register($src2$$reg),
10430 as_Register($src3$$reg));
10431 %}
10432
10433 ins_pipe(lmac_reg_reg);
10434 %}
10435
10436 // Combined Long Multiply & Neg
10437
10438 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10439 match(Set dst (MulL (SubL zero src1) src2));
10440
10441 ins_cost(INSN_COST * 5);
10442 format %{ "mneg $dst, $src1, $src2" %}
10443
10444 ins_encode %{
10445 __ mneg(as_Register($dst$$reg),
10446 as_Register($src1$$reg),
10447 as_Register($src2$$reg));
10448 %}
10449
10450 ins_pipe(lmac_reg_reg);
10451 %}
10452
10453 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10454
10455 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10456 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10457
10458 ins_cost(INSN_COST * 3);
10459 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10460
10461 ins_encode %{
10462 __ smaddl(as_Register($dst$$reg),
10463 as_Register($src1$$reg),
10464 as_Register($src2$$reg),
10465 as_Register($src3$$reg));
10466 %}
10467
10468 ins_pipe(imac_reg_reg);
10469 %}
10470
10471 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10472 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10473
10474 ins_cost(INSN_COST * 3);
10475 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10476
10477 ins_encode %{
10478 __ smsubl(as_Register($dst$$reg),
10479 as_Register($src1$$reg),
10480 as_Register($src2$$reg),
10481 as_Register($src3$$reg));
10482 %}
10483
10484 ins_pipe(imac_reg_reg);
10485 %}
10486
10487 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10488 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10489
10490 ins_cost(INSN_COST * 3);
10491 format %{ "smnegl $dst, $src1, $src2" %}
10492
10493 ins_encode %{
10494 __ smnegl(as_Register($dst$$reg),
10495 as_Register($src1$$reg),
10496 as_Register($src2$$reg));
10497 %}
10498
10499 ins_pipe(imac_reg_reg);
10500 %}
10501
10502 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10503
10504 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10505 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10506
10507 ins_cost(INSN_COST * 5);
10508 format %{ "mulw rscratch1, $src1, $src2\n\t"
10509 "maddw $dst, $src3, $src4, rscratch1" %}
10510
10511 ins_encode %{
10512 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10513 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10514
10515 ins_pipe(imac_reg_reg);
10516 %}
10517
10518 // Integer Divide
10519
10520 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10521 match(Set dst (DivI src1 src2));
10522
10523 ins_cost(INSN_COST * 19);
10524 format %{ "sdivw $dst, $src1, $src2" %}
10525
10526 ins_encode(aarch64_enc_divw(dst, src1, src2));
10527 ins_pipe(idiv_reg_reg);
10528 %}
10529
10530 // Long Divide
10531
10532 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10533 match(Set dst (DivL src1 src2));
10534
10535 ins_cost(INSN_COST * 35);
10536 format %{ "sdiv $dst, $src1, $src2" %}
10537
10538 ins_encode(aarch64_enc_div(dst, src1, src2));
10539 ins_pipe(ldiv_reg_reg);
10540 %}
10541
10542 // Integer Remainder
10543
10544 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10545 match(Set dst (ModI src1 src2));
10546
10547 ins_cost(INSN_COST * 22);
10548 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10549 "msubw $dst, rscratch1, $src2, $src1" %}
10550
10551 ins_encode(aarch64_enc_modw(dst, src1, src2));
10552 ins_pipe(idiv_reg_reg);
10553 %}
10554
10555 // Long Remainder
10556
10557 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10558 match(Set dst (ModL src1 src2));
10559
10560 ins_cost(INSN_COST * 38);
10561 format %{ "sdiv rscratch1, $src1, $src2\n"
10562 "msub $dst, rscratch1, $src2, $src1" %}
10563
10564 ins_encode(aarch64_enc_mod(dst, src1, src2));
10565 ins_pipe(ldiv_reg_reg);
10566 %}
10567
10568 // Unsigned Integer Divide
10569
10570 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10571 match(Set dst (UDivI src1 src2));
10572
10573 ins_cost(INSN_COST * 19);
10574 format %{ "udivw $dst, $src1, $src2" %}
10575
10576 ins_encode %{
10577 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10578 %}
10579
10580 ins_pipe(idiv_reg_reg);
10581 %}
10582
10583 // Unsigned Long Divide
10584
10585 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10586 match(Set dst (UDivL src1 src2));
10587
10588 ins_cost(INSN_COST * 35);
10589 format %{ "udiv $dst, $src1, $src2" %}
10590
10591 ins_encode %{
10592 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10593 %}
10594
10595 ins_pipe(ldiv_reg_reg);
10596 %}
10597
10598 // Unsigned Integer Remainder
10599
10600 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10601 match(Set dst (UModI src1 src2));
10602
10603 ins_cost(INSN_COST * 22);
10604 format %{ "udivw rscratch1, $src1, $src2\n\t"
10605 "msubw $dst, rscratch1, $src2, $src1" %}
10606
10607 ins_encode %{
10608 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10609 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10610 %}
10611
10612 ins_pipe(idiv_reg_reg);
10613 %}
10614
10615 // Unsigned Long Remainder
10616
10617 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10618 match(Set dst (UModL src1 src2));
10619
10620 ins_cost(INSN_COST * 38);
10621 format %{ "udiv rscratch1, $src1, $src2\n"
10622 "msub $dst, rscratch1, $src2, $src1" %}
10623
10624 ins_encode %{
10625 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10626 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10627 %}
10628
10629 ins_pipe(ldiv_reg_reg);
10630 %}
10631
10632 // Integer Shifts
10633
10634 // Shift Left Register
10635 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10636 match(Set dst (LShiftI src1 src2));
10637
10638 ins_cost(INSN_COST * 2);
10639 format %{ "lslvw $dst, $src1, $src2" %}
10640
10641 ins_encode %{
10642 __ lslvw(as_Register($dst$$reg),
10643 as_Register($src1$$reg),
10644 as_Register($src2$$reg));
10645 %}
10646
10647 ins_pipe(ialu_reg_reg_vshift);
10648 %}
10649
10650 // Shift Left Immediate
10651 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10652 match(Set dst (LShiftI src1 src2));
10653
10654 ins_cost(INSN_COST);
10655 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10656
10657 ins_encode %{
10658 __ lslw(as_Register($dst$$reg),
10659 as_Register($src1$$reg),
10660 $src2$$constant & 0x1f);
10661 %}
10662
10663 ins_pipe(ialu_reg_shift);
10664 %}
10665
10666 // Shift Right Logical Register
10667 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10668 match(Set dst (URShiftI src1 src2));
10669
10670 ins_cost(INSN_COST * 2);
10671 format %{ "lsrvw $dst, $src1, $src2" %}
10672
10673 ins_encode %{
10674 __ lsrvw(as_Register($dst$$reg),
10675 as_Register($src1$$reg),
10676 as_Register($src2$$reg));
10677 %}
10678
10679 ins_pipe(ialu_reg_reg_vshift);
10680 %}
10681
10682 // Shift Right Logical Immediate
10683 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10684 match(Set dst (URShiftI src1 src2));
10685
10686 ins_cost(INSN_COST);
10687 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10688
10689 ins_encode %{
10690 __ lsrw(as_Register($dst$$reg),
10691 as_Register($src1$$reg),
10692 $src2$$constant & 0x1f);
10693 %}
10694
10695 ins_pipe(ialu_reg_shift);
10696 %}
10697
10698 // Shift Right Arithmetic Register
10699 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10700 match(Set dst (RShiftI src1 src2));
10701
10702 ins_cost(INSN_COST * 2);
10703 format %{ "asrvw $dst, $src1, $src2" %}
10704
10705 ins_encode %{
10706 __ asrvw(as_Register($dst$$reg),
10707 as_Register($src1$$reg),
10708 as_Register($src2$$reg));
10709 %}
10710
10711 ins_pipe(ialu_reg_reg_vshift);
10712 %}
10713
10714 // Shift Right Arithmetic Immediate
10715 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10716 match(Set dst (RShiftI src1 src2));
10717
10718 ins_cost(INSN_COST);
10719 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10720
10721 ins_encode %{
10722 __ asrw(as_Register($dst$$reg),
10723 as_Register($src1$$reg),
10724 $src2$$constant & 0x1f);
10725 %}
10726
10727 ins_pipe(ialu_reg_shift);
10728 %}
10729
10730 // Combined Int Mask and Right Shift (using UBFM)
10731 // TODO
10732
10733 // Long Shifts
10734
10735 // Shift Left Register
10736 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10737 match(Set dst (LShiftL src1 src2));
10738
10739 ins_cost(INSN_COST * 2);
10740 format %{ "lslv $dst, $src1, $src2" %}
10741
10742 ins_encode %{
10743 __ lslv(as_Register($dst$$reg),
10744 as_Register($src1$$reg),
10745 as_Register($src2$$reg));
10746 %}
10747
10748 ins_pipe(ialu_reg_reg_vshift);
10749 %}
10750
10751 // Shift Left Immediate
10752 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10753 match(Set dst (LShiftL src1 src2));
10754
10755 ins_cost(INSN_COST);
10756 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10757
10758 ins_encode %{
10759 __ lsl(as_Register($dst$$reg),
10760 as_Register($src1$$reg),
10761 $src2$$constant & 0x3f);
10762 %}
10763
10764 ins_pipe(ialu_reg_shift);
10765 %}
10766
10767 // Shift Right Logical Register
10768 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10769 match(Set dst (URShiftL src1 src2));
10770
10771 ins_cost(INSN_COST * 2);
10772 format %{ "lsrv $dst, $src1, $src2" %}
10773
10774 ins_encode %{
10775 __ lsrv(as_Register($dst$$reg),
10776 as_Register($src1$$reg),
10777 as_Register($src2$$reg));
10778 %}
10779
10780 ins_pipe(ialu_reg_reg_vshift);
10781 %}
10782
10783 // Shift Right Logical Immediate
10784 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10785 match(Set dst (URShiftL src1 src2));
10786
10787 ins_cost(INSN_COST);
10788 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10789
10790 ins_encode %{
10791 __ lsr(as_Register($dst$$reg),
10792 as_Register($src1$$reg),
10793 $src2$$constant & 0x3f);
10794 %}
10795
10796 ins_pipe(ialu_reg_shift);
10797 %}
10798
10799 // A special-case pattern for card table stores.
10800 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10801 match(Set dst (URShiftL (CastP2X src1) src2));
10802
10803 ins_cost(INSN_COST);
10804 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10805
10806 ins_encode %{
10807 __ lsr(as_Register($dst$$reg),
10808 as_Register($src1$$reg),
10809 $src2$$constant & 0x3f);
10810 %}
10811
10812 ins_pipe(ialu_reg_shift);
10813 %}
10814
10815 // Shift Right Arithmetic Register
10816 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10817 match(Set dst (RShiftL src1 src2));
10818
10819 ins_cost(INSN_COST * 2);
10820 format %{ "asrv $dst, $src1, $src2" %}
10821
10822 ins_encode %{
10823 __ asrv(as_Register($dst$$reg),
10824 as_Register($src1$$reg),
10825 as_Register($src2$$reg));
10826 %}
10827
10828 ins_pipe(ialu_reg_reg_vshift);
10829 %}
10830
10831 // Shift Right Arithmetic Immediate
10832 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10833 match(Set dst (RShiftL src1 src2));
10834
10835 ins_cost(INSN_COST);
10836 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10837
10838 ins_encode %{
10839 __ asr(as_Register($dst$$reg),
10840 as_Register($src1$$reg),
10841 $src2$$constant & 0x3f);
10842 %}
10843
10844 ins_pipe(ialu_reg_shift);
10845 %}
10846
10847 // BEGIN This section of the file is automatically generated. Do not edit --------------
10848 // This section is generated from aarch64_ad.m4
10849
10850 // This pattern is automatically generated from aarch64_ad.m4
10851 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10852 instruct regL_not_reg(iRegLNoSp dst,
10853 iRegL src1, immL_M1 m1,
10854 rFlagsReg cr) %{
10855 match(Set dst (XorL src1 m1));
10856 ins_cost(INSN_COST);
10857 format %{ "eon $dst, $src1, zr" %}
10858
10859 ins_encode %{
10860 __ eon(as_Register($dst$$reg),
10861 as_Register($src1$$reg),
10862 zr,
10863 Assembler::LSL, 0);
10864 %}
10865
10866 ins_pipe(ialu_reg);
10867 %}
10868
10869 // This pattern is automatically generated from aarch64_ad.m4
10870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10871 instruct regI_not_reg(iRegINoSp dst,
10872 iRegIorL2I src1, immI_M1 m1,
10873 rFlagsReg cr) %{
10874 match(Set dst (XorI src1 m1));
10875 ins_cost(INSN_COST);
10876 format %{ "eonw $dst, $src1, zr" %}
10877
10878 ins_encode %{
10879 __ eonw(as_Register($dst$$reg),
10880 as_Register($src1$$reg),
10881 zr,
10882 Assembler::LSL, 0);
10883 %}
10884
10885 ins_pipe(ialu_reg);
10886 %}
10887
10888 // This pattern is automatically generated from aarch64_ad.m4
10889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10890 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10891 immI0 zero, iRegIorL2I src1, immI src2) %{
10892 match(Set dst (SubI zero (URShiftI src1 src2)));
10893
10894 ins_cost(1.9 * INSN_COST);
10895 format %{ "negw $dst, $src1, LSR $src2" %}
10896
10897 ins_encode %{
10898 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10899 Assembler::LSR, $src2$$constant & 0x1f);
10900 %}
10901
10902 ins_pipe(ialu_reg_shift);
10903 %}
10904
10905 // This pattern is automatically generated from aarch64_ad.m4
10906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10907 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10908 immI0 zero, iRegIorL2I src1, immI src2) %{
10909 match(Set dst (SubI zero (RShiftI src1 src2)));
10910
10911 ins_cost(1.9 * INSN_COST);
10912 format %{ "negw $dst, $src1, ASR $src2" %}
10913
10914 ins_encode %{
10915 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10916 Assembler::ASR, $src2$$constant & 0x1f);
10917 %}
10918
10919 ins_pipe(ialu_reg_shift);
10920 %}
10921
10922 // This pattern is automatically generated from aarch64_ad.m4
10923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10924 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10925 immI0 zero, iRegIorL2I src1, immI src2) %{
10926 match(Set dst (SubI zero (LShiftI src1 src2)));
10927
10928 ins_cost(1.9 * INSN_COST);
10929 format %{ "negw $dst, $src1, LSL $src2" %}
10930
10931 ins_encode %{
10932 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10933 Assembler::LSL, $src2$$constant & 0x1f);
10934 %}
10935
10936 ins_pipe(ialu_reg_shift);
10937 %}
10938
10939 // This pattern is automatically generated from aarch64_ad.m4
10940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10941 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10942 immL0 zero, iRegL src1, immI src2) %{
10943 match(Set dst (SubL zero (URShiftL src1 src2)));
10944
10945 ins_cost(1.9 * INSN_COST);
10946 format %{ "neg $dst, $src1, LSR $src2" %}
10947
10948 ins_encode %{
10949 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10950 Assembler::LSR, $src2$$constant & 0x3f);
10951 %}
10952
10953 ins_pipe(ialu_reg_shift);
10954 %}
10955
10956 // This pattern is automatically generated from aarch64_ad.m4
10957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10958 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10959 immL0 zero, iRegL src1, immI src2) %{
10960 match(Set dst (SubL zero (RShiftL src1 src2)));
10961
10962 ins_cost(1.9 * INSN_COST);
10963 format %{ "neg $dst, $src1, ASR $src2" %}
10964
10965 ins_encode %{
10966 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10967 Assembler::ASR, $src2$$constant & 0x3f);
10968 %}
10969
10970 ins_pipe(ialu_reg_shift);
10971 %}
10972
10973 // This pattern is automatically generated from aarch64_ad.m4
10974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10975 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10976 immL0 zero, iRegL src1, immI src2) %{
10977 match(Set dst (SubL zero (LShiftL src1 src2)));
10978
10979 ins_cost(1.9 * INSN_COST);
10980 format %{ "neg $dst, $src1, LSL $src2" %}
10981
10982 ins_encode %{
10983 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10984 Assembler::LSL, $src2$$constant & 0x3f);
10985 %}
10986
10987 ins_pipe(ialu_reg_shift);
10988 %}
10989
10990 // This pattern is automatically generated from aarch64_ad.m4
10991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10992 instruct AndI_reg_not_reg(iRegINoSp dst,
10993 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10994 match(Set dst (AndI src1 (XorI src2 m1)));
10995 ins_cost(INSN_COST);
10996 format %{ "bicw $dst, $src1, $src2" %}
10997
10998 ins_encode %{
10999 __ bicw(as_Register($dst$$reg),
11000 as_Register($src1$$reg),
11001 as_Register($src2$$reg),
11002 Assembler::LSL, 0);
11003 %}
11004
11005 ins_pipe(ialu_reg_reg);
11006 %}
11007
11008 // This pattern is automatically generated from aarch64_ad.m4
11009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11010 instruct AndL_reg_not_reg(iRegLNoSp dst,
11011 iRegL src1, iRegL src2, immL_M1 m1) %{
11012 match(Set dst (AndL src1 (XorL src2 m1)));
11013 ins_cost(INSN_COST);
11014 format %{ "bic $dst, $src1, $src2" %}
11015
11016 ins_encode %{
11017 __ bic(as_Register($dst$$reg),
11018 as_Register($src1$$reg),
11019 as_Register($src2$$reg),
11020 Assembler::LSL, 0);
11021 %}
11022
11023 ins_pipe(ialu_reg_reg);
11024 %}
11025
11026 // This pattern is automatically generated from aarch64_ad.m4
11027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11028 instruct OrI_reg_not_reg(iRegINoSp dst,
11029 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11030 match(Set dst (OrI src1 (XorI src2 m1)));
11031 ins_cost(INSN_COST);
11032 format %{ "ornw $dst, $src1, $src2" %}
11033
11034 ins_encode %{
11035 __ ornw(as_Register($dst$$reg),
11036 as_Register($src1$$reg),
11037 as_Register($src2$$reg),
11038 Assembler::LSL, 0);
11039 %}
11040
11041 ins_pipe(ialu_reg_reg);
11042 %}
11043
11044 // This pattern is automatically generated from aarch64_ad.m4
11045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11046 instruct OrL_reg_not_reg(iRegLNoSp dst,
11047 iRegL src1, iRegL src2, immL_M1 m1) %{
11048 match(Set dst (OrL src1 (XorL src2 m1)));
11049 ins_cost(INSN_COST);
11050 format %{ "orn $dst, $src1, $src2" %}
11051
11052 ins_encode %{
11053 __ orn(as_Register($dst$$reg),
11054 as_Register($src1$$reg),
11055 as_Register($src2$$reg),
11056 Assembler::LSL, 0);
11057 %}
11058
11059 ins_pipe(ialu_reg_reg);
11060 %}
11061
11062 // This pattern is automatically generated from aarch64_ad.m4
11063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11064 instruct XorI_reg_not_reg(iRegINoSp dst,
11065 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11066 match(Set dst (XorI m1 (XorI src2 src1)));
11067 ins_cost(INSN_COST);
11068 format %{ "eonw $dst, $src1, $src2" %}
11069
11070 ins_encode %{
11071 __ eonw(as_Register($dst$$reg),
11072 as_Register($src1$$reg),
11073 as_Register($src2$$reg),
11074 Assembler::LSL, 0);
11075 %}
11076
11077 ins_pipe(ialu_reg_reg);
11078 %}
11079
11080 // This pattern is automatically generated from aarch64_ad.m4
11081 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11082 instruct XorL_reg_not_reg(iRegLNoSp dst,
11083 iRegL src1, iRegL src2, immL_M1 m1) %{
11084 match(Set dst (XorL m1 (XorL src2 src1)));
11085 ins_cost(INSN_COST);
11086 format %{ "eon $dst, $src1, $src2" %}
11087
11088 ins_encode %{
11089 __ eon(as_Register($dst$$reg),
11090 as_Register($src1$$reg),
11091 as_Register($src2$$reg),
11092 Assembler::LSL, 0);
11093 %}
11094
11095 ins_pipe(ialu_reg_reg);
11096 %}
11097
11098 // This pattern is automatically generated from aarch64_ad.m4
11099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11100 // val & (-1 ^ (val >>> shift)) ==> bicw
11101 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11102 iRegIorL2I src1, iRegIorL2I src2,
11103 immI src3, immI_M1 src4) %{
11104 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11105 ins_cost(1.9 * INSN_COST);
11106 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11107
11108 ins_encode %{
11109 __ bicw(as_Register($dst$$reg),
11110 as_Register($src1$$reg),
11111 as_Register($src2$$reg),
11112 Assembler::LSR,
11113 $src3$$constant & 0x1f);
11114 %}
11115
11116 ins_pipe(ialu_reg_reg_shift);
11117 %}
11118
11119 // This pattern is automatically generated from aarch64_ad.m4
11120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11121 // val & (-1 ^ (val >>> shift)) ==> bic
11122 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11123 iRegL src1, iRegL src2,
11124 immI src3, immL_M1 src4) %{
11125 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11126 ins_cost(1.9 * INSN_COST);
11127 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11128
11129 ins_encode %{
11130 __ bic(as_Register($dst$$reg),
11131 as_Register($src1$$reg),
11132 as_Register($src2$$reg),
11133 Assembler::LSR,
11134 $src3$$constant & 0x3f);
11135 %}
11136
11137 ins_pipe(ialu_reg_reg_shift);
11138 %}
11139
11140 // This pattern is automatically generated from aarch64_ad.m4
11141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11142 // val & (-1 ^ (val >> shift)) ==> bicw
11143 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11144 iRegIorL2I src1, iRegIorL2I src2,
11145 immI src3, immI_M1 src4) %{
11146 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11147 ins_cost(1.9 * INSN_COST);
11148 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11149
11150 ins_encode %{
11151 __ bicw(as_Register($dst$$reg),
11152 as_Register($src1$$reg),
11153 as_Register($src2$$reg),
11154 Assembler::ASR,
11155 $src3$$constant & 0x1f);
11156 %}
11157
11158 ins_pipe(ialu_reg_reg_shift);
11159 %}
11160
11161 // This pattern is automatically generated from aarch64_ad.m4
11162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11163 // val & (-1 ^ (val >> shift)) ==> bic
11164 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11165 iRegL src1, iRegL src2,
11166 immI src3, immL_M1 src4) %{
11167 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11168 ins_cost(1.9 * INSN_COST);
11169 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11170
11171 ins_encode %{
11172 __ bic(as_Register($dst$$reg),
11173 as_Register($src1$$reg),
11174 as_Register($src2$$reg),
11175 Assembler::ASR,
11176 $src3$$constant & 0x3f);
11177 %}
11178
11179 ins_pipe(ialu_reg_reg_shift);
11180 %}
11181
11182 // This pattern is automatically generated from aarch64_ad.m4
11183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11184 // val & (-1 ^ (val ror shift)) ==> bicw
11185 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11186 iRegIorL2I src1, iRegIorL2I src2,
11187 immI src3, immI_M1 src4) %{
11188 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11189 ins_cost(1.9 * INSN_COST);
11190 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11191
11192 ins_encode %{
11193 __ bicw(as_Register($dst$$reg),
11194 as_Register($src1$$reg),
11195 as_Register($src2$$reg),
11196 Assembler::ROR,
11197 $src3$$constant & 0x1f);
11198 %}
11199
11200 ins_pipe(ialu_reg_reg_shift);
11201 %}
11202
11203 // This pattern is automatically generated from aarch64_ad.m4
11204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11205 // val & (-1 ^ (val ror shift)) ==> bic
11206 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11207 iRegL src1, iRegL src2,
11208 immI src3, immL_M1 src4) %{
11209 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11210 ins_cost(1.9 * INSN_COST);
11211 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11212
11213 ins_encode %{
11214 __ bic(as_Register($dst$$reg),
11215 as_Register($src1$$reg),
11216 as_Register($src2$$reg),
11217 Assembler::ROR,
11218 $src3$$constant & 0x3f);
11219 %}
11220
11221 ins_pipe(ialu_reg_reg_shift);
11222 %}
11223
11224 // This pattern is automatically generated from aarch64_ad.m4
11225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11226 // val & (-1 ^ (val << shift)) ==> bicw
11227 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11228 iRegIorL2I src1, iRegIorL2I src2,
11229 immI src3, immI_M1 src4) %{
11230 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11231 ins_cost(1.9 * INSN_COST);
11232 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11233
11234 ins_encode %{
11235 __ bicw(as_Register($dst$$reg),
11236 as_Register($src1$$reg),
11237 as_Register($src2$$reg),
11238 Assembler::LSL,
11239 $src3$$constant & 0x1f);
11240 %}
11241
11242 ins_pipe(ialu_reg_reg_shift);
11243 %}
11244
11245 // This pattern is automatically generated from aarch64_ad.m4
11246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11247 // val & (-1 ^ (val << shift)) ==> bic
11248 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11249 iRegL src1, iRegL src2,
11250 immI src3, immL_M1 src4) %{
11251 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11252 ins_cost(1.9 * INSN_COST);
11253 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11254
11255 ins_encode %{
11256 __ bic(as_Register($dst$$reg),
11257 as_Register($src1$$reg),
11258 as_Register($src2$$reg),
11259 Assembler::LSL,
11260 $src3$$constant & 0x3f);
11261 %}
11262
11263 ins_pipe(ialu_reg_reg_shift);
11264 %}
11265
11266 // This pattern is automatically generated from aarch64_ad.m4
11267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11268 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11269 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11270 iRegIorL2I src1, iRegIorL2I src2,
11271 immI src3, immI_M1 src4) %{
11272 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11273 ins_cost(1.9 * INSN_COST);
11274 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11275
11276 ins_encode %{
11277 __ eonw(as_Register($dst$$reg),
11278 as_Register($src1$$reg),
11279 as_Register($src2$$reg),
11280 Assembler::LSR,
11281 $src3$$constant & 0x1f);
11282 %}
11283
11284 ins_pipe(ialu_reg_reg_shift);
11285 %}
11286
11287 // This pattern is automatically generated from aarch64_ad.m4
11288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11289 // val ^ (-1 ^ (val >>> shift)) ==> eon
11290 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11291 iRegL src1, iRegL src2,
11292 immI src3, immL_M1 src4) %{
11293 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11294 ins_cost(1.9 * INSN_COST);
11295 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11296
11297 ins_encode %{
11298 __ eon(as_Register($dst$$reg),
11299 as_Register($src1$$reg),
11300 as_Register($src2$$reg),
11301 Assembler::LSR,
11302 $src3$$constant & 0x3f);
11303 %}
11304
11305 ins_pipe(ialu_reg_reg_shift);
11306 %}
11307
11308 // This pattern is automatically generated from aarch64_ad.m4
11309 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11310 // val ^ (-1 ^ (val >> shift)) ==> eonw
11311 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11312 iRegIorL2I src1, iRegIorL2I src2,
11313 immI src3, immI_M1 src4) %{
11314 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11315 ins_cost(1.9 * INSN_COST);
11316 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11317
11318 ins_encode %{
11319 __ eonw(as_Register($dst$$reg),
11320 as_Register($src1$$reg),
11321 as_Register($src2$$reg),
11322 Assembler::ASR,
11323 $src3$$constant & 0x1f);
11324 %}
11325
11326 ins_pipe(ialu_reg_reg_shift);
11327 %}
11328
11329 // This pattern is automatically generated from aarch64_ad.m4
11330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11331 // val ^ (-1 ^ (val >> shift)) ==> eon
11332 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11333 iRegL src1, iRegL src2,
11334 immI src3, immL_M1 src4) %{
11335 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11336 ins_cost(1.9 * INSN_COST);
11337 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11338
11339 ins_encode %{
11340 __ eon(as_Register($dst$$reg),
11341 as_Register($src1$$reg),
11342 as_Register($src2$$reg),
11343 Assembler::ASR,
11344 $src3$$constant & 0x3f);
11345 %}
11346
11347 ins_pipe(ialu_reg_reg_shift);
11348 %}
11349
11350 // This pattern is automatically generated from aarch64_ad.m4
11351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11352 // val ^ (-1 ^ (val ror shift)) ==> eonw
11353 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11354 iRegIorL2I src1, iRegIorL2I src2,
11355 immI src3, immI_M1 src4) %{
11356 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11357 ins_cost(1.9 * INSN_COST);
11358 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11359
11360 ins_encode %{
11361 __ eonw(as_Register($dst$$reg),
11362 as_Register($src1$$reg),
11363 as_Register($src2$$reg),
11364 Assembler::ROR,
11365 $src3$$constant & 0x1f);
11366 %}
11367
11368 ins_pipe(ialu_reg_reg_shift);
11369 %}
11370
11371 // This pattern is automatically generated from aarch64_ad.m4
11372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11373 // val ^ (-1 ^ (val ror shift)) ==> eon
11374 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11375 iRegL src1, iRegL src2,
11376 immI src3, immL_M1 src4) %{
11377 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11378 ins_cost(1.9 * INSN_COST);
11379 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11380
11381 ins_encode %{
11382 __ eon(as_Register($dst$$reg),
11383 as_Register($src1$$reg),
11384 as_Register($src2$$reg),
11385 Assembler::ROR,
11386 $src3$$constant & 0x3f);
11387 %}
11388
11389 ins_pipe(ialu_reg_reg_shift);
11390 %}
11391
11392 // This pattern is automatically generated from aarch64_ad.m4
11393 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11394 // val ^ (-1 ^ (val << shift)) ==> eonw
11395 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11396 iRegIorL2I src1, iRegIorL2I src2,
11397 immI src3, immI_M1 src4) %{
11398 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11399 ins_cost(1.9 * INSN_COST);
11400 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11401
11402 ins_encode %{
11403 __ eonw(as_Register($dst$$reg),
11404 as_Register($src1$$reg),
11405 as_Register($src2$$reg),
11406 Assembler::LSL,
11407 $src3$$constant & 0x1f);
11408 %}
11409
11410 ins_pipe(ialu_reg_reg_shift);
11411 %}
11412
11413 // This pattern is automatically generated from aarch64_ad.m4
11414 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11415 // val ^ (-1 ^ (val << shift)) ==> eon
11416 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11417 iRegL src1, iRegL src2,
11418 immI src3, immL_M1 src4) %{
11419 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11420 ins_cost(1.9 * INSN_COST);
11421 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11422
11423 ins_encode %{
11424 __ eon(as_Register($dst$$reg),
11425 as_Register($src1$$reg),
11426 as_Register($src2$$reg),
11427 Assembler::LSL,
11428 $src3$$constant & 0x3f);
11429 %}
11430
11431 ins_pipe(ialu_reg_reg_shift);
11432 %}
11433
11434 // This pattern is automatically generated from aarch64_ad.m4
11435 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11436 // val | (-1 ^ (val >>> shift)) ==> ornw
11437 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11438 iRegIorL2I src1, iRegIorL2I src2,
11439 immI src3, immI_M1 src4) %{
11440 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11441 ins_cost(1.9 * INSN_COST);
11442 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11443
11444 ins_encode %{
11445 __ ornw(as_Register($dst$$reg),
11446 as_Register($src1$$reg),
11447 as_Register($src2$$reg),
11448 Assembler::LSR,
11449 $src3$$constant & 0x1f);
11450 %}
11451
11452 ins_pipe(ialu_reg_reg_shift);
11453 %}
11454
11455 // This pattern is automatically generated from aarch64_ad.m4
11456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11457 // val | (-1 ^ (val >>> shift)) ==> orn
11458 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11459 iRegL src1, iRegL src2,
11460 immI src3, immL_M1 src4) %{
11461 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11462 ins_cost(1.9 * INSN_COST);
11463 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11464
11465 ins_encode %{
11466 __ orn(as_Register($dst$$reg),
11467 as_Register($src1$$reg),
11468 as_Register($src2$$reg),
11469 Assembler::LSR,
11470 $src3$$constant & 0x3f);
11471 %}
11472
11473 ins_pipe(ialu_reg_reg_shift);
11474 %}
11475
11476 // This pattern is automatically generated from aarch64_ad.m4
11477 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11478 // val | (-1 ^ (val >> shift)) ==> ornw
11479 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11480 iRegIorL2I src1, iRegIorL2I src2,
11481 immI src3, immI_M1 src4) %{
11482 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11483 ins_cost(1.9 * INSN_COST);
11484 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11485
11486 ins_encode %{
11487 __ ornw(as_Register($dst$$reg),
11488 as_Register($src1$$reg),
11489 as_Register($src2$$reg),
11490 Assembler::ASR,
11491 $src3$$constant & 0x1f);
11492 %}
11493
11494 ins_pipe(ialu_reg_reg_shift);
11495 %}
11496
11497 // This pattern is automatically generated from aarch64_ad.m4
11498 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11499 // val | (-1 ^ (val >> shift)) ==> orn
11500 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11501 iRegL src1, iRegL src2,
11502 immI src3, immL_M1 src4) %{
11503 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11504 ins_cost(1.9 * INSN_COST);
11505 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11506
11507 ins_encode %{
11508 __ orn(as_Register($dst$$reg),
11509 as_Register($src1$$reg),
11510 as_Register($src2$$reg),
11511 Assembler::ASR,
11512 $src3$$constant & 0x3f);
11513 %}
11514
11515 ins_pipe(ialu_reg_reg_shift);
11516 %}
11517
11518 // This pattern is automatically generated from aarch64_ad.m4
11519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11520 // val | (-1 ^ (val ror shift)) ==> ornw
11521 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11522 iRegIorL2I src1, iRegIorL2I src2,
11523 immI src3, immI_M1 src4) %{
11524 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11525 ins_cost(1.9 * INSN_COST);
11526 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11527
11528 ins_encode %{
11529 __ ornw(as_Register($dst$$reg),
11530 as_Register($src1$$reg),
11531 as_Register($src2$$reg),
11532 Assembler::ROR,
11533 $src3$$constant & 0x1f);
11534 %}
11535
11536 ins_pipe(ialu_reg_reg_shift);
11537 %}
11538
11539 // This pattern is automatically generated from aarch64_ad.m4
11540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11541 // val | (-1 ^ (val ror shift)) ==> orn
11542 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11543 iRegL src1, iRegL src2,
11544 immI src3, immL_M1 src4) %{
11545 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11546 ins_cost(1.9 * INSN_COST);
11547 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11548
11549 ins_encode %{
11550 __ orn(as_Register($dst$$reg),
11551 as_Register($src1$$reg),
11552 as_Register($src2$$reg),
11553 Assembler::ROR,
11554 $src3$$constant & 0x3f);
11555 %}
11556
11557 ins_pipe(ialu_reg_reg_shift);
11558 %}
11559
11560 // This pattern is automatically generated from aarch64_ad.m4
11561 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11562 // val | (-1 ^ (val << shift)) ==> ornw
11563 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11564 iRegIorL2I src1, iRegIorL2I src2,
11565 immI src3, immI_M1 src4) %{
11566 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11567 ins_cost(1.9 * INSN_COST);
11568 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11569
11570 ins_encode %{
11571 __ ornw(as_Register($dst$$reg),
11572 as_Register($src1$$reg),
11573 as_Register($src2$$reg),
11574 Assembler::LSL,
11575 $src3$$constant & 0x1f);
11576 %}
11577
11578 ins_pipe(ialu_reg_reg_shift);
11579 %}
11580
11581 // This pattern is automatically generated from aarch64_ad.m4
11582 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11583 // val | (-1 ^ (val << shift)) ==> orn
11584 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11585 iRegL src1, iRegL src2,
11586 immI src3, immL_M1 src4) %{
11587 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11588 ins_cost(1.9 * INSN_COST);
11589 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11590
11591 ins_encode %{
11592 __ orn(as_Register($dst$$reg),
11593 as_Register($src1$$reg),
11594 as_Register($src2$$reg),
11595 Assembler::LSL,
11596 $src3$$constant & 0x3f);
11597 %}
11598
11599 ins_pipe(ialu_reg_reg_shift);
11600 %}
11601
11602 // This pattern is automatically generated from aarch64_ad.m4
11603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11604 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11605 iRegIorL2I src1, iRegIorL2I src2,
11606 immI src3) %{
11607 match(Set dst (AndI src1 (URShiftI src2 src3)));
11608
11609 ins_cost(1.9 * INSN_COST);
11610 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11611
11612 ins_encode %{
11613 __ andw(as_Register($dst$$reg),
11614 as_Register($src1$$reg),
11615 as_Register($src2$$reg),
11616 Assembler::LSR,
11617 $src3$$constant & 0x1f);
11618 %}
11619
11620 ins_pipe(ialu_reg_reg_shift);
11621 %}
11622
11623 // This pattern is automatically generated from aarch64_ad.m4
11624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11625 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11626 iRegL src1, iRegL src2,
11627 immI src3) %{
11628 match(Set dst (AndL src1 (URShiftL src2 src3)));
11629
11630 ins_cost(1.9 * INSN_COST);
11631 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11632
11633 ins_encode %{
11634 __ andr(as_Register($dst$$reg),
11635 as_Register($src1$$reg),
11636 as_Register($src2$$reg),
11637 Assembler::LSR,
11638 $src3$$constant & 0x3f);
11639 %}
11640
11641 ins_pipe(ialu_reg_reg_shift);
11642 %}
11643
11644 // This pattern is automatically generated from aarch64_ad.m4
11645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11646 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11647 iRegIorL2I src1, iRegIorL2I src2,
11648 immI src3) %{
11649 match(Set dst (AndI src1 (RShiftI src2 src3)));
11650
11651 ins_cost(1.9 * INSN_COST);
11652 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11653
11654 ins_encode %{
11655 __ andw(as_Register($dst$$reg),
11656 as_Register($src1$$reg),
11657 as_Register($src2$$reg),
11658 Assembler::ASR,
11659 $src3$$constant & 0x1f);
11660 %}
11661
11662 ins_pipe(ialu_reg_reg_shift);
11663 %}
11664
11665 // This pattern is automatically generated from aarch64_ad.m4
11666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11667 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11668 iRegL src1, iRegL src2,
11669 immI src3) %{
11670 match(Set dst (AndL src1 (RShiftL src2 src3)));
11671
11672 ins_cost(1.9 * INSN_COST);
11673 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11674
11675 ins_encode %{
11676 __ andr(as_Register($dst$$reg),
11677 as_Register($src1$$reg),
11678 as_Register($src2$$reg),
11679 Assembler::ASR,
11680 $src3$$constant & 0x3f);
11681 %}
11682
11683 ins_pipe(ialu_reg_reg_shift);
11684 %}
11685
11686 // This pattern is automatically generated from aarch64_ad.m4
11687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11688 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11689 iRegIorL2I src1, iRegIorL2I src2,
11690 immI src3) %{
11691 match(Set dst (AndI src1 (LShiftI src2 src3)));
11692
11693 ins_cost(1.9 * INSN_COST);
11694 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11695
11696 ins_encode %{
11697 __ andw(as_Register($dst$$reg),
11698 as_Register($src1$$reg),
11699 as_Register($src2$$reg),
11700 Assembler::LSL,
11701 $src3$$constant & 0x1f);
11702 %}
11703
11704 ins_pipe(ialu_reg_reg_shift);
11705 %}
11706
11707 // This pattern is automatically generated from aarch64_ad.m4
11708 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11709 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11710 iRegL src1, iRegL src2,
11711 immI src3) %{
11712 match(Set dst (AndL src1 (LShiftL src2 src3)));
11713
11714 ins_cost(1.9 * INSN_COST);
11715 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11716
11717 ins_encode %{
11718 __ andr(as_Register($dst$$reg),
11719 as_Register($src1$$reg),
11720 as_Register($src2$$reg),
11721 Assembler::LSL,
11722 $src3$$constant & 0x3f);
11723 %}
11724
11725 ins_pipe(ialu_reg_reg_shift);
11726 %}
11727
11728 // This pattern is automatically generated from aarch64_ad.m4
11729 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11730 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11731 iRegIorL2I src1, iRegIorL2I src2,
11732 immI src3) %{
11733 match(Set dst (AndI src1 (RotateRight src2 src3)));
11734
11735 ins_cost(1.9 * INSN_COST);
11736 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11737
11738 ins_encode %{
11739 __ andw(as_Register($dst$$reg),
11740 as_Register($src1$$reg),
11741 as_Register($src2$$reg),
11742 Assembler::ROR,
11743 $src3$$constant & 0x1f);
11744 %}
11745
11746 ins_pipe(ialu_reg_reg_shift);
11747 %}
11748
11749 // This pattern is automatically generated from aarch64_ad.m4
11750 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11751 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11752 iRegL src1, iRegL src2,
11753 immI src3) %{
11754 match(Set dst (AndL src1 (RotateRight src2 src3)));
11755
11756 ins_cost(1.9 * INSN_COST);
11757 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11758
11759 ins_encode %{
11760 __ andr(as_Register($dst$$reg),
11761 as_Register($src1$$reg),
11762 as_Register($src2$$reg),
11763 Assembler::ROR,
11764 $src3$$constant & 0x3f);
11765 %}
11766
11767 ins_pipe(ialu_reg_reg_shift);
11768 %}
11769
11770 // This pattern is automatically generated from aarch64_ad.m4
11771 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11772 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11773 iRegIorL2I src1, iRegIorL2I src2,
11774 immI src3) %{
11775 match(Set dst (XorI src1 (URShiftI src2 src3)));
11776
11777 ins_cost(1.9 * INSN_COST);
11778 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11779
11780 ins_encode %{
11781 __ eorw(as_Register($dst$$reg),
11782 as_Register($src1$$reg),
11783 as_Register($src2$$reg),
11784 Assembler::LSR,
11785 $src3$$constant & 0x1f);
11786 %}
11787
11788 ins_pipe(ialu_reg_reg_shift);
11789 %}
11790
11791 // This pattern is automatically generated from aarch64_ad.m4
11792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11793 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11794 iRegL src1, iRegL src2,
11795 immI src3) %{
11796 match(Set dst (XorL src1 (URShiftL src2 src3)));
11797
11798 ins_cost(1.9 * INSN_COST);
11799 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11800
11801 ins_encode %{
11802 __ eor(as_Register($dst$$reg),
11803 as_Register($src1$$reg),
11804 as_Register($src2$$reg),
11805 Assembler::LSR,
11806 $src3$$constant & 0x3f);
11807 %}
11808
11809 ins_pipe(ialu_reg_reg_shift);
11810 %}
11811
11812 // This pattern is automatically generated from aarch64_ad.m4
11813 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11814 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11815 iRegIorL2I src1, iRegIorL2I src2,
11816 immI src3) %{
11817 match(Set dst (XorI src1 (RShiftI src2 src3)));
11818
11819 ins_cost(1.9 * INSN_COST);
11820 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11821
11822 ins_encode %{
11823 __ eorw(as_Register($dst$$reg),
11824 as_Register($src1$$reg),
11825 as_Register($src2$$reg),
11826 Assembler::ASR,
11827 $src3$$constant & 0x1f);
11828 %}
11829
11830 ins_pipe(ialu_reg_reg_shift);
11831 %}
11832
11833 // This pattern is automatically generated from aarch64_ad.m4
11834 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11835 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11836 iRegL src1, iRegL src2,
11837 immI src3) %{
11838 match(Set dst (XorL src1 (RShiftL src2 src3)));
11839
11840 ins_cost(1.9 * INSN_COST);
11841 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11842
11843 ins_encode %{
11844 __ eor(as_Register($dst$$reg),
11845 as_Register($src1$$reg),
11846 as_Register($src2$$reg),
11847 Assembler::ASR,
11848 $src3$$constant & 0x3f);
11849 %}
11850
11851 ins_pipe(ialu_reg_reg_shift);
11852 %}
11853
11854 // This pattern is automatically generated from aarch64_ad.m4
11855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11856 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11857 iRegIorL2I src1, iRegIorL2I src2,
11858 immI src3) %{
11859 match(Set dst (XorI src1 (LShiftI src2 src3)));
11860
11861 ins_cost(1.9 * INSN_COST);
11862 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11863
11864 ins_encode %{
11865 __ eorw(as_Register($dst$$reg),
11866 as_Register($src1$$reg),
11867 as_Register($src2$$reg),
11868 Assembler::LSL,
11869 $src3$$constant & 0x1f);
11870 %}
11871
11872 ins_pipe(ialu_reg_reg_shift);
11873 %}
11874
11875 // This pattern is automatically generated from aarch64_ad.m4
11876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11877 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11878 iRegL src1, iRegL src2,
11879 immI src3) %{
11880 match(Set dst (XorL src1 (LShiftL src2 src3)));
11881
11882 ins_cost(1.9 * INSN_COST);
11883 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11884
11885 ins_encode %{
11886 __ eor(as_Register($dst$$reg),
11887 as_Register($src1$$reg),
11888 as_Register($src2$$reg),
11889 Assembler::LSL,
11890 $src3$$constant & 0x3f);
11891 %}
11892
11893 ins_pipe(ialu_reg_reg_shift);
11894 %}
11895
11896 // This pattern is automatically generated from aarch64_ad.m4
11897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11898 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11899 iRegIorL2I src1, iRegIorL2I src2,
11900 immI src3) %{
11901 match(Set dst (XorI src1 (RotateRight src2 src3)));
11902
11903 ins_cost(1.9 * INSN_COST);
11904 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11905
11906 ins_encode %{
11907 __ eorw(as_Register($dst$$reg),
11908 as_Register($src1$$reg),
11909 as_Register($src2$$reg),
11910 Assembler::ROR,
11911 $src3$$constant & 0x1f);
11912 %}
11913
11914 ins_pipe(ialu_reg_reg_shift);
11915 %}
11916
11917 // This pattern is automatically generated from aarch64_ad.m4
11918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11919 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11920 iRegL src1, iRegL src2,
11921 immI src3) %{
11922 match(Set dst (XorL src1 (RotateRight src2 src3)));
11923
11924 ins_cost(1.9 * INSN_COST);
11925 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11926
11927 ins_encode %{
11928 __ eor(as_Register($dst$$reg),
11929 as_Register($src1$$reg),
11930 as_Register($src2$$reg),
11931 Assembler::ROR,
11932 $src3$$constant & 0x3f);
11933 %}
11934
11935 ins_pipe(ialu_reg_reg_shift);
11936 %}
11937
11938 // This pattern is automatically generated from aarch64_ad.m4
11939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11940 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11941 iRegIorL2I src1, iRegIorL2I src2,
11942 immI src3) %{
11943 match(Set dst (OrI src1 (URShiftI src2 src3)));
11944
11945 ins_cost(1.9 * INSN_COST);
11946 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11947
11948 ins_encode %{
11949 __ orrw(as_Register($dst$$reg),
11950 as_Register($src1$$reg),
11951 as_Register($src2$$reg),
11952 Assembler::LSR,
11953 $src3$$constant & 0x1f);
11954 %}
11955
11956 ins_pipe(ialu_reg_reg_shift);
11957 %}
11958
11959 // This pattern is automatically generated from aarch64_ad.m4
11960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11961 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11962 iRegL src1, iRegL src2,
11963 immI src3) %{
11964 match(Set dst (OrL src1 (URShiftL src2 src3)));
11965
11966 ins_cost(1.9 * INSN_COST);
11967 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11968
11969 ins_encode %{
11970 __ orr(as_Register($dst$$reg),
11971 as_Register($src1$$reg),
11972 as_Register($src2$$reg),
11973 Assembler::LSR,
11974 $src3$$constant & 0x3f);
11975 %}
11976
11977 ins_pipe(ialu_reg_reg_shift);
11978 %}
11979
11980 // This pattern is automatically generated from aarch64_ad.m4
11981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11982 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11983 iRegIorL2I src1, iRegIorL2I src2,
11984 immI src3) %{
11985 match(Set dst (OrI src1 (RShiftI src2 src3)));
11986
11987 ins_cost(1.9 * INSN_COST);
11988 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11989
11990 ins_encode %{
11991 __ orrw(as_Register($dst$$reg),
11992 as_Register($src1$$reg),
11993 as_Register($src2$$reg),
11994 Assembler::ASR,
11995 $src3$$constant & 0x1f);
11996 %}
11997
11998 ins_pipe(ialu_reg_reg_shift);
11999 %}
12000
12001 // This pattern is automatically generated from aarch64_ad.m4
12002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12003 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12004 iRegL src1, iRegL src2,
12005 immI src3) %{
12006 match(Set dst (OrL src1 (RShiftL src2 src3)));
12007
12008 ins_cost(1.9 * INSN_COST);
12009 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12010
12011 ins_encode %{
12012 __ orr(as_Register($dst$$reg),
12013 as_Register($src1$$reg),
12014 as_Register($src2$$reg),
12015 Assembler::ASR,
12016 $src3$$constant & 0x3f);
12017 %}
12018
12019 ins_pipe(ialu_reg_reg_shift);
12020 %}
12021
12022 // This pattern is automatically generated from aarch64_ad.m4
12023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12024 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12025 iRegIorL2I src1, iRegIorL2I src2,
12026 immI src3) %{
12027 match(Set dst (OrI src1 (LShiftI src2 src3)));
12028
12029 ins_cost(1.9 * INSN_COST);
12030 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12031
12032 ins_encode %{
12033 __ orrw(as_Register($dst$$reg),
12034 as_Register($src1$$reg),
12035 as_Register($src2$$reg),
12036 Assembler::LSL,
12037 $src3$$constant & 0x1f);
12038 %}
12039
12040 ins_pipe(ialu_reg_reg_shift);
12041 %}
12042
12043 // This pattern is automatically generated from aarch64_ad.m4
12044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12045 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12046 iRegL src1, iRegL src2,
12047 immI src3) %{
12048 match(Set dst (OrL src1 (LShiftL src2 src3)));
12049
12050 ins_cost(1.9 * INSN_COST);
12051 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12052
12053 ins_encode %{
12054 __ orr(as_Register($dst$$reg),
12055 as_Register($src1$$reg),
12056 as_Register($src2$$reg),
12057 Assembler::LSL,
12058 $src3$$constant & 0x3f);
12059 %}
12060
12061 ins_pipe(ialu_reg_reg_shift);
12062 %}
12063
12064 // This pattern is automatically generated from aarch64_ad.m4
12065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12066 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12067 iRegIorL2I src1, iRegIorL2I src2,
12068 immI src3) %{
12069 match(Set dst (OrI src1 (RotateRight src2 src3)));
12070
12071 ins_cost(1.9 * INSN_COST);
12072 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12073
12074 ins_encode %{
12075 __ orrw(as_Register($dst$$reg),
12076 as_Register($src1$$reg),
12077 as_Register($src2$$reg),
12078 Assembler::ROR,
12079 $src3$$constant & 0x1f);
12080 %}
12081
12082 ins_pipe(ialu_reg_reg_shift);
12083 %}
12084
12085 // This pattern is automatically generated from aarch64_ad.m4
12086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12087 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12088 iRegL src1, iRegL src2,
12089 immI src3) %{
12090 match(Set dst (OrL src1 (RotateRight src2 src3)));
12091
12092 ins_cost(1.9 * INSN_COST);
12093 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12094
12095 ins_encode %{
12096 __ orr(as_Register($dst$$reg),
12097 as_Register($src1$$reg),
12098 as_Register($src2$$reg),
12099 Assembler::ROR,
12100 $src3$$constant & 0x3f);
12101 %}
12102
12103 ins_pipe(ialu_reg_reg_shift);
12104 %}
12105
12106 // This pattern is automatically generated from aarch64_ad.m4
12107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12108 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12109 iRegIorL2I src1, iRegIorL2I src2,
12110 immI src3) %{
12111 match(Set dst (AddI src1 (URShiftI src2 src3)));
12112
12113 ins_cost(1.9 * INSN_COST);
12114 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12115
12116 ins_encode %{
12117 __ addw(as_Register($dst$$reg),
12118 as_Register($src1$$reg),
12119 as_Register($src2$$reg),
12120 Assembler::LSR,
12121 $src3$$constant & 0x1f);
12122 %}
12123
12124 ins_pipe(ialu_reg_reg_shift);
12125 %}
12126
12127 // This pattern is automatically generated from aarch64_ad.m4
12128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12129 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12130 iRegL src1, iRegL src2,
12131 immI src3) %{
12132 match(Set dst (AddL src1 (URShiftL src2 src3)));
12133
12134 ins_cost(1.9 * INSN_COST);
12135 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12136
12137 ins_encode %{
12138 __ add(as_Register($dst$$reg),
12139 as_Register($src1$$reg),
12140 as_Register($src2$$reg),
12141 Assembler::LSR,
12142 $src3$$constant & 0x3f);
12143 %}
12144
12145 ins_pipe(ialu_reg_reg_shift);
12146 %}
12147
12148 // This pattern is automatically generated from aarch64_ad.m4
12149 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12150 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12151 iRegIorL2I src1, iRegIorL2I src2,
12152 immI src3) %{
12153 match(Set dst (AddI src1 (RShiftI src2 src3)));
12154
12155 ins_cost(1.9 * INSN_COST);
12156 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12157
12158 ins_encode %{
12159 __ addw(as_Register($dst$$reg),
12160 as_Register($src1$$reg),
12161 as_Register($src2$$reg),
12162 Assembler::ASR,
12163 $src3$$constant & 0x1f);
12164 %}
12165
12166 ins_pipe(ialu_reg_reg_shift);
12167 %}
12168
12169 // This pattern is automatically generated from aarch64_ad.m4
12170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12171 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12172 iRegL src1, iRegL src2,
12173 immI src3) %{
12174 match(Set dst (AddL src1 (RShiftL src2 src3)));
12175
12176 ins_cost(1.9 * INSN_COST);
12177 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12178
12179 ins_encode %{
12180 __ add(as_Register($dst$$reg),
12181 as_Register($src1$$reg),
12182 as_Register($src2$$reg),
12183 Assembler::ASR,
12184 $src3$$constant & 0x3f);
12185 %}
12186
12187 ins_pipe(ialu_reg_reg_shift);
12188 %}
12189
12190 // This pattern is automatically generated from aarch64_ad.m4
12191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12192 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12193 iRegIorL2I src1, iRegIorL2I src2,
12194 immI src3) %{
12195 match(Set dst (AddI src1 (LShiftI src2 src3)));
12196
12197 ins_cost(1.9 * INSN_COST);
12198 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12199
12200 ins_encode %{
12201 __ addw(as_Register($dst$$reg),
12202 as_Register($src1$$reg),
12203 as_Register($src2$$reg),
12204 Assembler::LSL,
12205 $src3$$constant & 0x1f);
12206 %}
12207
12208 ins_pipe(ialu_reg_reg_shift);
12209 %}
12210
12211 // This pattern is automatically generated from aarch64_ad.m4
12212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12213 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12214 iRegL src1, iRegL src2,
12215 immI src3) %{
12216 match(Set dst (AddL src1 (LShiftL src2 src3)));
12217
12218 ins_cost(1.9 * INSN_COST);
12219 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12220
12221 ins_encode %{
12222 __ add(as_Register($dst$$reg),
12223 as_Register($src1$$reg),
12224 as_Register($src2$$reg),
12225 Assembler::LSL,
12226 $src3$$constant & 0x3f);
12227 %}
12228
12229 ins_pipe(ialu_reg_reg_shift);
12230 %}
12231
12232 // This pattern is automatically generated from aarch64_ad.m4
12233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12234 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12235 iRegIorL2I src1, iRegIorL2I src2,
12236 immI src3) %{
12237 match(Set dst (SubI src1 (URShiftI src2 src3)));
12238
12239 ins_cost(1.9 * INSN_COST);
12240 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12241
12242 ins_encode %{
12243 __ subw(as_Register($dst$$reg),
12244 as_Register($src1$$reg),
12245 as_Register($src2$$reg),
12246 Assembler::LSR,
12247 $src3$$constant & 0x1f);
12248 %}
12249
12250 ins_pipe(ialu_reg_reg_shift);
12251 %}
12252
12253 // This pattern is automatically generated from aarch64_ad.m4
12254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12255 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12256 iRegL src1, iRegL src2,
12257 immI src3) %{
12258 match(Set dst (SubL src1 (URShiftL src2 src3)));
12259
12260 ins_cost(1.9 * INSN_COST);
12261 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12262
12263 ins_encode %{
12264 __ sub(as_Register($dst$$reg),
12265 as_Register($src1$$reg),
12266 as_Register($src2$$reg),
12267 Assembler::LSR,
12268 $src3$$constant & 0x3f);
12269 %}
12270
12271 ins_pipe(ialu_reg_reg_shift);
12272 %}
12273
12274 // This pattern is automatically generated from aarch64_ad.m4
12275 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12276 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12277 iRegIorL2I src1, iRegIorL2I src2,
12278 immI src3) %{
12279 match(Set dst (SubI src1 (RShiftI src2 src3)));
12280
12281 ins_cost(1.9 * INSN_COST);
12282 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12283
12284 ins_encode %{
12285 __ subw(as_Register($dst$$reg),
12286 as_Register($src1$$reg),
12287 as_Register($src2$$reg),
12288 Assembler::ASR,
12289 $src3$$constant & 0x1f);
12290 %}
12291
12292 ins_pipe(ialu_reg_reg_shift);
12293 %}
12294
12295 // This pattern is automatically generated from aarch64_ad.m4
12296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12297 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12298 iRegL src1, iRegL src2,
12299 immI src3) %{
12300 match(Set dst (SubL src1 (RShiftL src2 src3)));
12301
12302 ins_cost(1.9 * INSN_COST);
12303 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12304
12305 ins_encode %{
12306 __ sub(as_Register($dst$$reg),
12307 as_Register($src1$$reg),
12308 as_Register($src2$$reg),
12309 Assembler::ASR,
12310 $src3$$constant & 0x3f);
12311 %}
12312
12313 ins_pipe(ialu_reg_reg_shift);
12314 %}
12315
12316 // This pattern is automatically generated from aarch64_ad.m4
12317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12318 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12319 iRegIorL2I src1, iRegIorL2I src2,
12320 immI src3) %{
12321 match(Set dst (SubI src1 (LShiftI src2 src3)));
12322
12323 ins_cost(1.9 * INSN_COST);
12324 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12325
12326 ins_encode %{
12327 __ subw(as_Register($dst$$reg),
12328 as_Register($src1$$reg),
12329 as_Register($src2$$reg),
12330 Assembler::LSL,
12331 $src3$$constant & 0x1f);
12332 %}
12333
12334 ins_pipe(ialu_reg_reg_shift);
12335 %}
12336
12337 // This pattern is automatically generated from aarch64_ad.m4
12338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12339 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12340 iRegL src1, iRegL src2,
12341 immI src3) %{
12342 match(Set dst (SubL src1 (LShiftL src2 src3)));
12343
12344 ins_cost(1.9 * INSN_COST);
12345 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12346
12347 ins_encode %{
12348 __ sub(as_Register($dst$$reg),
12349 as_Register($src1$$reg),
12350 as_Register($src2$$reg),
12351 Assembler::LSL,
12352 $src3$$constant & 0x3f);
12353 %}
12354
12355 ins_pipe(ialu_reg_reg_shift);
12356 %}
12357
12358 // This pattern is automatically generated from aarch64_ad.m4
12359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12360
12361 // Shift Left followed by Shift Right.
12362 // This idiom is used by the compiler for the i2b bytecode etc.
12363 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12364 %{
12365 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12366 ins_cost(INSN_COST * 2);
12367 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12368 ins_encode %{
12369 int lshift = $lshift_count$$constant & 63;
12370 int rshift = $rshift_count$$constant & 63;
12371 int s = 63 - lshift;
12372 int r = (rshift - lshift) & 63;
12373 __ sbfm(as_Register($dst$$reg),
12374 as_Register($src$$reg),
12375 r, s);
12376 %}
12377
12378 ins_pipe(ialu_reg_shift);
12379 %}
12380
12381 // This pattern is automatically generated from aarch64_ad.m4
12382 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12383
12384 // Shift Left followed by Shift Right.
12385 // This idiom is used by the compiler for the i2b bytecode etc.
12386 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12387 %{
12388 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12389 ins_cost(INSN_COST * 2);
12390 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12391 ins_encode %{
12392 int lshift = $lshift_count$$constant & 31;
12393 int rshift = $rshift_count$$constant & 31;
12394 int s = 31 - lshift;
12395 int r = (rshift - lshift) & 31;
12396 __ sbfmw(as_Register($dst$$reg),
12397 as_Register($src$$reg),
12398 r, s);
12399 %}
12400
12401 ins_pipe(ialu_reg_shift);
12402 %}
12403
12404 // This pattern is automatically generated from aarch64_ad.m4
12405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12406
12407 // Shift Left followed by Shift Right.
12408 // This idiom is used by the compiler for the i2b bytecode etc.
12409 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12410 %{
12411 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12412 ins_cost(INSN_COST * 2);
12413 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12414 ins_encode %{
12415 int lshift = $lshift_count$$constant & 63;
12416 int rshift = $rshift_count$$constant & 63;
12417 int s = 63 - lshift;
12418 int r = (rshift - lshift) & 63;
12419 __ ubfm(as_Register($dst$$reg),
12420 as_Register($src$$reg),
12421 r, s);
12422 %}
12423
12424 ins_pipe(ialu_reg_shift);
12425 %}
12426
12427 // This pattern is automatically generated from aarch64_ad.m4
12428 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12429
12430 // Shift Left followed by Shift Right.
12431 // This idiom is used by the compiler for the i2b bytecode etc.
12432 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12433 %{
12434 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12435 ins_cost(INSN_COST * 2);
12436 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12437 ins_encode %{
12438 int lshift = $lshift_count$$constant & 31;
12439 int rshift = $rshift_count$$constant & 31;
12440 int s = 31 - lshift;
12441 int r = (rshift - lshift) & 31;
12442 __ ubfmw(as_Register($dst$$reg),
12443 as_Register($src$$reg),
12444 r, s);
12445 %}
12446
12447 ins_pipe(ialu_reg_shift);
12448 %}
12449
12450 // Bitfield extract with shift & mask
12451
12452 // This pattern is automatically generated from aarch64_ad.m4
12453 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12454 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12455 %{
12456 match(Set dst (AndI (URShiftI src rshift) mask));
12457 // Make sure we are not going to exceed what ubfxw can do.
12458 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12459
12460 ins_cost(INSN_COST);
12461 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12462 ins_encode %{
12463 int rshift = $rshift$$constant & 31;
12464 intptr_t mask = $mask$$constant;
12465 int width = exact_log2(mask+1);
12466 __ ubfxw(as_Register($dst$$reg),
12467 as_Register($src$$reg), rshift, width);
12468 %}
12469 ins_pipe(ialu_reg_shift);
12470 %}
12471
12472 // This pattern is automatically generated from aarch64_ad.m4
12473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12474 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12475 %{
12476 match(Set dst (AndL (URShiftL src rshift) mask));
12477 // Make sure we are not going to exceed what ubfx can do.
12478 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12479
12480 ins_cost(INSN_COST);
12481 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12482 ins_encode %{
12483 int rshift = $rshift$$constant & 63;
12484 intptr_t mask = $mask$$constant;
12485 int width = exact_log2_long(mask+1);
12486 __ ubfx(as_Register($dst$$reg),
12487 as_Register($src$$reg), rshift, width);
12488 %}
12489 ins_pipe(ialu_reg_shift);
12490 %}
12491
12492
12493 // This pattern is automatically generated from aarch64_ad.m4
12494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12495
12496 // We can use ubfx when extending an And with a mask when we know mask
12497 // is positive. We know that because immI_bitmask guarantees it.
12498 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12499 %{
12500 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12501 // Make sure we are not going to exceed what ubfxw can do.
12502 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12503
12504 ins_cost(INSN_COST * 2);
12505 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12506 ins_encode %{
12507 int rshift = $rshift$$constant & 31;
12508 intptr_t mask = $mask$$constant;
12509 int width = exact_log2(mask+1);
12510 __ ubfx(as_Register($dst$$reg),
12511 as_Register($src$$reg), rshift, width);
12512 %}
12513 ins_pipe(ialu_reg_shift);
12514 %}
12515
12516
12517 // This pattern is automatically generated from aarch64_ad.m4
12518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12519
12520 // We can use ubfiz when masking by a positive number and then left shifting the result.
12521 // We know that the mask is positive because immI_bitmask guarantees it.
12522 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12523 %{
12524 match(Set dst (LShiftI (AndI src mask) lshift));
12525 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12526
12527 ins_cost(INSN_COST);
12528 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12529 ins_encode %{
12530 int lshift = $lshift$$constant & 31;
12531 intptr_t mask = $mask$$constant;
12532 int width = exact_log2(mask+1);
12533 __ ubfizw(as_Register($dst$$reg),
12534 as_Register($src$$reg), lshift, width);
12535 %}
12536 ins_pipe(ialu_reg_shift);
12537 %}
12538
12539 // This pattern is automatically generated from aarch64_ad.m4
12540 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12541
12542 // We can use ubfiz when masking by a positive number and then left shifting the result.
12543 // We know that the mask is positive because immL_bitmask guarantees it.
12544 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12545 %{
12546 match(Set dst (LShiftL (AndL src mask) lshift));
12547 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12548
12549 ins_cost(INSN_COST);
12550 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12551 ins_encode %{
12552 int lshift = $lshift$$constant & 63;
12553 intptr_t mask = $mask$$constant;
12554 int width = exact_log2_long(mask+1);
12555 __ ubfiz(as_Register($dst$$reg),
12556 as_Register($src$$reg), lshift, width);
12557 %}
12558 ins_pipe(ialu_reg_shift);
12559 %}
12560
12561 // This pattern is automatically generated from aarch64_ad.m4
12562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12563
12564 // We can use ubfiz when masking by a positive number and then left shifting the result.
12565 // We know that the mask is positive because immI_bitmask guarantees it.
12566 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12567 %{
12568 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12569 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12570
12571 ins_cost(INSN_COST);
12572 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12573 ins_encode %{
12574 int lshift = $lshift$$constant & 31;
12575 intptr_t mask = $mask$$constant;
12576 int width = exact_log2(mask+1);
12577 __ ubfizw(as_Register($dst$$reg),
12578 as_Register($src$$reg), lshift, width);
12579 %}
12580 ins_pipe(ialu_reg_shift);
12581 %}
12582
12583 // This pattern is automatically generated from aarch64_ad.m4
12584 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12585
12586 // We can use ubfiz when masking by a positive number and then left shifting the result.
12587 // We know that the mask is positive because immL_bitmask guarantees it.
12588 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12589 %{
12590 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12591 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12592
12593 ins_cost(INSN_COST);
12594 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12595 ins_encode %{
12596 int lshift = $lshift$$constant & 63;
12597 intptr_t mask = $mask$$constant;
12598 int width = exact_log2_long(mask+1);
12599 __ ubfiz(as_Register($dst$$reg),
12600 as_Register($src$$reg), lshift, width);
12601 %}
12602 ins_pipe(ialu_reg_shift);
12603 %}
12604
12605
12606 // This pattern is automatically generated from aarch64_ad.m4
12607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12608
12609 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12610 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12611 %{
12612 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12613 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12614
12615 ins_cost(INSN_COST);
12616 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12617 ins_encode %{
12618 int lshift = $lshift$$constant & 63;
12619 intptr_t mask = $mask$$constant;
12620 int width = exact_log2(mask+1);
12621 __ ubfiz(as_Register($dst$$reg),
12622 as_Register($src$$reg), lshift, width);
12623 %}
12624 ins_pipe(ialu_reg_shift);
12625 %}
12626
12627 // This pattern is automatically generated from aarch64_ad.m4
12628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12629
12630 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12631 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12632 %{
12633 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12634 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12635
12636 ins_cost(INSN_COST);
12637 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12638 ins_encode %{
12639 int lshift = $lshift$$constant & 31;
12640 intptr_t mask = $mask$$constant;
12641 int width = exact_log2(mask+1);
12642 __ ubfiz(as_Register($dst$$reg),
12643 as_Register($src$$reg), lshift, width);
12644 %}
12645 ins_pipe(ialu_reg_shift);
12646 %}
12647
12648 // This pattern is automatically generated from aarch64_ad.m4
12649 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12650
12651 // Can skip int2long conversions after AND with small bitmask
12652 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12653 %{
12654 match(Set dst (ConvI2L (AndI src msk)));
12655 ins_cost(INSN_COST);
12656 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12657 ins_encode %{
12658 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12659 %}
12660 ins_pipe(ialu_reg_shift);
12661 %}
12662
12663
12664 // Rotations
12665
12666 // This pattern is automatically generated from aarch64_ad.m4
12667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12668 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12669 %{
12670 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12671 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12672
12673 ins_cost(INSN_COST);
12674 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12675
12676 ins_encode %{
12677 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12678 $rshift$$constant & 63);
12679 %}
12680 ins_pipe(ialu_reg_reg_extr);
12681 %}
12682
12683
12684 // This pattern is automatically generated from aarch64_ad.m4
12685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12686 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12687 %{
12688 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12689 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12690
12691 ins_cost(INSN_COST);
12692 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12693
12694 ins_encode %{
12695 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12696 $rshift$$constant & 31);
12697 %}
12698 ins_pipe(ialu_reg_reg_extr);
12699 %}
12700
12701
12702 // This pattern is automatically generated from aarch64_ad.m4
12703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12704 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12705 %{
12706 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12707 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12708
12709 ins_cost(INSN_COST);
12710 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12711
12712 ins_encode %{
12713 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12714 $rshift$$constant & 63);
12715 %}
12716 ins_pipe(ialu_reg_reg_extr);
12717 %}
12718
12719
12720 // This pattern is automatically generated from aarch64_ad.m4
12721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12722 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12723 %{
12724 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12725 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12726
12727 ins_cost(INSN_COST);
12728 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12729
12730 ins_encode %{
12731 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12732 $rshift$$constant & 31);
12733 %}
12734 ins_pipe(ialu_reg_reg_extr);
12735 %}
12736
12737 // This pattern is automatically generated from aarch64_ad.m4
12738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12739 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12740 %{
12741 match(Set dst (RotateRight src shift));
12742
12743 ins_cost(INSN_COST);
12744 format %{ "ror $dst, $src, $shift" %}
12745
12746 ins_encode %{
12747 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12748 $shift$$constant & 0x1f);
12749 %}
12750 ins_pipe(ialu_reg_reg_vshift);
12751 %}
12752
12753 // This pattern is automatically generated from aarch64_ad.m4
12754 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12755 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12756 %{
12757 match(Set dst (RotateRight src shift));
12758
12759 ins_cost(INSN_COST);
12760 format %{ "ror $dst, $src, $shift" %}
12761
12762 ins_encode %{
12763 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12764 $shift$$constant & 0x3f);
12765 %}
12766 ins_pipe(ialu_reg_reg_vshift);
12767 %}
12768
12769 // This pattern is automatically generated from aarch64_ad.m4
12770 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12771 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12772 %{
12773 match(Set dst (RotateRight src shift));
12774
12775 ins_cost(INSN_COST);
12776 format %{ "ror $dst, $src, $shift" %}
12777
12778 ins_encode %{
12779 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12780 %}
12781 ins_pipe(ialu_reg_reg_vshift);
12782 %}
12783
12784 // This pattern is automatically generated from aarch64_ad.m4
12785 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12786 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12787 %{
12788 match(Set dst (RotateRight src shift));
12789
12790 ins_cost(INSN_COST);
12791 format %{ "ror $dst, $src, $shift" %}
12792
12793 ins_encode %{
12794 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12795 %}
12796 ins_pipe(ialu_reg_reg_vshift);
12797 %}
12798
12799 // This pattern is automatically generated from aarch64_ad.m4
12800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12801 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12802 %{
12803 match(Set dst (RotateLeft src shift));
12804
12805 ins_cost(INSN_COST);
12806 format %{ "rol $dst, $src, $shift" %}
12807
12808 ins_encode %{
12809 __ subw(rscratch1, zr, as_Register($shift$$reg));
12810 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12811 %}
12812 ins_pipe(ialu_reg_reg_vshift);
12813 %}
12814
12815 // This pattern is automatically generated from aarch64_ad.m4
12816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12817 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12818 %{
12819 match(Set dst (RotateLeft src shift));
12820
12821 ins_cost(INSN_COST);
12822 format %{ "rol $dst, $src, $shift" %}
12823
12824 ins_encode %{
12825 __ subw(rscratch1, zr, as_Register($shift$$reg));
12826 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12827 %}
12828 ins_pipe(ialu_reg_reg_vshift);
12829 %}
12830
12831
12832 // Add/subtract (extended)
12833
12834 // This pattern is automatically generated from aarch64_ad.m4
12835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12836 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12837 %{
12838 match(Set dst (AddL src1 (ConvI2L src2)));
12839 ins_cost(INSN_COST);
12840 format %{ "add $dst, $src1, $src2, sxtw" %}
12841
12842 ins_encode %{
12843 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12844 as_Register($src2$$reg), ext::sxtw);
12845 %}
12846 ins_pipe(ialu_reg_reg);
12847 %}
12848
12849 // This pattern is automatically generated from aarch64_ad.m4
12850 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12851 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12852 %{
12853 match(Set dst (SubL src1 (ConvI2L src2)));
12854 ins_cost(INSN_COST);
12855 format %{ "sub $dst, $src1, $src2, sxtw" %}
12856
12857 ins_encode %{
12858 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12859 as_Register($src2$$reg), ext::sxtw);
12860 %}
12861 ins_pipe(ialu_reg_reg);
12862 %}
12863
12864 // This pattern is automatically generated from aarch64_ad.m4
12865 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12866 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12867 %{
12868 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12869 ins_cost(INSN_COST);
12870 format %{ "add $dst, $src1, $src2, sxth" %}
12871
12872 ins_encode %{
12873 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12874 as_Register($src2$$reg), ext::sxth);
12875 %}
12876 ins_pipe(ialu_reg_reg);
12877 %}
12878
12879 // This pattern is automatically generated from aarch64_ad.m4
12880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12881 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12882 %{
12883 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12884 ins_cost(INSN_COST);
12885 format %{ "add $dst, $src1, $src2, sxtb" %}
12886
12887 ins_encode %{
12888 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12889 as_Register($src2$$reg), ext::sxtb);
12890 %}
12891 ins_pipe(ialu_reg_reg);
12892 %}
12893
12894 // This pattern is automatically generated from aarch64_ad.m4
12895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12896 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12897 %{
12898 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12899 ins_cost(INSN_COST);
12900 format %{ "add $dst, $src1, $src2, uxtb" %}
12901
12902 ins_encode %{
12903 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12904 as_Register($src2$$reg), ext::uxtb);
12905 %}
12906 ins_pipe(ialu_reg_reg);
12907 %}
12908
12909 // This pattern is automatically generated from aarch64_ad.m4
12910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12911 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12912 %{
12913 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12914 ins_cost(INSN_COST);
12915 format %{ "add $dst, $src1, $src2, sxth" %}
12916
12917 ins_encode %{
12918 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12919 as_Register($src2$$reg), ext::sxth);
12920 %}
12921 ins_pipe(ialu_reg_reg);
12922 %}
12923
12924 // This pattern is automatically generated from aarch64_ad.m4
12925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12926 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12927 %{
12928 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12929 ins_cost(INSN_COST);
12930 format %{ "add $dst, $src1, $src2, sxtw" %}
12931
12932 ins_encode %{
12933 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12934 as_Register($src2$$reg), ext::sxtw);
12935 %}
12936 ins_pipe(ialu_reg_reg);
12937 %}
12938
12939 // This pattern is automatically generated from aarch64_ad.m4
12940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12941 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12942 %{
12943 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12944 ins_cost(INSN_COST);
12945 format %{ "add $dst, $src1, $src2, sxtb" %}
12946
12947 ins_encode %{
12948 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12949 as_Register($src2$$reg), ext::sxtb);
12950 %}
12951 ins_pipe(ialu_reg_reg);
12952 %}
12953
12954 // This pattern is automatically generated from aarch64_ad.m4
12955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12956 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12957 %{
12958 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12959 ins_cost(INSN_COST);
12960 format %{ "add $dst, $src1, $src2, uxtb" %}
12961
12962 ins_encode %{
12963 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12964 as_Register($src2$$reg), ext::uxtb);
12965 %}
12966 ins_pipe(ialu_reg_reg);
12967 %}
12968
12969 // This pattern is automatically generated from aarch64_ad.m4
12970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12971 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12972 %{
12973 match(Set dst (AddI src1 (AndI src2 mask)));
12974 ins_cost(INSN_COST);
12975 format %{ "addw $dst, $src1, $src2, uxtb" %}
12976
12977 ins_encode %{
12978 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12979 as_Register($src2$$reg), ext::uxtb);
12980 %}
12981 ins_pipe(ialu_reg_reg);
12982 %}
12983
12984 // This pattern is automatically generated from aarch64_ad.m4
12985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12986 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12987 %{
12988 match(Set dst (AddI src1 (AndI src2 mask)));
12989 ins_cost(INSN_COST);
12990 format %{ "addw $dst, $src1, $src2, uxth" %}
12991
12992 ins_encode %{
12993 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12994 as_Register($src2$$reg), ext::uxth);
12995 %}
12996 ins_pipe(ialu_reg_reg);
12997 %}
12998
12999 // This pattern is automatically generated from aarch64_ad.m4
13000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13001 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13002 %{
13003 match(Set dst (AddL src1 (AndL src2 mask)));
13004 ins_cost(INSN_COST);
13005 format %{ "add $dst, $src1, $src2, uxtb" %}
13006
13007 ins_encode %{
13008 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13009 as_Register($src2$$reg), ext::uxtb);
13010 %}
13011 ins_pipe(ialu_reg_reg);
13012 %}
13013
13014 // This pattern is automatically generated from aarch64_ad.m4
13015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13016 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13017 %{
13018 match(Set dst (AddL src1 (AndL src2 mask)));
13019 ins_cost(INSN_COST);
13020 format %{ "add $dst, $src1, $src2, uxth" %}
13021
13022 ins_encode %{
13023 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13024 as_Register($src2$$reg), ext::uxth);
13025 %}
13026 ins_pipe(ialu_reg_reg);
13027 %}
13028
13029 // This pattern is automatically generated from aarch64_ad.m4
13030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13031 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13032 %{
13033 match(Set dst (AddL src1 (AndL src2 mask)));
13034 ins_cost(INSN_COST);
13035 format %{ "add $dst, $src1, $src2, uxtw" %}
13036
13037 ins_encode %{
13038 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13039 as_Register($src2$$reg), ext::uxtw);
13040 %}
13041 ins_pipe(ialu_reg_reg);
13042 %}
13043
13044 // This pattern is automatically generated from aarch64_ad.m4
13045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13046 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13047 %{
13048 match(Set dst (SubI src1 (AndI src2 mask)));
13049 ins_cost(INSN_COST);
13050 format %{ "subw $dst, $src1, $src2, uxtb" %}
13051
13052 ins_encode %{
13053 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13054 as_Register($src2$$reg), ext::uxtb);
13055 %}
13056 ins_pipe(ialu_reg_reg);
13057 %}
13058
13059 // This pattern is automatically generated from aarch64_ad.m4
13060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13061 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13062 %{
13063 match(Set dst (SubI src1 (AndI src2 mask)));
13064 ins_cost(INSN_COST);
13065 format %{ "subw $dst, $src1, $src2, uxth" %}
13066
13067 ins_encode %{
13068 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13069 as_Register($src2$$reg), ext::uxth);
13070 %}
13071 ins_pipe(ialu_reg_reg);
13072 %}
13073
13074 // This pattern is automatically generated from aarch64_ad.m4
13075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13076 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13077 %{
13078 match(Set dst (SubL src1 (AndL src2 mask)));
13079 ins_cost(INSN_COST);
13080 format %{ "sub $dst, $src1, $src2, uxtb" %}
13081
13082 ins_encode %{
13083 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13084 as_Register($src2$$reg), ext::uxtb);
13085 %}
13086 ins_pipe(ialu_reg_reg);
13087 %}
13088
13089 // This pattern is automatically generated from aarch64_ad.m4
13090 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13091 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13092 %{
13093 match(Set dst (SubL src1 (AndL src2 mask)));
13094 ins_cost(INSN_COST);
13095 format %{ "sub $dst, $src1, $src2, uxth" %}
13096
13097 ins_encode %{
13098 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13099 as_Register($src2$$reg), ext::uxth);
13100 %}
13101 ins_pipe(ialu_reg_reg);
13102 %}
13103
13104 // This pattern is automatically generated from aarch64_ad.m4
13105 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13106 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13107 %{
13108 match(Set dst (SubL src1 (AndL src2 mask)));
13109 ins_cost(INSN_COST);
13110 format %{ "sub $dst, $src1, $src2, uxtw" %}
13111
13112 ins_encode %{
13113 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13114 as_Register($src2$$reg), ext::uxtw);
13115 %}
13116 ins_pipe(ialu_reg_reg);
13117 %}
13118
13119
13120 // This pattern is automatically generated from aarch64_ad.m4
13121 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13122 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13123 %{
13124 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13125 ins_cost(1.9 * INSN_COST);
13126 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13127
13128 ins_encode %{
13129 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13130 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13131 %}
13132 ins_pipe(ialu_reg_reg_shift);
13133 %}
13134
13135 // This pattern is automatically generated from aarch64_ad.m4
13136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13137 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13138 %{
13139 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13140 ins_cost(1.9 * INSN_COST);
13141 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13142
13143 ins_encode %{
13144 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13145 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13146 %}
13147 ins_pipe(ialu_reg_reg_shift);
13148 %}
13149
13150 // This pattern is automatically generated from aarch64_ad.m4
13151 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13152 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13153 %{
13154 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13155 ins_cost(1.9 * INSN_COST);
13156 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13157
13158 ins_encode %{
13159 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13160 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13161 %}
13162 ins_pipe(ialu_reg_reg_shift);
13163 %}
13164
13165 // This pattern is automatically generated from aarch64_ad.m4
13166 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13167 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13168 %{
13169 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13170 ins_cost(1.9 * INSN_COST);
13171 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13172
13173 ins_encode %{
13174 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13175 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13176 %}
13177 ins_pipe(ialu_reg_reg_shift);
13178 %}
13179
13180 // This pattern is automatically generated from aarch64_ad.m4
13181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13182 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13183 %{
13184 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13185 ins_cost(1.9 * INSN_COST);
13186 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13187
13188 ins_encode %{
13189 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13190 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13191 %}
13192 ins_pipe(ialu_reg_reg_shift);
13193 %}
13194
13195 // This pattern is automatically generated from aarch64_ad.m4
13196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13197 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13198 %{
13199 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13200 ins_cost(1.9 * INSN_COST);
13201 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13202
13203 ins_encode %{
13204 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13205 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13206 %}
13207 ins_pipe(ialu_reg_reg_shift);
13208 %}
13209
13210 // This pattern is automatically generated from aarch64_ad.m4
13211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13212 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13213 %{
13214 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13215 ins_cost(1.9 * INSN_COST);
13216 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13217
13218 ins_encode %{
13219 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13220 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13221 %}
13222 ins_pipe(ialu_reg_reg_shift);
13223 %}
13224
13225 // This pattern is automatically generated from aarch64_ad.m4
13226 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13227 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13228 %{
13229 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13230 ins_cost(1.9 * INSN_COST);
13231 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13232
13233 ins_encode %{
13234 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13235 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13236 %}
13237 ins_pipe(ialu_reg_reg_shift);
13238 %}
13239
13240 // This pattern is automatically generated from aarch64_ad.m4
13241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13242 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13243 %{
13244 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13245 ins_cost(1.9 * INSN_COST);
13246 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13247
13248 ins_encode %{
13249 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13250 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13251 %}
13252 ins_pipe(ialu_reg_reg_shift);
13253 %}
13254
13255 // This pattern is automatically generated from aarch64_ad.m4
13256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13257 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13258 %{
13259 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13260 ins_cost(1.9 * INSN_COST);
13261 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13262
13263 ins_encode %{
13264 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13265 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13266 %}
13267 ins_pipe(ialu_reg_reg_shift);
13268 %}
13269
13270 // This pattern is automatically generated from aarch64_ad.m4
13271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13272 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13273 %{
13274 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13275 ins_cost(1.9 * INSN_COST);
13276 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13277
13278 ins_encode %{
13279 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13280 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13281 %}
13282 ins_pipe(ialu_reg_reg_shift);
13283 %}
13284
13285 // This pattern is automatically generated from aarch64_ad.m4
13286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13287 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13288 %{
13289 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13290 ins_cost(1.9 * INSN_COST);
13291 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13292
13293 ins_encode %{
13294 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13295 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13296 %}
13297 ins_pipe(ialu_reg_reg_shift);
13298 %}
13299
13300 // This pattern is automatically generated from aarch64_ad.m4
13301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13302 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13303 %{
13304 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13305 ins_cost(1.9 * INSN_COST);
13306 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13307
13308 ins_encode %{
13309 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13310 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13311 %}
13312 ins_pipe(ialu_reg_reg_shift);
13313 %}
13314
13315 // This pattern is automatically generated from aarch64_ad.m4
13316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13317 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13318 %{
13319 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13320 ins_cost(1.9 * INSN_COST);
13321 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13322
13323 ins_encode %{
13324 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13325 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13326 %}
13327 ins_pipe(ialu_reg_reg_shift);
13328 %}
13329
13330 // This pattern is automatically generated from aarch64_ad.m4
13331 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13332 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13333 %{
13334 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13335 ins_cost(1.9 * INSN_COST);
13336 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13337
13338 ins_encode %{
13339 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13340 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13341 %}
13342 ins_pipe(ialu_reg_reg_shift);
13343 %}
13344
13345 // This pattern is automatically generated from aarch64_ad.m4
13346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13347 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13348 %{
13349 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13350 ins_cost(1.9 * INSN_COST);
13351 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13352
13353 ins_encode %{
13354 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13355 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13356 %}
13357 ins_pipe(ialu_reg_reg_shift);
13358 %}
13359
13360 // This pattern is automatically generated from aarch64_ad.m4
13361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13362 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13363 %{
13364 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13365 ins_cost(1.9 * INSN_COST);
13366 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13367
13368 ins_encode %{
13369 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13370 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13371 %}
13372 ins_pipe(ialu_reg_reg_shift);
13373 %}
13374
13375 // This pattern is automatically generated from aarch64_ad.m4
13376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13377 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13378 %{
13379 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13380 ins_cost(1.9 * INSN_COST);
13381 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13382
13383 ins_encode %{
13384 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13385 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13386 %}
13387 ins_pipe(ialu_reg_reg_shift);
13388 %}
13389
13390 // This pattern is automatically generated from aarch64_ad.m4
13391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13392 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13393 %{
13394 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13395 ins_cost(1.9 * INSN_COST);
13396 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13397
13398 ins_encode %{
13399 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13400 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13401 %}
13402 ins_pipe(ialu_reg_reg_shift);
13403 %}
13404
13405 // This pattern is automatically generated from aarch64_ad.m4
13406 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13407 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13408 %{
13409 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13410 ins_cost(1.9 * INSN_COST);
13411 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13412
13413 ins_encode %{
13414 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13415 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13416 %}
13417 ins_pipe(ialu_reg_reg_shift);
13418 %}
13419
13420 // This pattern is automatically generated from aarch64_ad.m4
13421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13422 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13423 %{
13424 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13425 ins_cost(1.9 * INSN_COST);
13426 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13427
13428 ins_encode %{
13429 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13430 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13431 %}
13432 ins_pipe(ialu_reg_reg_shift);
13433 %}
13434
13435 // This pattern is automatically generated from aarch64_ad.m4
13436 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13437 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13438 %{
13439 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13440 ins_cost(1.9 * INSN_COST);
13441 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13442
13443 ins_encode %{
13444 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13445 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13446 %}
13447 ins_pipe(ialu_reg_reg_shift);
13448 %}
13449
13450 // This pattern is automatically generated from aarch64_ad.m4
13451 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13452 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13453 %{
13454 effect(DEF dst, USE src1, USE src2, USE cr);
13455 ins_cost(INSN_COST * 2);
13456 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13457
13458 ins_encode %{
13459 __ cselw($dst$$Register,
13460 $src1$$Register,
13461 $src2$$Register,
13462 Assembler::LT);
13463 %}
13464 ins_pipe(icond_reg_reg);
13465 %}
13466
13467 // This pattern is automatically generated from aarch64_ad.m4
13468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13469 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13470 %{
13471 effect(DEF dst, USE src1, USE src2, USE cr);
13472 ins_cost(INSN_COST * 2);
13473 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13474
13475 ins_encode %{
13476 __ cselw($dst$$Register,
13477 $src1$$Register,
13478 $src2$$Register,
13479 Assembler::GT);
13480 %}
13481 ins_pipe(icond_reg_reg);
13482 %}
13483
13484 // This pattern is automatically generated from aarch64_ad.m4
13485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13486 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13487 %{
13488 effect(DEF dst, USE src1, USE cr);
13489 ins_cost(INSN_COST * 2);
13490 format %{ "cselw $dst, $src1, zr lt\t" %}
13491
13492 ins_encode %{
13493 __ cselw($dst$$Register,
13494 $src1$$Register,
13495 zr,
13496 Assembler::LT);
13497 %}
13498 ins_pipe(icond_reg);
13499 %}
13500
13501 // This pattern is automatically generated from aarch64_ad.m4
13502 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13503 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13504 %{
13505 effect(DEF dst, USE src1, USE cr);
13506 ins_cost(INSN_COST * 2);
13507 format %{ "cselw $dst, $src1, zr gt\t" %}
13508
13509 ins_encode %{
13510 __ cselw($dst$$Register,
13511 $src1$$Register,
13512 zr,
13513 Assembler::GT);
13514 %}
13515 ins_pipe(icond_reg);
13516 %}
13517
13518 // This pattern is automatically generated from aarch64_ad.m4
13519 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13520 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13521 %{
13522 effect(DEF dst, USE src1, USE cr);
13523 ins_cost(INSN_COST * 2);
13524 format %{ "csincw $dst, $src1, zr le\t" %}
13525
13526 ins_encode %{
13527 __ csincw($dst$$Register,
13528 $src1$$Register,
13529 zr,
13530 Assembler::LE);
13531 %}
13532 ins_pipe(icond_reg);
13533 %}
13534
13535 // This pattern is automatically generated from aarch64_ad.m4
13536 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13537 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13538 %{
13539 effect(DEF dst, USE src1, USE cr);
13540 ins_cost(INSN_COST * 2);
13541 format %{ "csincw $dst, $src1, zr gt\t" %}
13542
13543 ins_encode %{
13544 __ csincw($dst$$Register,
13545 $src1$$Register,
13546 zr,
13547 Assembler::GT);
13548 %}
13549 ins_pipe(icond_reg);
13550 %}
13551
13552 // This pattern is automatically generated from aarch64_ad.m4
13553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13554 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13555 %{
13556 effect(DEF dst, USE src1, USE cr);
13557 ins_cost(INSN_COST * 2);
13558 format %{ "csinvw $dst, $src1, zr lt\t" %}
13559
13560 ins_encode %{
13561 __ csinvw($dst$$Register,
13562 $src1$$Register,
13563 zr,
13564 Assembler::LT);
13565 %}
13566 ins_pipe(icond_reg);
13567 %}
13568
13569 // This pattern is automatically generated from aarch64_ad.m4
13570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13571 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13572 %{
13573 effect(DEF dst, USE src1, USE cr);
13574 ins_cost(INSN_COST * 2);
13575 format %{ "csinvw $dst, $src1, zr ge\t" %}
13576
13577 ins_encode %{
13578 __ csinvw($dst$$Register,
13579 $src1$$Register,
13580 zr,
13581 Assembler::GE);
13582 %}
13583 ins_pipe(icond_reg);
13584 %}
13585
13586 // This pattern is automatically generated from aarch64_ad.m4
13587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13588 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13589 %{
13590 match(Set dst (MinI src imm));
13591 ins_cost(INSN_COST * 3);
13592 expand %{
13593 rFlagsReg cr;
13594 compI_reg_imm0(cr, src);
13595 cmovI_reg_imm0_lt(dst, src, cr);
13596 %}
13597 %}
13598
13599 // This pattern is automatically generated from aarch64_ad.m4
13600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13601 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13602 %{
13603 match(Set dst (MinI imm src));
13604 ins_cost(INSN_COST * 3);
13605 expand %{
13606 rFlagsReg cr;
13607 compI_reg_imm0(cr, src);
13608 cmovI_reg_imm0_lt(dst, src, cr);
13609 %}
13610 %}
13611
13612 // This pattern is automatically generated from aarch64_ad.m4
13613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13614 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13615 %{
13616 match(Set dst (MinI src imm));
13617 ins_cost(INSN_COST * 3);
13618 expand %{
13619 rFlagsReg cr;
13620 compI_reg_imm0(cr, src);
13621 cmovI_reg_imm1_le(dst, src, cr);
13622 %}
13623 %}
13624
13625 // This pattern is automatically generated from aarch64_ad.m4
13626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13627 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13628 %{
13629 match(Set dst (MinI imm src));
13630 ins_cost(INSN_COST * 3);
13631 expand %{
13632 rFlagsReg cr;
13633 compI_reg_imm0(cr, src);
13634 cmovI_reg_imm1_le(dst, src, cr);
13635 %}
13636 %}
13637
13638 // This pattern is automatically generated from aarch64_ad.m4
13639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13640 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13641 %{
13642 match(Set dst (MinI src imm));
13643 ins_cost(INSN_COST * 3);
13644 expand %{
13645 rFlagsReg cr;
13646 compI_reg_imm0(cr, src);
13647 cmovI_reg_immM1_lt(dst, src, cr);
13648 %}
13649 %}
13650
13651 // This pattern is automatically generated from aarch64_ad.m4
13652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13653 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13654 %{
13655 match(Set dst (MinI imm src));
13656 ins_cost(INSN_COST * 3);
13657 expand %{
13658 rFlagsReg cr;
13659 compI_reg_imm0(cr, src);
13660 cmovI_reg_immM1_lt(dst, src, cr);
13661 %}
13662 %}
13663
13664 // This pattern is automatically generated from aarch64_ad.m4
13665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13666 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13667 %{
13668 match(Set dst (MaxI src imm));
13669 ins_cost(INSN_COST * 3);
13670 expand %{
13671 rFlagsReg cr;
13672 compI_reg_imm0(cr, src);
13673 cmovI_reg_imm0_gt(dst, src, cr);
13674 %}
13675 %}
13676
13677 // This pattern is automatically generated from aarch64_ad.m4
13678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13679 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13680 %{
13681 match(Set dst (MaxI imm src));
13682 ins_cost(INSN_COST * 3);
13683 expand %{
13684 rFlagsReg cr;
13685 compI_reg_imm0(cr, src);
13686 cmovI_reg_imm0_gt(dst, src, cr);
13687 %}
13688 %}
13689
13690 // This pattern is automatically generated from aarch64_ad.m4
13691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13692 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13693 %{
13694 match(Set dst (MaxI src imm));
13695 ins_cost(INSN_COST * 3);
13696 expand %{
13697 rFlagsReg cr;
13698 compI_reg_imm0(cr, src);
13699 cmovI_reg_imm1_gt(dst, src, cr);
13700 %}
13701 %}
13702
13703 // This pattern is automatically generated from aarch64_ad.m4
13704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13705 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13706 %{
13707 match(Set dst (MaxI imm src));
13708 ins_cost(INSN_COST * 3);
13709 expand %{
13710 rFlagsReg cr;
13711 compI_reg_imm0(cr, src);
13712 cmovI_reg_imm1_gt(dst, src, cr);
13713 %}
13714 %}
13715
13716 // This pattern is automatically generated from aarch64_ad.m4
13717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13718 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13719 %{
13720 match(Set dst (MaxI src imm));
13721 ins_cost(INSN_COST * 3);
13722 expand %{
13723 rFlagsReg cr;
13724 compI_reg_imm0(cr, src);
13725 cmovI_reg_immM1_ge(dst, src, cr);
13726 %}
13727 %}
13728
13729 // This pattern is automatically generated from aarch64_ad.m4
13730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13731 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13732 %{
13733 match(Set dst (MaxI imm src));
13734 ins_cost(INSN_COST * 3);
13735 expand %{
13736 rFlagsReg cr;
13737 compI_reg_imm0(cr, src);
13738 cmovI_reg_immM1_ge(dst, src, cr);
13739 %}
13740 %}
13741
13742 // This pattern is automatically generated from aarch64_ad.m4
13743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13744 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13745 %{
13746 match(Set dst (ReverseI src));
13747 ins_cost(INSN_COST);
13748 format %{ "rbitw $dst, $src" %}
13749 ins_encode %{
13750 __ rbitw($dst$$Register, $src$$Register);
13751 %}
13752 ins_pipe(ialu_reg);
13753 %}
13754
13755 // This pattern is automatically generated from aarch64_ad.m4
13756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13757 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13758 %{
13759 match(Set dst (ReverseL src));
13760 ins_cost(INSN_COST);
13761 format %{ "rbit $dst, $src" %}
13762 ins_encode %{
13763 __ rbit($dst$$Register, $src$$Register);
13764 %}
13765 ins_pipe(ialu_reg);
13766 %}
13767
13768
13769 // END This section of the file is automatically generated. Do not edit --------------
13770
13771
13772 // ============================================================================
13773 // Floating Point Arithmetic Instructions
13774
13775 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13776 match(Set dst (AddHF src1 src2));
13777 format %{ "faddh $dst, $src1, $src2" %}
13778 ins_encode %{
13779 __ faddh($dst$$FloatRegister,
13780 $src1$$FloatRegister,
13781 $src2$$FloatRegister);
13782 %}
13783 ins_pipe(fp_dop_reg_reg_s);
13784 %}
13785
13786 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13787 match(Set dst (AddF src1 src2));
13788
13789 ins_cost(INSN_COST * 5);
13790 format %{ "fadds $dst, $src1, $src2" %}
13791
13792 ins_encode %{
13793 __ fadds(as_FloatRegister($dst$$reg),
13794 as_FloatRegister($src1$$reg),
13795 as_FloatRegister($src2$$reg));
13796 %}
13797
13798 ins_pipe(fp_dop_reg_reg_s);
13799 %}
13800
13801 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13802 match(Set dst (AddD src1 src2));
13803
13804 ins_cost(INSN_COST * 5);
13805 format %{ "faddd $dst, $src1, $src2" %}
13806
13807 ins_encode %{
13808 __ faddd(as_FloatRegister($dst$$reg),
13809 as_FloatRegister($src1$$reg),
13810 as_FloatRegister($src2$$reg));
13811 %}
13812
13813 ins_pipe(fp_dop_reg_reg_d);
13814 %}
13815
13816 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13817 match(Set dst (SubHF src1 src2));
13818 format %{ "fsubh $dst, $src1, $src2" %}
13819 ins_encode %{
13820 __ fsubh($dst$$FloatRegister,
13821 $src1$$FloatRegister,
13822 $src2$$FloatRegister);
13823 %}
13824 ins_pipe(fp_dop_reg_reg_s);
13825 %}
13826
13827 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13828 match(Set dst (SubF src1 src2));
13829
13830 ins_cost(INSN_COST * 5);
13831 format %{ "fsubs $dst, $src1, $src2" %}
13832
13833 ins_encode %{
13834 __ fsubs(as_FloatRegister($dst$$reg),
13835 as_FloatRegister($src1$$reg),
13836 as_FloatRegister($src2$$reg));
13837 %}
13838
13839 ins_pipe(fp_dop_reg_reg_s);
13840 %}
13841
13842 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13843 match(Set dst (SubD src1 src2));
13844
13845 ins_cost(INSN_COST * 5);
13846 format %{ "fsubd $dst, $src1, $src2" %}
13847
13848 ins_encode %{
13849 __ fsubd(as_FloatRegister($dst$$reg),
13850 as_FloatRegister($src1$$reg),
13851 as_FloatRegister($src2$$reg));
13852 %}
13853
13854 ins_pipe(fp_dop_reg_reg_d);
13855 %}
13856
13857 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13858 match(Set dst (MulHF src1 src2));
13859 format %{ "fmulh $dst, $src1, $src2" %}
13860 ins_encode %{
13861 __ fmulh($dst$$FloatRegister,
13862 $src1$$FloatRegister,
13863 $src2$$FloatRegister);
13864 %}
13865 ins_pipe(fp_dop_reg_reg_s);
13866 %}
13867
13868 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13869 match(Set dst (MulF src1 src2));
13870
13871 ins_cost(INSN_COST * 6);
13872 format %{ "fmuls $dst, $src1, $src2" %}
13873
13874 ins_encode %{
13875 __ fmuls(as_FloatRegister($dst$$reg),
13876 as_FloatRegister($src1$$reg),
13877 as_FloatRegister($src2$$reg));
13878 %}
13879
13880 ins_pipe(fp_dop_reg_reg_s);
13881 %}
13882
13883 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13884 match(Set dst (MulD src1 src2));
13885
13886 ins_cost(INSN_COST * 6);
13887 format %{ "fmuld $dst, $src1, $src2" %}
13888
13889 ins_encode %{
13890 __ fmuld(as_FloatRegister($dst$$reg),
13891 as_FloatRegister($src1$$reg),
13892 as_FloatRegister($src2$$reg));
13893 %}
13894
13895 ins_pipe(fp_dop_reg_reg_d);
13896 %}
13897
13898 // src1 * src2 + src3 (half-precision float)
13899 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13900 match(Set dst (FmaHF src3 (Binary src1 src2)));
13901 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13902 ins_encode %{
13903 assert(UseFMA, "Needs FMA instructions support.");
13904 __ fmaddh($dst$$FloatRegister,
13905 $src1$$FloatRegister,
13906 $src2$$FloatRegister,
13907 $src3$$FloatRegister);
13908 %}
13909 ins_pipe(pipe_class_default);
13910 %}
13911
13912 // src1 * src2 + src3
13913 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13914 match(Set dst (FmaF src3 (Binary src1 src2)));
13915
13916 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13917
13918 ins_encode %{
13919 assert(UseFMA, "Needs FMA instructions support.");
13920 __ fmadds(as_FloatRegister($dst$$reg),
13921 as_FloatRegister($src1$$reg),
13922 as_FloatRegister($src2$$reg),
13923 as_FloatRegister($src3$$reg));
13924 %}
13925
13926 ins_pipe(pipe_class_default);
13927 %}
13928
13929 // src1 * src2 + src3
13930 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13931 match(Set dst (FmaD src3 (Binary src1 src2)));
13932
13933 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13934
13935 ins_encode %{
13936 assert(UseFMA, "Needs FMA instructions support.");
13937 __ fmaddd(as_FloatRegister($dst$$reg),
13938 as_FloatRegister($src1$$reg),
13939 as_FloatRegister($src2$$reg),
13940 as_FloatRegister($src3$$reg));
13941 %}
13942
13943 ins_pipe(pipe_class_default);
13944 %}
13945
13946 // src1 * (-src2) + src3
13947 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13948 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13949 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13950
13951 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13952
13953 ins_encode %{
13954 assert(UseFMA, "Needs FMA instructions support.");
13955 __ fmsubs(as_FloatRegister($dst$$reg),
13956 as_FloatRegister($src1$$reg),
13957 as_FloatRegister($src2$$reg),
13958 as_FloatRegister($src3$$reg));
13959 %}
13960
13961 ins_pipe(pipe_class_default);
13962 %}
13963
13964 // src1 * (-src2) + src3
13965 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13966 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13967 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13968
13969 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13970
13971 ins_encode %{
13972 assert(UseFMA, "Needs FMA instructions support.");
13973 __ fmsubd(as_FloatRegister($dst$$reg),
13974 as_FloatRegister($src1$$reg),
13975 as_FloatRegister($src2$$reg),
13976 as_FloatRegister($src3$$reg));
13977 %}
13978
13979 ins_pipe(pipe_class_default);
13980 %}
13981
13982 // src1 * (-src2) - src3
13983 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13984 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13985 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13986
13987 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13988
13989 ins_encode %{
13990 assert(UseFMA, "Needs FMA instructions support.");
13991 __ fnmadds(as_FloatRegister($dst$$reg),
13992 as_FloatRegister($src1$$reg),
13993 as_FloatRegister($src2$$reg),
13994 as_FloatRegister($src3$$reg));
13995 %}
13996
13997 ins_pipe(pipe_class_default);
13998 %}
13999
14000 // src1 * (-src2) - src3
14001 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14002 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14003 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14004
14005 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14006
14007 ins_encode %{
14008 assert(UseFMA, "Needs FMA instructions support.");
14009 __ fnmaddd(as_FloatRegister($dst$$reg),
14010 as_FloatRegister($src1$$reg),
14011 as_FloatRegister($src2$$reg),
14012 as_FloatRegister($src3$$reg));
14013 %}
14014
14015 ins_pipe(pipe_class_default);
14016 %}
14017
14018 // src1 * src2 - src3
14019 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14020 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14021
14022 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14023
14024 ins_encode %{
14025 assert(UseFMA, "Needs FMA instructions support.");
14026 __ fnmsubs(as_FloatRegister($dst$$reg),
14027 as_FloatRegister($src1$$reg),
14028 as_FloatRegister($src2$$reg),
14029 as_FloatRegister($src3$$reg));
14030 %}
14031
14032 ins_pipe(pipe_class_default);
14033 %}
14034
14035 // src1 * src2 - src3
14036 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14037 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14038
14039 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14040
14041 ins_encode %{
14042 assert(UseFMA, "Needs FMA instructions support.");
14043 // n.b. insn name should be fnmsubd
14044 __ fnmsub(as_FloatRegister($dst$$reg),
14045 as_FloatRegister($src1$$reg),
14046 as_FloatRegister($src2$$reg),
14047 as_FloatRegister($src3$$reg));
14048 %}
14049
14050 ins_pipe(pipe_class_default);
14051 %}
14052
14053 // Math.max(HH)H (half-precision float)
14054 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14055 match(Set dst (MaxHF src1 src2));
14056 format %{ "fmaxh $dst, $src1, $src2" %}
14057 ins_encode %{
14058 __ fmaxh($dst$$FloatRegister,
14059 $src1$$FloatRegister,
14060 $src2$$FloatRegister);
14061 %}
14062 ins_pipe(fp_dop_reg_reg_s);
14063 %}
14064
14065 // Math.min(HH)H (half-precision float)
14066 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14067 match(Set dst (MinHF src1 src2));
14068 format %{ "fminh $dst, $src1, $src2" %}
14069 ins_encode %{
14070 __ fminh($dst$$FloatRegister,
14071 $src1$$FloatRegister,
14072 $src2$$FloatRegister);
14073 %}
14074 ins_pipe(fp_dop_reg_reg_s);
14075 %}
14076
14077 // Math.max(FF)F
14078 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14079 match(Set dst (MaxF src1 src2));
14080
14081 format %{ "fmaxs $dst, $src1, $src2" %}
14082 ins_encode %{
14083 __ fmaxs(as_FloatRegister($dst$$reg),
14084 as_FloatRegister($src1$$reg),
14085 as_FloatRegister($src2$$reg));
14086 %}
14087
14088 ins_pipe(fp_dop_reg_reg_s);
14089 %}
14090
14091 // Math.min(FF)F
14092 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14093 match(Set dst (MinF src1 src2));
14094
14095 format %{ "fmins $dst, $src1, $src2" %}
14096 ins_encode %{
14097 __ fmins(as_FloatRegister($dst$$reg),
14098 as_FloatRegister($src1$$reg),
14099 as_FloatRegister($src2$$reg));
14100 %}
14101
14102 ins_pipe(fp_dop_reg_reg_s);
14103 %}
14104
14105 // Math.max(DD)D
14106 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14107 match(Set dst (MaxD src1 src2));
14108
14109 format %{ "fmaxd $dst, $src1, $src2" %}
14110 ins_encode %{
14111 __ fmaxd(as_FloatRegister($dst$$reg),
14112 as_FloatRegister($src1$$reg),
14113 as_FloatRegister($src2$$reg));
14114 %}
14115
14116 ins_pipe(fp_dop_reg_reg_d);
14117 %}
14118
14119 // Math.min(DD)D
14120 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14121 match(Set dst (MinD src1 src2));
14122
14123 format %{ "fmind $dst, $src1, $src2" %}
14124 ins_encode %{
14125 __ fmind(as_FloatRegister($dst$$reg),
14126 as_FloatRegister($src1$$reg),
14127 as_FloatRegister($src2$$reg));
14128 %}
14129
14130 ins_pipe(fp_dop_reg_reg_d);
14131 %}
14132
14133 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14134 match(Set dst (DivHF src1 src2));
14135 format %{ "fdivh $dst, $src1, $src2" %}
14136 ins_encode %{
14137 __ fdivh($dst$$FloatRegister,
14138 $src1$$FloatRegister,
14139 $src2$$FloatRegister);
14140 %}
14141 ins_pipe(fp_div_s);
14142 %}
14143
14144 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14145 match(Set dst (DivF src1 src2));
14146
14147 ins_cost(INSN_COST * 18);
14148 format %{ "fdivs $dst, $src1, $src2" %}
14149
14150 ins_encode %{
14151 __ fdivs(as_FloatRegister($dst$$reg),
14152 as_FloatRegister($src1$$reg),
14153 as_FloatRegister($src2$$reg));
14154 %}
14155
14156 ins_pipe(fp_div_s);
14157 %}
14158
14159 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14160 match(Set dst (DivD src1 src2));
14161
14162 ins_cost(INSN_COST * 32);
14163 format %{ "fdivd $dst, $src1, $src2" %}
14164
14165 ins_encode %{
14166 __ fdivd(as_FloatRegister($dst$$reg),
14167 as_FloatRegister($src1$$reg),
14168 as_FloatRegister($src2$$reg));
14169 %}
14170
14171 ins_pipe(fp_div_d);
14172 %}
14173
14174 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14175 match(Set dst (NegF src));
14176
14177 ins_cost(INSN_COST * 3);
14178 format %{ "fneg $dst, $src" %}
14179
14180 ins_encode %{
14181 __ fnegs(as_FloatRegister($dst$$reg),
14182 as_FloatRegister($src$$reg));
14183 %}
14184
14185 ins_pipe(fp_uop_s);
14186 %}
14187
14188 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14189 match(Set dst (NegD src));
14190
14191 ins_cost(INSN_COST * 3);
14192 format %{ "fnegd $dst, $src" %}
14193
14194 ins_encode %{
14195 __ fnegd(as_FloatRegister($dst$$reg),
14196 as_FloatRegister($src$$reg));
14197 %}
14198
14199 ins_pipe(fp_uop_d);
14200 %}
14201
14202 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14203 %{
14204 match(Set dst (AbsI src));
14205
14206 effect(KILL cr);
14207 ins_cost(INSN_COST * 2);
14208 format %{ "cmpw $src, zr\n\t"
14209 "cnegw $dst, $src, Assembler::LT\t# int abs"
14210 %}
14211
14212 ins_encode %{
14213 __ cmpw(as_Register($src$$reg), zr);
14214 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14215 %}
14216 ins_pipe(pipe_class_default);
14217 %}
14218
14219 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14220 %{
14221 match(Set dst (AbsL src));
14222
14223 effect(KILL cr);
14224 ins_cost(INSN_COST * 2);
14225 format %{ "cmp $src, zr\n\t"
14226 "cneg $dst, $src, Assembler::LT\t# long abs"
14227 %}
14228
14229 ins_encode %{
14230 __ cmp(as_Register($src$$reg), zr);
14231 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14232 %}
14233 ins_pipe(pipe_class_default);
14234 %}
14235
14236 instruct absF_reg(vRegF dst, vRegF src) %{
14237 match(Set dst (AbsF src));
14238
14239 ins_cost(INSN_COST * 3);
14240 format %{ "fabss $dst, $src" %}
14241 ins_encode %{
14242 __ fabss(as_FloatRegister($dst$$reg),
14243 as_FloatRegister($src$$reg));
14244 %}
14245
14246 ins_pipe(fp_uop_s);
14247 %}
14248
14249 instruct absD_reg(vRegD dst, vRegD src) %{
14250 match(Set dst (AbsD src));
14251
14252 ins_cost(INSN_COST * 3);
14253 format %{ "fabsd $dst, $src" %}
14254 ins_encode %{
14255 __ fabsd(as_FloatRegister($dst$$reg),
14256 as_FloatRegister($src$$reg));
14257 %}
14258
14259 ins_pipe(fp_uop_d);
14260 %}
14261
14262 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14263 match(Set dst (AbsF (SubF src1 src2)));
14264
14265 ins_cost(INSN_COST * 3);
14266 format %{ "fabds $dst, $src1, $src2" %}
14267 ins_encode %{
14268 __ fabds(as_FloatRegister($dst$$reg),
14269 as_FloatRegister($src1$$reg),
14270 as_FloatRegister($src2$$reg));
14271 %}
14272
14273 ins_pipe(fp_uop_s);
14274 %}
14275
14276 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14277 match(Set dst (AbsD (SubD src1 src2)));
14278
14279 ins_cost(INSN_COST * 3);
14280 format %{ "fabdd $dst, $src1, $src2" %}
14281 ins_encode %{
14282 __ fabdd(as_FloatRegister($dst$$reg),
14283 as_FloatRegister($src1$$reg),
14284 as_FloatRegister($src2$$reg));
14285 %}
14286
14287 ins_pipe(fp_uop_d);
14288 %}
14289
14290 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14291 match(Set dst (SqrtD src));
14292
14293 ins_cost(INSN_COST * 50);
14294 format %{ "fsqrtd $dst, $src" %}
14295 ins_encode %{
14296 __ fsqrtd(as_FloatRegister($dst$$reg),
14297 as_FloatRegister($src$$reg));
14298 %}
14299
14300 ins_pipe(fp_div_s);
14301 %}
14302
14303 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14304 match(Set dst (SqrtF src));
14305
14306 ins_cost(INSN_COST * 50);
14307 format %{ "fsqrts $dst, $src" %}
14308 ins_encode %{
14309 __ fsqrts(as_FloatRegister($dst$$reg),
14310 as_FloatRegister($src$$reg));
14311 %}
14312
14313 ins_pipe(fp_div_d);
14314 %}
14315
14316 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14317 match(Set dst (SqrtHF src));
14318 format %{ "fsqrth $dst, $src" %}
14319 ins_encode %{
14320 __ fsqrth($dst$$FloatRegister,
14321 $src$$FloatRegister);
14322 %}
14323 ins_pipe(fp_div_s);
14324 %}
14325
14326 // Math.rint, floor, ceil
14327 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14328 match(Set dst (RoundDoubleMode src rmode));
14329 format %{ "frint $dst, $src, $rmode" %}
14330 ins_encode %{
14331 switch ($rmode$$constant) {
14332 case RoundDoubleModeNode::rmode_rint:
14333 __ frintnd(as_FloatRegister($dst$$reg),
14334 as_FloatRegister($src$$reg));
14335 break;
14336 case RoundDoubleModeNode::rmode_floor:
14337 __ frintmd(as_FloatRegister($dst$$reg),
14338 as_FloatRegister($src$$reg));
14339 break;
14340 case RoundDoubleModeNode::rmode_ceil:
14341 __ frintpd(as_FloatRegister($dst$$reg),
14342 as_FloatRegister($src$$reg));
14343 break;
14344 }
14345 %}
14346 ins_pipe(fp_uop_d);
14347 %}
14348
14349 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14350 match(Set dst (CopySignD src1 (Binary src2 zero)));
14351 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14352 format %{ "CopySignD $dst $src1 $src2" %}
14353 ins_encode %{
14354 FloatRegister dst = as_FloatRegister($dst$$reg),
14355 src1 = as_FloatRegister($src1$$reg),
14356 src2 = as_FloatRegister($src2$$reg),
14357 zero = as_FloatRegister($zero$$reg);
14358 __ fnegd(dst, zero);
14359 __ bsl(dst, __ T8B, src2, src1);
14360 %}
14361 ins_pipe(fp_uop_d);
14362 %}
14363
14364 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14365 match(Set dst (CopySignF src1 src2));
14366 effect(TEMP_DEF dst, USE src1, USE src2);
14367 format %{ "CopySignF $dst $src1 $src2" %}
14368 ins_encode %{
14369 FloatRegister dst = as_FloatRegister($dst$$reg),
14370 src1 = as_FloatRegister($src1$$reg),
14371 src2 = as_FloatRegister($src2$$reg);
14372 __ movi(dst, __ T2S, 0x80, 24);
14373 __ bsl(dst, __ T8B, src2, src1);
14374 %}
14375 ins_pipe(fp_uop_d);
14376 %}
14377
14378 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14379 match(Set dst (SignumD src (Binary zero one)));
14380 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14381 format %{ "signumD $dst, $src" %}
14382 ins_encode %{
14383 FloatRegister src = as_FloatRegister($src$$reg),
14384 dst = as_FloatRegister($dst$$reg),
14385 zero = as_FloatRegister($zero$$reg),
14386 one = as_FloatRegister($one$$reg);
14387 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14388 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14389 // Bit selection instruction gets bit from "one" for each enabled bit in
14390 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14391 // NaN the whole "src" will be copied because "dst" is zero. For all other
14392 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14393 // from "src", and all other bits are copied from 1.0.
14394 __ bsl(dst, __ T8B, one, src);
14395 %}
14396 ins_pipe(fp_uop_d);
14397 %}
14398
14399 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14400 match(Set dst (SignumF src (Binary zero one)));
14401 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14402 format %{ "signumF $dst, $src" %}
14403 ins_encode %{
14404 FloatRegister src = as_FloatRegister($src$$reg),
14405 dst = as_FloatRegister($dst$$reg),
14406 zero = as_FloatRegister($zero$$reg),
14407 one = as_FloatRegister($one$$reg);
14408 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14409 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14410 // Bit selection instruction gets bit from "one" for each enabled bit in
14411 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14412 // NaN the whole "src" will be copied because "dst" is zero. For all other
14413 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14414 // from "src", and all other bits are copied from 1.0.
14415 __ bsl(dst, __ T8B, one, src);
14416 %}
14417 ins_pipe(fp_uop_d);
14418 %}
14419
14420 instruct onspinwait() %{
14421 match(OnSpinWait);
14422 ins_cost(INSN_COST);
14423
14424 format %{ "onspinwait" %}
14425
14426 ins_encode %{
14427 __ spin_wait();
14428 %}
14429 ins_pipe(pipe_class_empty);
14430 %}
14431
14432 // ============================================================================
14433 // Logical Instructions
14434
14435 // Integer Logical Instructions
14436
14437 // And Instructions
14438
14439
14440 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14441 match(Set dst (AndI src1 src2));
14442
14443 format %{ "andw $dst, $src1, $src2\t# int" %}
14444
14445 ins_cost(INSN_COST);
14446 ins_encode %{
14447 __ andw(as_Register($dst$$reg),
14448 as_Register($src1$$reg),
14449 as_Register($src2$$reg));
14450 %}
14451
14452 ins_pipe(ialu_reg_reg);
14453 %}
14454
14455 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14456 match(Set dst (AndI src1 src2));
14457
14458 format %{ "andsw $dst, $src1, $src2\t# int" %}
14459
14460 ins_cost(INSN_COST);
14461 ins_encode %{
14462 __ andw(as_Register($dst$$reg),
14463 as_Register($src1$$reg),
14464 (uint64_t)($src2$$constant));
14465 %}
14466
14467 ins_pipe(ialu_reg_imm);
14468 %}
14469
14470 // Or Instructions
14471
14472 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14473 match(Set dst (OrI src1 src2));
14474
14475 format %{ "orrw $dst, $src1, $src2\t# int" %}
14476
14477 ins_cost(INSN_COST);
14478 ins_encode %{
14479 __ orrw(as_Register($dst$$reg),
14480 as_Register($src1$$reg),
14481 as_Register($src2$$reg));
14482 %}
14483
14484 ins_pipe(ialu_reg_reg);
14485 %}
14486
14487 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14488 match(Set dst (OrI src1 src2));
14489
14490 format %{ "orrw $dst, $src1, $src2\t# int" %}
14491
14492 ins_cost(INSN_COST);
14493 ins_encode %{
14494 __ orrw(as_Register($dst$$reg),
14495 as_Register($src1$$reg),
14496 (uint64_t)($src2$$constant));
14497 %}
14498
14499 ins_pipe(ialu_reg_imm);
14500 %}
14501
14502 // Xor Instructions
14503
14504 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14505 match(Set dst (XorI src1 src2));
14506
14507 format %{ "eorw $dst, $src1, $src2\t# int" %}
14508
14509 ins_cost(INSN_COST);
14510 ins_encode %{
14511 __ eorw(as_Register($dst$$reg),
14512 as_Register($src1$$reg),
14513 as_Register($src2$$reg));
14514 %}
14515
14516 ins_pipe(ialu_reg_reg);
14517 %}
14518
14519 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14520 match(Set dst (XorI src1 src2));
14521
14522 format %{ "eorw $dst, $src1, $src2\t# int" %}
14523
14524 ins_cost(INSN_COST);
14525 ins_encode %{
14526 __ eorw(as_Register($dst$$reg),
14527 as_Register($src1$$reg),
14528 (uint64_t)($src2$$constant));
14529 %}
14530
14531 ins_pipe(ialu_reg_imm);
14532 %}
14533
14534 // Long Logical Instructions
14535 // TODO
14536
14537 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14538 match(Set dst (AndL src1 src2));
14539
14540 format %{ "and $dst, $src1, $src2\t# int" %}
14541
14542 ins_cost(INSN_COST);
14543 ins_encode %{
14544 __ andr(as_Register($dst$$reg),
14545 as_Register($src1$$reg),
14546 as_Register($src2$$reg));
14547 %}
14548
14549 ins_pipe(ialu_reg_reg);
14550 %}
14551
14552 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14553 match(Set dst (AndL src1 src2));
14554
14555 format %{ "and $dst, $src1, $src2\t# int" %}
14556
14557 ins_cost(INSN_COST);
14558 ins_encode %{
14559 __ andr(as_Register($dst$$reg),
14560 as_Register($src1$$reg),
14561 (uint64_t)($src2$$constant));
14562 %}
14563
14564 ins_pipe(ialu_reg_imm);
14565 %}
14566
14567 // Or Instructions
14568
14569 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14570 match(Set dst (OrL src1 src2));
14571
14572 format %{ "orr $dst, $src1, $src2\t# int" %}
14573
14574 ins_cost(INSN_COST);
14575 ins_encode %{
14576 __ orr(as_Register($dst$$reg),
14577 as_Register($src1$$reg),
14578 as_Register($src2$$reg));
14579 %}
14580
14581 ins_pipe(ialu_reg_reg);
14582 %}
14583
14584 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14585 match(Set dst (OrL src1 src2));
14586
14587 format %{ "orr $dst, $src1, $src2\t# int" %}
14588
14589 ins_cost(INSN_COST);
14590 ins_encode %{
14591 __ orr(as_Register($dst$$reg),
14592 as_Register($src1$$reg),
14593 (uint64_t)($src2$$constant));
14594 %}
14595
14596 ins_pipe(ialu_reg_imm);
14597 %}
14598
14599 // Xor Instructions
14600
14601 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14602 match(Set dst (XorL src1 src2));
14603
14604 format %{ "eor $dst, $src1, $src2\t# int" %}
14605
14606 ins_cost(INSN_COST);
14607 ins_encode %{
14608 __ eor(as_Register($dst$$reg),
14609 as_Register($src1$$reg),
14610 as_Register($src2$$reg));
14611 %}
14612
14613 ins_pipe(ialu_reg_reg);
14614 %}
14615
14616 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14617 match(Set dst (XorL src1 src2));
14618
14619 ins_cost(INSN_COST);
14620 format %{ "eor $dst, $src1, $src2\t# int" %}
14621
14622 ins_encode %{
14623 __ eor(as_Register($dst$$reg),
14624 as_Register($src1$$reg),
14625 (uint64_t)($src2$$constant));
14626 %}
14627
14628 ins_pipe(ialu_reg_imm);
14629 %}
14630
14631 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14632 %{
14633 match(Set dst (ConvI2L src));
14634
14635 ins_cost(INSN_COST);
14636 format %{ "sxtw $dst, $src\t# i2l" %}
14637 ins_encode %{
14638 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14639 %}
14640 ins_pipe(ialu_reg_shift);
14641 %}
14642
14643 // this pattern occurs in bigmath arithmetic
14644 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14645 %{
14646 match(Set dst (AndL (ConvI2L src) mask));
14647
14648 ins_cost(INSN_COST);
14649 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14650 ins_encode %{
14651 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14652 %}
14653
14654 ins_pipe(ialu_reg_shift);
14655 %}
14656
14657 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14658 match(Set dst (ConvL2I src));
14659
14660 ins_cost(INSN_COST);
14661 format %{ "movw $dst, $src \t// l2i" %}
14662
14663 ins_encode %{
14664 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14665 %}
14666
14667 ins_pipe(ialu_reg);
14668 %}
14669
14670 instruct convD2F_reg(vRegF dst, vRegD src) %{
14671 match(Set dst (ConvD2F src));
14672
14673 ins_cost(INSN_COST * 5);
14674 format %{ "fcvtd $dst, $src \t// d2f" %}
14675
14676 ins_encode %{
14677 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14678 %}
14679
14680 ins_pipe(fp_d2f);
14681 %}
14682
14683 instruct convF2D_reg(vRegD dst, vRegF src) %{
14684 match(Set dst (ConvF2D src));
14685
14686 ins_cost(INSN_COST * 5);
14687 format %{ "fcvts $dst, $src \t// f2d" %}
14688
14689 ins_encode %{
14690 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14691 %}
14692
14693 ins_pipe(fp_f2d);
14694 %}
14695
14696 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14697 match(Set dst (ConvF2I src));
14698
14699 ins_cost(INSN_COST * 5);
14700 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14701
14702 ins_encode %{
14703 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14704 %}
14705
14706 ins_pipe(fp_f2i);
14707 %}
14708
14709 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14710 match(Set dst (ConvF2L src));
14711
14712 ins_cost(INSN_COST * 5);
14713 format %{ "fcvtzs $dst, $src \t// f2l" %}
14714
14715 ins_encode %{
14716 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14717 %}
14718
14719 ins_pipe(fp_f2l);
14720 %}
14721
14722 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14723 match(Set dst (ConvF2HF src));
14724 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14725 "smov $dst, $tmp\t# move result from $tmp to $dst"
14726 %}
14727 effect(TEMP tmp);
14728 ins_encode %{
14729 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14730 %}
14731 ins_pipe(pipe_slow);
14732 %}
14733
14734 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14735 match(Set dst (ConvHF2F src));
14736 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14737 "fcvt $dst, $tmp\t# convert half to single precision"
14738 %}
14739 effect(TEMP tmp);
14740 ins_encode %{
14741 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14742 %}
14743 ins_pipe(pipe_slow);
14744 %}
14745
14746 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14747 match(Set dst (ConvI2F src));
14748
14749 ins_cost(INSN_COST * 5);
14750 format %{ "scvtfws $dst, $src \t// i2f" %}
14751
14752 ins_encode %{
14753 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14754 %}
14755
14756 ins_pipe(fp_i2f);
14757 %}
14758
14759 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14760 match(Set dst (ConvL2F src));
14761
14762 ins_cost(INSN_COST * 5);
14763 format %{ "scvtfs $dst, $src \t// l2f" %}
14764
14765 ins_encode %{
14766 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14767 %}
14768
14769 ins_pipe(fp_l2f);
14770 %}
14771
14772 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14773 match(Set dst (ConvD2I src));
14774
14775 ins_cost(INSN_COST * 5);
14776 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14777
14778 ins_encode %{
14779 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14780 %}
14781
14782 ins_pipe(fp_d2i);
14783 %}
14784
14785 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14786 match(Set dst (ConvD2L src));
14787
14788 ins_cost(INSN_COST * 5);
14789 format %{ "fcvtzd $dst, $src \t// d2l" %}
14790
14791 ins_encode %{
14792 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14793 %}
14794
14795 ins_pipe(fp_d2l);
14796 %}
14797
14798 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14799 match(Set dst (ConvI2D src));
14800
14801 ins_cost(INSN_COST * 5);
14802 format %{ "scvtfwd $dst, $src \t// i2d" %}
14803
14804 ins_encode %{
14805 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14806 %}
14807
14808 ins_pipe(fp_i2d);
14809 %}
14810
14811 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14812 match(Set dst (ConvL2D src));
14813
14814 ins_cost(INSN_COST * 5);
14815 format %{ "scvtfd $dst, $src \t// l2d" %}
14816
14817 ins_encode %{
14818 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14819 %}
14820
14821 ins_pipe(fp_l2d);
14822 %}
14823
14824 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14825 %{
14826 match(Set dst (RoundD src));
14827 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14828 format %{ "java_round_double $dst,$src"%}
14829 ins_encode %{
14830 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14831 as_FloatRegister($ftmp$$reg));
14832 %}
14833 ins_pipe(pipe_slow);
14834 %}
14835
14836 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14837 %{
14838 match(Set dst (RoundF src));
14839 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14840 format %{ "java_round_float $dst,$src"%}
14841 ins_encode %{
14842 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14843 as_FloatRegister($ftmp$$reg));
14844 %}
14845 ins_pipe(pipe_slow);
14846 %}
14847
14848 // stack <-> reg and reg <-> reg shuffles with no conversion
14849
14850 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14851
14852 match(Set dst (MoveF2I src));
14853
14854 effect(DEF dst, USE src);
14855
14856 ins_cost(4 * INSN_COST);
14857
14858 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14859
14860 ins_encode %{
14861 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14862 %}
14863
14864 ins_pipe(iload_reg_reg);
14865
14866 %}
14867
14868 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14869
14870 match(Set dst (MoveI2F src));
14871
14872 effect(DEF dst, USE src);
14873
14874 ins_cost(4 * INSN_COST);
14875
14876 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14877
14878 ins_encode %{
14879 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14880 %}
14881
14882 ins_pipe(pipe_class_memory);
14883
14884 %}
14885
14886 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14887
14888 match(Set dst (MoveD2L src));
14889
14890 effect(DEF dst, USE src);
14891
14892 ins_cost(4 * INSN_COST);
14893
14894 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14895
14896 ins_encode %{
14897 __ ldr($dst$$Register, Address(sp, $src$$disp));
14898 %}
14899
14900 ins_pipe(iload_reg_reg);
14901
14902 %}
14903
14904 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14905
14906 match(Set dst (MoveL2D src));
14907
14908 effect(DEF dst, USE src);
14909
14910 ins_cost(4 * INSN_COST);
14911
14912 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14913
14914 ins_encode %{
14915 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14916 %}
14917
14918 ins_pipe(pipe_class_memory);
14919
14920 %}
14921
14922 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14923
14924 match(Set dst (MoveF2I src));
14925
14926 effect(DEF dst, USE src);
14927
14928 ins_cost(INSN_COST);
14929
14930 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14931
14932 ins_encode %{
14933 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14934 %}
14935
14936 ins_pipe(pipe_class_memory);
14937
14938 %}
14939
14940 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14941
14942 match(Set dst (MoveI2F src));
14943
14944 effect(DEF dst, USE src);
14945
14946 ins_cost(INSN_COST);
14947
14948 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14949
14950 ins_encode %{
14951 __ strw($src$$Register, Address(sp, $dst$$disp));
14952 %}
14953
14954 ins_pipe(istore_reg_reg);
14955
14956 %}
14957
14958 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14959
14960 match(Set dst (MoveD2L src));
14961
14962 effect(DEF dst, USE src);
14963
14964 ins_cost(INSN_COST);
14965
14966 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14967
14968 ins_encode %{
14969 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14970 %}
14971
14972 ins_pipe(pipe_class_memory);
14973
14974 %}
14975
14976 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14977
14978 match(Set dst (MoveL2D src));
14979
14980 effect(DEF dst, USE src);
14981
14982 ins_cost(INSN_COST);
14983
14984 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14985
14986 ins_encode %{
14987 __ str($src$$Register, Address(sp, $dst$$disp));
14988 %}
14989
14990 ins_pipe(istore_reg_reg);
14991
14992 %}
14993
14994 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14995
14996 match(Set dst (MoveF2I src));
14997
14998 effect(DEF dst, USE src);
14999
15000 ins_cost(INSN_COST);
15001
15002 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15003
15004 ins_encode %{
15005 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15006 %}
15007
15008 ins_pipe(fp_f2i);
15009
15010 %}
15011
15012 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15013
15014 match(Set dst (MoveI2F src));
15015
15016 effect(DEF dst, USE src);
15017
15018 ins_cost(INSN_COST);
15019
15020 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15021
15022 ins_encode %{
15023 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15024 %}
15025
15026 ins_pipe(fp_i2f);
15027
15028 %}
15029
15030 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15031
15032 match(Set dst (MoveD2L src));
15033
15034 effect(DEF dst, USE src);
15035
15036 ins_cost(INSN_COST);
15037
15038 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15039
15040 ins_encode %{
15041 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15042 %}
15043
15044 ins_pipe(fp_d2l);
15045
15046 %}
15047
15048 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15049
15050 match(Set dst (MoveL2D src));
15051
15052 effect(DEF dst, USE src);
15053
15054 ins_cost(INSN_COST);
15055
15056 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15057
15058 ins_encode %{
15059 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15060 %}
15061
15062 ins_pipe(fp_l2d);
15063
15064 %}
15065
15066 // ============================================================================
15067 // clearing of an array
15068
15069 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15070 %{
15071 match(Set dummy (ClearArray cnt base));
15072 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15073
15074 ins_cost(4 * INSN_COST);
15075 format %{ "ClearArray $cnt, $base" %}
15076
15077 ins_encode %{
15078 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15079 if (tpc == nullptr) {
15080 ciEnv::current()->record_failure("CodeCache is full");
15081 return;
15082 }
15083 %}
15084
15085 ins_pipe(pipe_class_memory);
15086 %}
15087
15088 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15089 %{
15090 predicate((uint64_t)n->in(2)->get_long()
15091 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15092 match(Set dummy (ClearArray cnt base));
15093 effect(TEMP temp, USE_KILL base, KILL cr);
15094
15095 ins_cost(4 * INSN_COST);
15096 format %{ "ClearArray $cnt, $base" %}
15097
15098 ins_encode %{
15099 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15100 if (tpc == nullptr) {
15101 ciEnv::current()->record_failure("CodeCache is full");
15102 return;
15103 }
15104 %}
15105
15106 ins_pipe(pipe_class_memory);
15107 %}
15108
15109 // ============================================================================
15110 // Overflow Math Instructions
15111
15112 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15113 %{
15114 match(Set cr (OverflowAddI op1 op2));
15115
15116 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15117 ins_cost(INSN_COST);
15118 ins_encode %{
15119 __ cmnw($op1$$Register, $op2$$Register);
15120 %}
15121
15122 ins_pipe(icmp_reg_reg);
15123 %}
15124
15125 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15126 %{
15127 match(Set cr (OverflowAddI op1 op2));
15128
15129 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15130 ins_cost(INSN_COST);
15131 ins_encode %{
15132 __ cmnw($op1$$Register, $op2$$constant);
15133 %}
15134
15135 ins_pipe(icmp_reg_imm);
15136 %}
15137
15138 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15139 %{
15140 match(Set cr (OverflowAddL op1 op2));
15141
15142 format %{ "cmn $op1, $op2\t# overflow check long" %}
15143 ins_cost(INSN_COST);
15144 ins_encode %{
15145 __ cmn($op1$$Register, $op2$$Register);
15146 %}
15147
15148 ins_pipe(icmp_reg_reg);
15149 %}
15150
15151 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15152 %{
15153 match(Set cr (OverflowAddL op1 op2));
15154
15155 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15156 ins_cost(INSN_COST);
15157 ins_encode %{
15158 __ adds(zr, $op1$$Register, $op2$$constant);
15159 %}
15160
15161 ins_pipe(icmp_reg_imm);
15162 %}
15163
15164 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15165 %{
15166 match(Set cr (OverflowSubI op1 op2));
15167
15168 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15169 ins_cost(INSN_COST);
15170 ins_encode %{
15171 __ cmpw($op1$$Register, $op2$$Register);
15172 %}
15173
15174 ins_pipe(icmp_reg_reg);
15175 %}
15176
15177 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15178 %{
15179 match(Set cr (OverflowSubI op1 op2));
15180
15181 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15182 ins_cost(INSN_COST);
15183 ins_encode %{
15184 __ cmpw($op1$$Register, $op2$$constant);
15185 %}
15186
15187 ins_pipe(icmp_reg_imm);
15188 %}
15189
15190 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15191 %{
15192 match(Set cr (OverflowSubL op1 op2));
15193
15194 format %{ "cmp $op1, $op2\t# overflow check long" %}
15195 ins_cost(INSN_COST);
15196 ins_encode %{
15197 __ cmp($op1$$Register, $op2$$Register);
15198 %}
15199
15200 ins_pipe(icmp_reg_reg);
15201 %}
15202
15203 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15204 %{
15205 match(Set cr (OverflowSubL op1 op2));
15206
15207 format %{ "cmp $op1, $op2\t# overflow check long" %}
15208 ins_cost(INSN_COST);
15209 ins_encode %{
15210 __ subs(zr, $op1$$Register, $op2$$constant);
15211 %}
15212
15213 ins_pipe(icmp_reg_imm);
15214 %}
15215
15216 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15217 %{
15218 match(Set cr (OverflowSubI zero op1));
15219
15220 format %{ "cmpw zr, $op1\t# overflow check int" %}
15221 ins_cost(INSN_COST);
15222 ins_encode %{
15223 __ cmpw(zr, $op1$$Register);
15224 %}
15225
15226 ins_pipe(icmp_reg_imm);
15227 %}
15228
15229 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15230 %{
15231 match(Set cr (OverflowSubL zero op1));
15232
15233 format %{ "cmp zr, $op1\t# overflow check long" %}
15234 ins_cost(INSN_COST);
15235 ins_encode %{
15236 __ cmp(zr, $op1$$Register);
15237 %}
15238
15239 ins_pipe(icmp_reg_imm);
15240 %}
15241
15242 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15243 %{
15244 match(Set cr (OverflowMulI op1 op2));
15245
15246 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15247 "cmp rscratch1, rscratch1, sxtw\n\t"
15248 "movw rscratch1, #0x80000000\n\t"
15249 "cselw rscratch1, rscratch1, zr, NE\n\t"
15250 "cmpw rscratch1, #1" %}
15251 ins_cost(5 * INSN_COST);
15252 ins_encode %{
15253 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15254 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15255 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15256 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15257 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15258 %}
15259
15260 ins_pipe(pipe_slow);
15261 %}
15262
15263 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15264 %{
15265 match(If cmp (OverflowMulI op1 op2));
15266 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15267 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15268 effect(USE labl, KILL cr);
15269
15270 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15271 "cmp rscratch1, rscratch1, sxtw\n\t"
15272 "b$cmp $labl" %}
15273 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15274 ins_encode %{
15275 Label* L = $labl$$label;
15276 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15277 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15278 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15279 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15280 %}
15281
15282 ins_pipe(pipe_serial);
15283 %}
15284
15285 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15286 %{
15287 match(Set cr (OverflowMulL op1 op2));
15288
15289 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15290 "smulh rscratch2, $op1, $op2\n\t"
15291 "cmp rscratch2, rscratch1, ASR #63\n\t"
15292 "movw rscratch1, #0x80000000\n\t"
15293 "cselw rscratch1, rscratch1, zr, NE\n\t"
15294 "cmpw rscratch1, #1" %}
15295 ins_cost(6 * INSN_COST);
15296 ins_encode %{
15297 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15298 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15299 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15300 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15301 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15302 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15303 %}
15304
15305 ins_pipe(pipe_slow);
15306 %}
15307
15308 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15309 %{
15310 match(If cmp (OverflowMulL op1 op2));
15311 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15312 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15313 effect(USE labl, KILL cr);
15314
15315 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15316 "smulh rscratch2, $op1, $op2\n\t"
15317 "cmp rscratch2, rscratch1, ASR #63\n\t"
15318 "b$cmp $labl" %}
15319 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15320 ins_encode %{
15321 Label* L = $labl$$label;
15322 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15323 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15324 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15325 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15326 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15327 %}
15328
15329 ins_pipe(pipe_serial);
15330 %}
15331
15332 // ============================================================================
15333 // Compare Instructions
15334
15335 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15336 %{
15337 match(Set cr (CmpI op1 op2));
15338
15339 effect(DEF cr, USE op1, USE op2);
15340
15341 ins_cost(INSN_COST);
15342 format %{ "cmpw $op1, $op2" %}
15343
15344 ins_encode(aarch64_enc_cmpw(op1, op2));
15345
15346 ins_pipe(icmp_reg_reg);
15347 %}
15348
15349 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15350 %{
15351 match(Set cr (CmpI op1 zero));
15352
15353 effect(DEF cr, USE op1);
15354
15355 ins_cost(INSN_COST);
15356 format %{ "cmpw $op1, 0" %}
15357
15358 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15359
15360 ins_pipe(icmp_reg_imm);
15361 %}
15362
15363 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15364 %{
15365 match(Set cr (CmpI op1 op2));
15366
15367 effect(DEF cr, USE op1);
15368
15369 ins_cost(INSN_COST);
15370 format %{ "cmpw $op1, $op2" %}
15371
15372 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15373
15374 ins_pipe(icmp_reg_imm);
15375 %}
15376
15377 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15378 %{
15379 match(Set cr (CmpI op1 op2));
15380
15381 effect(DEF cr, USE op1);
15382
15383 ins_cost(INSN_COST * 2);
15384 format %{ "cmpw $op1, $op2" %}
15385
15386 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15387
15388 ins_pipe(icmp_reg_imm);
15389 %}
15390
15391 // Unsigned compare Instructions; really, same as signed compare
15392 // except it should only be used to feed an If or a CMovI which takes a
15393 // cmpOpU.
15394
15395 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15396 %{
15397 match(Set cr (CmpU op1 op2));
15398
15399 effect(DEF cr, USE op1, USE op2);
15400
15401 ins_cost(INSN_COST);
15402 format %{ "cmpw $op1, $op2\t# unsigned" %}
15403
15404 ins_encode(aarch64_enc_cmpw(op1, op2));
15405
15406 ins_pipe(icmp_reg_reg);
15407 %}
15408
15409 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15410 %{
15411 match(Set cr (CmpU op1 zero));
15412
15413 effect(DEF cr, USE op1);
15414
15415 ins_cost(INSN_COST);
15416 format %{ "cmpw $op1, #0\t# unsigned" %}
15417
15418 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15419
15420 ins_pipe(icmp_reg_imm);
15421 %}
15422
15423 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15424 %{
15425 match(Set cr (CmpU op1 op2));
15426
15427 effect(DEF cr, USE op1);
15428
15429 ins_cost(INSN_COST);
15430 format %{ "cmpw $op1, $op2\t# unsigned" %}
15431
15432 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15433
15434 ins_pipe(icmp_reg_imm);
15435 %}
15436
15437 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15438 %{
15439 match(Set cr (CmpU op1 op2));
15440
15441 effect(DEF cr, USE op1);
15442
15443 ins_cost(INSN_COST * 2);
15444 format %{ "cmpw $op1, $op2\t# unsigned" %}
15445
15446 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15447
15448 ins_pipe(icmp_reg_imm);
15449 %}
15450
15451 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15452 %{
15453 match(Set cr (CmpL op1 op2));
15454
15455 effect(DEF cr, USE op1, USE op2);
15456
15457 ins_cost(INSN_COST);
15458 format %{ "cmp $op1, $op2" %}
15459
15460 ins_encode(aarch64_enc_cmp(op1, op2));
15461
15462 ins_pipe(icmp_reg_reg);
15463 %}
15464
15465 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15466 %{
15467 match(Set cr (CmpL op1 zero));
15468
15469 effect(DEF cr, USE op1);
15470
15471 ins_cost(INSN_COST);
15472 format %{ "tst $op1" %}
15473
15474 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15475
15476 ins_pipe(icmp_reg_imm);
15477 %}
15478
15479 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15480 %{
15481 match(Set cr (CmpL op1 op2));
15482
15483 effect(DEF cr, USE op1);
15484
15485 ins_cost(INSN_COST);
15486 format %{ "cmp $op1, $op2" %}
15487
15488 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15489
15490 ins_pipe(icmp_reg_imm);
15491 %}
15492
15493 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15494 %{
15495 match(Set cr (CmpL op1 op2));
15496
15497 effect(DEF cr, USE op1);
15498
15499 ins_cost(INSN_COST * 2);
15500 format %{ "cmp $op1, $op2" %}
15501
15502 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15503
15504 ins_pipe(icmp_reg_imm);
15505 %}
15506
15507 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15508 %{
15509 match(Set cr (CmpUL op1 op2));
15510
15511 effect(DEF cr, USE op1, USE op2);
15512
15513 ins_cost(INSN_COST);
15514 format %{ "cmp $op1, $op2" %}
15515
15516 ins_encode(aarch64_enc_cmp(op1, op2));
15517
15518 ins_pipe(icmp_reg_reg);
15519 %}
15520
15521 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15522 %{
15523 match(Set cr (CmpUL op1 zero));
15524
15525 effect(DEF cr, USE op1);
15526
15527 ins_cost(INSN_COST);
15528 format %{ "tst $op1" %}
15529
15530 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15531
15532 ins_pipe(icmp_reg_imm);
15533 %}
15534
15535 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15536 %{
15537 match(Set cr (CmpUL op1 op2));
15538
15539 effect(DEF cr, USE op1);
15540
15541 ins_cost(INSN_COST);
15542 format %{ "cmp $op1, $op2" %}
15543
15544 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15545
15546 ins_pipe(icmp_reg_imm);
15547 %}
15548
15549 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15550 %{
15551 match(Set cr (CmpUL op1 op2));
15552
15553 effect(DEF cr, USE op1);
15554
15555 ins_cost(INSN_COST * 2);
15556 format %{ "cmp $op1, $op2" %}
15557
15558 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15559
15560 ins_pipe(icmp_reg_imm);
15561 %}
15562
15563 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15564 %{
15565 match(Set cr (CmpP op1 op2));
15566
15567 effect(DEF cr, USE op1, USE op2);
15568
15569 ins_cost(INSN_COST);
15570 format %{ "cmp $op1, $op2\t // ptr" %}
15571
15572 ins_encode(aarch64_enc_cmpp(op1, op2));
15573
15574 ins_pipe(icmp_reg_reg);
15575 %}
15576
15577 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15578 %{
15579 match(Set cr (CmpN op1 op2));
15580
15581 effect(DEF cr, USE op1, USE op2);
15582
15583 ins_cost(INSN_COST);
15584 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15585
15586 ins_encode(aarch64_enc_cmpn(op1, op2));
15587
15588 ins_pipe(icmp_reg_reg);
15589 %}
15590
15591 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15592 %{
15593 match(Set cr (CmpP op1 zero));
15594
15595 effect(DEF cr, USE op1, USE zero);
15596
15597 ins_cost(INSN_COST);
15598 format %{ "cmp $op1, 0\t // ptr" %}
15599
15600 ins_encode(aarch64_enc_testp(op1));
15601
15602 ins_pipe(icmp_reg_imm);
15603 %}
15604
15605 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15606 %{
15607 match(Set cr (CmpN op1 zero));
15608
15609 effect(DEF cr, USE op1, USE zero);
15610
15611 ins_cost(INSN_COST);
15612 format %{ "cmp $op1, 0\t // compressed ptr" %}
15613
15614 ins_encode(aarch64_enc_testn(op1));
15615
15616 ins_pipe(icmp_reg_imm);
15617 %}
15618
15619 // FP comparisons
15620 //
15621 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15622 // using normal cmpOp. See declaration of rFlagsReg for details.
15623
15624 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15625 %{
15626 match(Set cr (CmpF src1 src2));
15627
15628 ins_cost(3 * INSN_COST);
15629 format %{ "fcmps $src1, $src2" %}
15630
15631 ins_encode %{
15632 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15633 %}
15634
15635 ins_pipe(pipe_class_compare);
15636 %}
15637
15638 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15639 %{
15640 match(Set cr (CmpF src1 src2));
15641
15642 ins_cost(3 * INSN_COST);
15643 format %{ "fcmps $src1, 0.0" %}
15644
15645 ins_encode %{
15646 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15647 %}
15648
15649 ins_pipe(pipe_class_compare);
15650 %}
15651 // FROM HERE
15652
15653 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15654 %{
15655 match(Set cr (CmpD src1 src2));
15656
15657 ins_cost(3 * INSN_COST);
15658 format %{ "fcmpd $src1, $src2" %}
15659
15660 ins_encode %{
15661 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15662 %}
15663
15664 ins_pipe(pipe_class_compare);
15665 %}
15666
15667 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15668 %{
15669 match(Set cr (CmpD src1 src2));
15670
15671 ins_cost(3 * INSN_COST);
15672 format %{ "fcmpd $src1, 0.0" %}
15673
15674 ins_encode %{
15675 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15676 %}
15677
15678 ins_pipe(pipe_class_compare);
15679 %}
15680
15681 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15682 %{
15683 match(Set dst (CmpF3 src1 src2));
15684 effect(KILL cr);
15685
15686 ins_cost(5 * INSN_COST);
15687 format %{ "fcmps $src1, $src2\n\t"
15688 "csinvw($dst, zr, zr, eq\n\t"
15689 "csnegw($dst, $dst, $dst, lt)"
15690 %}
15691
15692 ins_encode %{
15693 Label done;
15694 FloatRegister s1 = as_FloatRegister($src1$$reg);
15695 FloatRegister s2 = as_FloatRegister($src2$$reg);
15696 Register d = as_Register($dst$$reg);
15697 __ fcmps(s1, s2);
15698 // installs 0 if EQ else -1
15699 __ csinvw(d, zr, zr, Assembler::EQ);
15700 // keeps -1 if less or unordered else installs 1
15701 __ csnegw(d, d, d, Assembler::LT);
15702 __ bind(done);
15703 %}
15704
15705 ins_pipe(pipe_class_default);
15706
15707 %}
15708
15709 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15710 %{
15711 match(Set dst (CmpD3 src1 src2));
15712 effect(KILL cr);
15713
15714 ins_cost(5 * INSN_COST);
15715 format %{ "fcmpd $src1, $src2\n\t"
15716 "csinvw($dst, zr, zr, eq\n\t"
15717 "csnegw($dst, $dst, $dst, lt)"
15718 %}
15719
15720 ins_encode %{
15721 Label done;
15722 FloatRegister s1 = as_FloatRegister($src1$$reg);
15723 FloatRegister s2 = as_FloatRegister($src2$$reg);
15724 Register d = as_Register($dst$$reg);
15725 __ fcmpd(s1, s2);
15726 // installs 0 if EQ else -1
15727 __ csinvw(d, zr, zr, Assembler::EQ);
15728 // keeps -1 if less or unordered else installs 1
15729 __ csnegw(d, d, d, Assembler::LT);
15730 __ bind(done);
15731 %}
15732 ins_pipe(pipe_class_default);
15733
15734 %}
15735
15736 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15737 %{
15738 match(Set dst (CmpF3 src1 zero));
15739 effect(KILL cr);
15740
15741 ins_cost(5 * INSN_COST);
15742 format %{ "fcmps $src1, 0.0\n\t"
15743 "csinvw($dst, zr, zr, eq\n\t"
15744 "csnegw($dst, $dst, $dst, lt)"
15745 %}
15746
15747 ins_encode %{
15748 Label done;
15749 FloatRegister s1 = as_FloatRegister($src1$$reg);
15750 Register d = as_Register($dst$$reg);
15751 __ fcmps(s1, 0.0);
15752 // installs 0 if EQ else -1
15753 __ csinvw(d, zr, zr, Assembler::EQ);
15754 // keeps -1 if less or unordered else installs 1
15755 __ csnegw(d, d, d, Assembler::LT);
15756 __ bind(done);
15757 %}
15758
15759 ins_pipe(pipe_class_default);
15760
15761 %}
15762
15763 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15764 %{
15765 match(Set dst (CmpD3 src1 zero));
15766 effect(KILL cr);
15767
15768 ins_cost(5 * INSN_COST);
15769 format %{ "fcmpd $src1, 0.0\n\t"
15770 "csinvw($dst, zr, zr, eq\n\t"
15771 "csnegw($dst, $dst, $dst, lt)"
15772 %}
15773
15774 ins_encode %{
15775 Label done;
15776 FloatRegister s1 = as_FloatRegister($src1$$reg);
15777 Register d = as_Register($dst$$reg);
15778 __ fcmpd(s1, 0.0);
15779 // installs 0 if EQ else -1
15780 __ csinvw(d, zr, zr, Assembler::EQ);
15781 // keeps -1 if less or unordered else installs 1
15782 __ csnegw(d, d, d, Assembler::LT);
15783 __ bind(done);
15784 %}
15785 ins_pipe(pipe_class_default);
15786
15787 %}
15788
15789 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15790 %{
15791 match(Set dst (CmpLTMask p q));
15792 effect(KILL cr);
15793
15794 ins_cost(3 * INSN_COST);
15795
15796 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15797 "csetw $dst, lt\n\t"
15798 "subw $dst, zr, $dst"
15799 %}
15800
15801 ins_encode %{
15802 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15803 __ csetw(as_Register($dst$$reg), Assembler::LT);
15804 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15805 %}
15806
15807 ins_pipe(ialu_reg_reg);
15808 %}
15809
15810 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15811 %{
15812 match(Set dst (CmpLTMask src zero));
15813 effect(KILL cr);
15814
15815 ins_cost(INSN_COST);
15816
15817 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15818
15819 ins_encode %{
15820 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15821 %}
15822
15823 ins_pipe(ialu_reg_shift);
15824 %}
15825
15826 // ============================================================================
15827 // Max and Min
15828
15829 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15830
15831 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15832 %{
15833 effect(DEF cr, USE src);
15834 ins_cost(INSN_COST);
15835 format %{ "cmpw $src, 0" %}
15836
15837 ins_encode %{
15838 __ cmpw($src$$Register, 0);
15839 %}
15840 ins_pipe(icmp_reg_imm);
15841 %}
15842
15843 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15844 %{
15845 match(Set dst (MinI src1 src2));
15846 ins_cost(INSN_COST * 3);
15847
15848 expand %{
15849 rFlagsReg cr;
15850 compI_reg_reg(cr, src1, src2);
15851 cmovI_reg_reg_lt(dst, src1, src2, cr);
15852 %}
15853 %}
15854
15855 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15856 %{
15857 match(Set dst (MaxI src1 src2));
15858 ins_cost(INSN_COST * 3);
15859
15860 expand %{
15861 rFlagsReg cr;
15862 compI_reg_reg(cr, src1, src2);
15863 cmovI_reg_reg_gt(dst, src1, src2, cr);
15864 %}
15865 %}
15866
15867
15868 // ============================================================================
15869 // Branch Instructions
15870
15871 // Direct Branch.
15872 instruct branch(label lbl)
15873 %{
15874 match(Goto);
15875
15876 effect(USE lbl);
15877
15878 ins_cost(BRANCH_COST);
15879 format %{ "b $lbl" %}
15880
15881 ins_encode(aarch64_enc_b(lbl));
15882
15883 ins_pipe(pipe_branch);
15884 %}
15885
15886 // Conditional Near Branch
15887 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15888 %{
15889 // Same match rule as `branchConFar'.
15890 match(If cmp cr);
15891
15892 effect(USE lbl);
15893
15894 ins_cost(BRANCH_COST);
15895 // If set to 1 this indicates that the current instruction is a
15896 // short variant of a long branch. This avoids using this
15897 // instruction in first-pass matching. It will then only be used in
15898 // the `Shorten_branches' pass.
15899 // ins_short_branch(1);
15900 format %{ "b$cmp $lbl" %}
15901
15902 ins_encode(aarch64_enc_br_con(cmp, lbl));
15903
15904 ins_pipe(pipe_branch_cond);
15905 %}
15906
15907 // Conditional Near Branch Unsigned
15908 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15909 %{
15910 // Same match rule as `branchConFar'.
15911 match(If cmp cr);
15912
15913 effect(USE lbl);
15914
15915 ins_cost(BRANCH_COST);
15916 // If set to 1 this indicates that the current instruction is a
15917 // short variant of a long branch. This avoids using this
15918 // instruction in first-pass matching. It will then only be used in
15919 // the `Shorten_branches' pass.
15920 // ins_short_branch(1);
15921 format %{ "b$cmp $lbl\t# unsigned" %}
15922
15923 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15924
15925 ins_pipe(pipe_branch_cond);
15926 %}
15927
15928 // Make use of CBZ and CBNZ. These instructions, as well as being
15929 // shorter than (cmp; branch), have the additional benefit of not
15930 // killing the flags.
15931
15932 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15933 match(If cmp (CmpI op1 op2));
15934 effect(USE labl);
15935
15936 ins_cost(BRANCH_COST);
15937 format %{ "cbw$cmp $op1, $labl" %}
15938 ins_encode %{
15939 Label* L = $labl$$label;
15940 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15941 if (cond == Assembler::EQ)
15942 __ cbzw($op1$$Register, *L);
15943 else
15944 __ cbnzw($op1$$Register, *L);
15945 %}
15946 ins_pipe(pipe_cmp_branch);
15947 %}
15948
15949 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15950 match(If cmp (CmpL op1 op2));
15951 effect(USE labl);
15952
15953 ins_cost(BRANCH_COST);
15954 format %{ "cb$cmp $op1, $labl" %}
15955 ins_encode %{
15956 Label* L = $labl$$label;
15957 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15958 if (cond == Assembler::EQ)
15959 __ cbz($op1$$Register, *L);
15960 else
15961 __ cbnz($op1$$Register, *L);
15962 %}
15963 ins_pipe(pipe_cmp_branch);
15964 %}
15965
15966 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15967 match(If cmp (CmpP op1 op2));
15968 effect(USE labl);
15969
15970 ins_cost(BRANCH_COST);
15971 format %{ "cb$cmp $op1, $labl" %}
15972 ins_encode %{
15973 Label* L = $labl$$label;
15974 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15975 if (cond == Assembler::EQ)
15976 __ cbz($op1$$Register, *L);
15977 else
15978 __ cbnz($op1$$Register, *L);
15979 %}
15980 ins_pipe(pipe_cmp_branch);
15981 %}
15982
15983 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15984 match(If cmp (CmpN op1 op2));
15985 effect(USE labl);
15986
15987 ins_cost(BRANCH_COST);
15988 format %{ "cbw$cmp $op1, $labl" %}
15989 ins_encode %{
15990 Label* L = $labl$$label;
15991 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15992 if (cond == Assembler::EQ)
15993 __ cbzw($op1$$Register, *L);
15994 else
15995 __ cbnzw($op1$$Register, *L);
15996 %}
15997 ins_pipe(pipe_cmp_branch);
15998 %}
15999
16000 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16001 match(If cmp (CmpP (DecodeN oop) zero));
16002 effect(USE labl);
16003
16004 ins_cost(BRANCH_COST);
16005 format %{ "cb$cmp $oop, $labl" %}
16006 ins_encode %{
16007 Label* L = $labl$$label;
16008 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16009 if (cond == Assembler::EQ)
16010 __ cbzw($oop$$Register, *L);
16011 else
16012 __ cbnzw($oop$$Register, *L);
16013 %}
16014 ins_pipe(pipe_cmp_branch);
16015 %}
16016
16017 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16018 match(If cmp (CmpU op1 op2));
16019 effect(USE labl);
16020
16021 ins_cost(BRANCH_COST);
16022 format %{ "cbw$cmp $op1, $labl" %}
16023 ins_encode %{
16024 Label* L = $labl$$label;
16025 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16026 if (cond == Assembler::EQ || cond == Assembler::LS) {
16027 __ cbzw($op1$$Register, *L);
16028 } else {
16029 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16030 __ cbnzw($op1$$Register, *L);
16031 }
16032 %}
16033 ins_pipe(pipe_cmp_branch);
16034 %}
16035
16036 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16037 match(If cmp (CmpUL op1 op2));
16038 effect(USE labl);
16039
16040 ins_cost(BRANCH_COST);
16041 format %{ "cb$cmp $op1, $labl" %}
16042 ins_encode %{
16043 Label* L = $labl$$label;
16044 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16045 if (cond == Assembler::EQ || cond == Assembler::LS) {
16046 __ cbz($op1$$Register, *L);
16047 } else {
16048 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16049 __ cbnz($op1$$Register, *L);
16050 }
16051 %}
16052 ins_pipe(pipe_cmp_branch);
16053 %}
16054
16055 // Test bit and Branch
16056
16057 // Patterns for short (< 32KiB) variants
16058 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16059 match(If cmp (CmpL op1 op2));
16060 effect(USE labl);
16061
16062 ins_cost(BRANCH_COST);
16063 format %{ "cb$cmp $op1, $labl # long" %}
16064 ins_encode %{
16065 Label* L = $labl$$label;
16066 Assembler::Condition cond =
16067 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16068 __ tbr(cond, $op1$$Register, 63, *L);
16069 %}
16070 ins_pipe(pipe_cmp_branch);
16071 ins_short_branch(1);
16072 %}
16073
16074 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16075 match(If cmp (CmpI op1 op2));
16076 effect(USE labl);
16077
16078 ins_cost(BRANCH_COST);
16079 format %{ "cb$cmp $op1, $labl # int" %}
16080 ins_encode %{
16081 Label* L = $labl$$label;
16082 Assembler::Condition cond =
16083 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16084 __ tbr(cond, $op1$$Register, 31, *L);
16085 %}
16086 ins_pipe(pipe_cmp_branch);
16087 ins_short_branch(1);
16088 %}
16089
16090 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16091 match(If cmp (CmpL (AndL op1 op2) op3));
16092 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16093 effect(USE labl);
16094
16095 ins_cost(BRANCH_COST);
16096 format %{ "tb$cmp $op1, $op2, $labl" %}
16097 ins_encode %{
16098 Label* L = $labl$$label;
16099 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16100 int bit = exact_log2_long($op2$$constant);
16101 __ tbr(cond, $op1$$Register, bit, *L);
16102 %}
16103 ins_pipe(pipe_cmp_branch);
16104 ins_short_branch(1);
16105 %}
16106
16107 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16108 match(If cmp (CmpI (AndI op1 op2) op3));
16109 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16110 effect(USE labl);
16111
16112 ins_cost(BRANCH_COST);
16113 format %{ "tb$cmp $op1, $op2, $labl" %}
16114 ins_encode %{
16115 Label* L = $labl$$label;
16116 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16117 int bit = exact_log2((juint)$op2$$constant);
16118 __ tbr(cond, $op1$$Register, bit, *L);
16119 %}
16120 ins_pipe(pipe_cmp_branch);
16121 ins_short_branch(1);
16122 %}
16123
16124 // And far variants
16125 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16126 match(If cmp (CmpL op1 op2));
16127 effect(USE labl);
16128
16129 ins_cost(BRANCH_COST);
16130 format %{ "cb$cmp $op1, $labl # long" %}
16131 ins_encode %{
16132 Label* L = $labl$$label;
16133 Assembler::Condition cond =
16134 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16135 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16136 %}
16137 ins_pipe(pipe_cmp_branch);
16138 %}
16139
16140 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16141 match(If cmp (CmpI op1 op2));
16142 effect(USE labl);
16143
16144 ins_cost(BRANCH_COST);
16145 format %{ "cb$cmp $op1, $labl # int" %}
16146 ins_encode %{
16147 Label* L = $labl$$label;
16148 Assembler::Condition cond =
16149 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16150 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16151 %}
16152 ins_pipe(pipe_cmp_branch);
16153 %}
16154
16155 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16156 match(If cmp (CmpL (AndL op1 op2) op3));
16157 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16158 effect(USE labl);
16159
16160 ins_cost(BRANCH_COST);
16161 format %{ "tb$cmp $op1, $op2, $labl" %}
16162 ins_encode %{
16163 Label* L = $labl$$label;
16164 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16165 int bit = exact_log2_long($op2$$constant);
16166 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16167 %}
16168 ins_pipe(pipe_cmp_branch);
16169 %}
16170
16171 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16172 match(If cmp (CmpI (AndI op1 op2) op3));
16173 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16174 effect(USE labl);
16175
16176 ins_cost(BRANCH_COST);
16177 format %{ "tb$cmp $op1, $op2, $labl" %}
16178 ins_encode %{
16179 Label* L = $labl$$label;
16180 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16181 int bit = exact_log2((juint)$op2$$constant);
16182 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16183 %}
16184 ins_pipe(pipe_cmp_branch);
16185 %}
16186
16187 // Test bits
16188
16189 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16190 match(Set cr (CmpL (AndL op1 op2) op3));
16191 predicate(Assembler::operand_valid_for_logical_immediate
16192 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16193
16194 ins_cost(INSN_COST);
16195 format %{ "tst $op1, $op2 # long" %}
16196 ins_encode %{
16197 __ tst($op1$$Register, $op2$$constant);
16198 %}
16199 ins_pipe(ialu_reg_reg);
16200 %}
16201
16202 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16203 match(Set cr (CmpI (AndI op1 op2) op3));
16204 predicate(Assembler::operand_valid_for_logical_immediate
16205 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16206
16207 ins_cost(INSN_COST);
16208 format %{ "tst $op1, $op2 # int" %}
16209 ins_encode %{
16210 __ tstw($op1$$Register, $op2$$constant);
16211 %}
16212 ins_pipe(ialu_reg_reg);
16213 %}
16214
16215 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16216 match(Set cr (CmpL (AndL op1 op2) op3));
16217
16218 ins_cost(INSN_COST);
16219 format %{ "tst $op1, $op2 # long" %}
16220 ins_encode %{
16221 __ tst($op1$$Register, $op2$$Register);
16222 %}
16223 ins_pipe(ialu_reg_reg);
16224 %}
16225
16226 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16227 match(Set cr (CmpI (AndI op1 op2) op3));
16228
16229 ins_cost(INSN_COST);
16230 format %{ "tstw $op1, $op2 # int" %}
16231 ins_encode %{
16232 __ tstw($op1$$Register, $op2$$Register);
16233 %}
16234 ins_pipe(ialu_reg_reg);
16235 %}
16236
16237
16238 // Conditional Far Branch
16239 // Conditional Far Branch Unsigned
16240 // TODO: fixme
16241
16242 // counted loop end branch near
16243 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16244 %{
16245 match(CountedLoopEnd cmp cr);
16246
16247 effect(USE lbl);
16248
16249 ins_cost(BRANCH_COST);
16250 // short variant.
16251 // ins_short_branch(1);
16252 format %{ "b$cmp $lbl \t// counted loop end" %}
16253
16254 ins_encode(aarch64_enc_br_con(cmp, lbl));
16255
16256 ins_pipe(pipe_branch);
16257 %}
16258
16259 // counted loop end branch far
16260 // TODO: fixme
16261
16262 // ============================================================================
16263 // inlined locking and unlocking
16264
16265 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16266 %{
16267 match(Set cr (FastLock object box));
16268 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16269
16270 ins_cost(5 * INSN_COST);
16271 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16272
16273 ins_encode %{
16274 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16275 %}
16276
16277 ins_pipe(pipe_serial);
16278 %}
16279
16280 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16281 %{
16282 match(Set cr (FastUnlock object box));
16283 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16284
16285 ins_cost(5 * INSN_COST);
16286 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16287
16288 ins_encode %{
16289 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16290 %}
16291
16292 ins_pipe(pipe_serial);
16293 %}
16294
16295 // ============================================================================
16296 // Safepoint Instructions
16297
16298 // TODO
16299 // provide a near and far version of this code
16300
16301 instruct safePoint(rFlagsReg cr, iRegP poll)
16302 %{
16303 match(SafePoint poll);
16304 effect(KILL cr);
16305
16306 format %{
16307 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16308 %}
16309 ins_encode %{
16310 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16311 %}
16312 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16313 %}
16314
16315
16316 // ============================================================================
16317 // Procedure Call/Return Instructions
16318
16319 // Call Java Static Instruction
16320
16321 instruct CallStaticJavaDirect(method meth)
16322 %{
16323 match(CallStaticJava);
16324
16325 effect(USE meth);
16326
16327 ins_cost(CALL_COST);
16328
16329 format %{ "call,static $meth \t// ==> " %}
16330
16331 ins_encode(aarch64_enc_java_static_call(meth),
16332 aarch64_enc_call_epilog);
16333
16334 ins_pipe(pipe_class_call);
16335 %}
16336
16337 // TO HERE
16338
16339 // Call Java Dynamic Instruction
16340 instruct CallDynamicJavaDirect(method meth)
16341 %{
16342 match(CallDynamicJava);
16343
16344 effect(USE meth);
16345
16346 ins_cost(CALL_COST);
16347
16348 format %{ "CALL,dynamic $meth \t// ==> " %}
16349
16350 ins_encode(aarch64_enc_java_dynamic_call(meth),
16351 aarch64_enc_call_epilog);
16352
16353 ins_pipe(pipe_class_call);
16354 %}
16355
16356 // Call Runtime Instruction
16357
16358 instruct CallRuntimeDirect(method meth)
16359 %{
16360 match(CallRuntime);
16361
16362 effect(USE meth);
16363
16364 ins_cost(CALL_COST);
16365
16366 format %{ "CALL, runtime $meth" %}
16367
16368 ins_encode( aarch64_enc_java_to_runtime(meth) );
16369
16370 ins_pipe(pipe_class_call);
16371 %}
16372
16373 // Call Runtime Instruction
16374
16375 instruct CallLeafDirect(method meth)
16376 %{
16377 match(CallLeaf);
16378
16379 effect(USE meth);
16380
16381 ins_cost(CALL_COST);
16382
16383 format %{ "CALL, runtime leaf $meth" %}
16384
16385 ins_encode( aarch64_enc_java_to_runtime(meth) );
16386
16387 ins_pipe(pipe_class_call);
16388 %}
16389
16390 // Call Runtime Instruction without safepoint and with vector arguments
16391 instruct CallLeafDirectVector(method meth)
16392 %{
16393 match(CallLeafVector);
16394
16395 effect(USE meth);
16396
16397 ins_cost(CALL_COST);
16398
16399 format %{ "CALL, runtime leaf vector $meth" %}
16400
16401 ins_encode(aarch64_enc_java_to_runtime(meth));
16402
16403 ins_pipe(pipe_class_call);
16404 %}
16405
16406 // Call Runtime Instruction
16407
16408 instruct CallLeafNoFPDirect(method meth)
16409 %{
16410 match(CallLeafNoFP);
16411
16412 effect(USE meth);
16413
16414 ins_cost(CALL_COST);
16415
16416 format %{ "CALL, runtime leaf nofp $meth" %}
16417
16418 ins_encode( aarch64_enc_java_to_runtime(meth) );
16419
16420 ins_pipe(pipe_class_call);
16421 %}
16422
16423 // Tail Call; Jump from runtime stub to Java code.
16424 // Also known as an 'interprocedural jump'.
16425 // Target of jump will eventually return to caller.
16426 // TailJump below removes the return address.
16427 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16428 // emitted just above the TailCall which has reset rfp to the caller state.
16429 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16430 %{
16431 match(TailCall jump_target method_ptr);
16432
16433 ins_cost(CALL_COST);
16434
16435 format %{ "br $jump_target\t# $method_ptr holds method" %}
16436
16437 ins_encode(aarch64_enc_tail_call(jump_target));
16438
16439 ins_pipe(pipe_class_call);
16440 %}
16441
16442 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16443 %{
16444 match(TailJump jump_target ex_oop);
16445
16446 ins_cost(CALL_COST);
16447
16448 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16449
16450 ins_encode(aarch64_enc_tail_jmp(jump_target));
16451
16452 ins_pipe(pipe_class_call);
16453 %}
16454
16455 // Forward exception.
16456 instruct ForwardExceptionjmp()
16457 %{
16458 match(ForwardException);
16459 ins_cost(CALL_COST);
16460
16461 format %{ "b forward_exception_stub" %}
16462 ins_encode %{
16463 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16464 %}
16465 ins_pipe(pipe_class_call);
16466 %}
16467
16468 // Create exception oop: created by stack-crawling runtime code.
16469 // Created exception is now available to this handler, and is setup
16470 // just prior to jumping to this handler. No code emitted.
16471 // TODO check
16472 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16473 instruct CreateException(iRegP_R0 ex_oop)
16474 %{
16475 match(Set ex_oop (CreateEx));
16476
16477 format %{ " -- \t// exception oop; no code emitted" %}
16478
16479 size(0);
16480
16481 ins_encode( /*empty*/ );
16482
16483 ins_pipe(pipe_class_empty);
16484 %}
16485
16486 // Rethrow exception: The exception oop will come in the first
16487 // argument position. Then JUMP (not call) to the rethrow stub code.
16488 instruct RethrowException() %{
16489 match(Rethrow);
16490 ins_cost(CALL_COST);
16491
16492 format %{ "b rethrow_stub" %}
16493
16494 ins_encode( aarch64_enc_rethrow() );
16495
16496 ins_pipe(pipe_class_call);
16497 %}
16498
16499
16500 // Return Instruction
16501 // epilog node loads ret address into lr as part of frame pop
16502 instruct Ret()
16503 %{
16504 match(Return);
16505
16506 format %{ "ret\t// return register" %}
16507
16508 ins_encode( aarch64_enc_ret() );
16509
16510 ins_pipe(pipe_branch);
16511 %}
16512
16513 // Die now.
16514 instruct ShouldNotReachHere() %{
16515 match(Halt);
16516
16517 ins_cost(CALL_COST);
16518 format %{ "ShouldNotReachHere" %}
16519
16520 ins_encode %{
16521 if (is_reachable()) {
16522 const char* str = __ code_string(_halt_reason);
16523 __ stop(str);
16524 }
16525 %}
16526
16527 ins_pipe(pipe_class_default);
16528 %}
16529
16530 // ============================================================================
16531 // Partial Subtype Check
16532 //
16533 // superklass array for an instance of the superklass. Set a hidden
16534 // internal cache on a hit (cache is checked with exposed code in
16535 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16536 // encoding ALSO sets flags.
16537
16538 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16539 %{
16540 match(Set result (PartialSubtypeCheck sub super));
16541 predicate(!UseSecondarySupersTable);
16542 effect(KILL cr, KILL temp);
16543
16544 ins_cost(20 * INSN_COST); // slightly larger than the next version
16545 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16546
16547 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16548
16549 opcode(0x1); // Force zero of result reg on hit
16550
16551 ins_pipe(pipe_class_memory);
16552 %}
16553
16554 // Two versions of partialSubtypeCheck, both used when we need to
16555 // search for a super class in the secondary supers array. The first
16556 // is used when we don't know _a priori_ the class being searched
16557 // for. The second, far more common, is used when we do know: this is
16558 // used for instanceof, checkcast, and any case where C2 can determine
16559 // it by constant propagation.
16560
16561 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16562 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16563 rFlagsReg cr)
16564 %{
16565 match(Set result (PartialSubtypeCheck sub super));
16566 predicate(UseSecondarySupersTable);
16567 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16568
16569 ins_cost(10 * INSN_COST); // slightly larger than the next version
16570 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16571
16572 ins_encode %{
16573 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16574 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16575 $vtemp$$FloatRegister,
16576 $result$$Register, /*L_success*/nullptr);
16577 %}
16578
16579 ins_pipe(pipe_class_memory);
16580 %}
16581
16582 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16583 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16584 rFlagsReg cr)
16585 %{
16586 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16587 predicate(UseSecondarySupersTable);
16588 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16589
16590 ins_cost(5 * INSN_COST); // smaller than the next version
16591 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16592
16593 ins_encode %{
16594 bool success = false;
16595 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16596 if (InlineSecondarySupersTest) {
16597 success =
16598 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16599 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16600 $vtemp$$FloatRegister,
16601 $result$$Register,
16602 super_klass_slot);
16603 } else {
16604 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16605 success = (call != nullptr);
16606 }
16607 if (!success) {
16608 ciEnv::current()->record_failure("CodeCache is full");
16609 return;
16610 }
16611 %}
16612
16613 ins_pipe(pipe_class_memory);
16614 %}
16615
16616 // Intrisics for String.compareTo()
16617
16618 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16619 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16620 %{
16621 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16622 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16623 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16624
16625 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16626 ins_encode %{
16627 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16628 __ string_compare($str1$$Register, $str2$$Register,
16629 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16630 $tmp1$$Register, $tmp2$$Register,
16631 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16632 %}
16633 ins_pipe(pipe_class_memory);
16634 %}
16635
16636 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16637 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16638 %{
16639 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16640 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16641 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16642
16643 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16644 ins_encode %{
16645 __ string_compare($str1$$Register, $str2$$Register,
16646 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16647 $tmp1$$Register, $tmp2$$Register,
16648 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16649 %}
16650 ins_pipe(pipe_class_memory);
16651 %}
16652
16653 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16654 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16655 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16656 %{
16657 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16658 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16659 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16660 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16661
16662 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16663 ins_encode %{
16664 __ string_compare($str1$$Register, $str2$$Register,
16665 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16666 $tmp1$$Register, $tmp2$$Register,
16667 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16668 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16669 %}
16670 ins_pipe(pipe_class_memory);
16671 %}
16672
16673 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16674 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16675 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16676 %{
16677 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16678 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16679 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16680 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16681
16682 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16683 ins_encode %{
16684 __ string_compare($str1$$Register, $str2$$Register,
16685 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16686 $tmp1$$Register, $tmp2$$Register,
16687 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16688 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16689 %}
16690 ins_pipe(pipe_class_memory);
16691 %}
16692
16693 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16694 // these string_compare variants as NEON register type for convenience so that the prototype of
16695 // string_compare can be shared with all variants.
16696
16697 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16698 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16699 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16700 pRegGov_P1 pgtmp2, rFlagsReg cr)
16701 %{
16702 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16703 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16704 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16705 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16706
16707 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16708 ins_encode %{
16709 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16710 __ string_compare($str1$$Register, $str2$$Register,
16711 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16712 $tmp1$$Register, $tmp2$$Register,
16713 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16714 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16715 StrIntrinsicNode::LL);
16716 %}
16717 ins_pipe(pipe_class_memory);
16718 %}
16719
16720 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16721 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16722 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16723 pRegGov_P1 pgtmp2, rFlagsReg cr)
16724 %{
16725 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16726 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16727 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16728 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16729
16730 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16731 ins_encode %{
16732 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16733 __ string_compare($str1$$Register, $str2$$Register,
16734 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16735 $tmp1$$Register, $tmp2$$Register,
16736 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16737 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16738 StrIntrinsicNode::LU);
16739 %}
16740 ins_pipe(pipe_class_memory);
16741 %}
16742
16743 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16744 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16745 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16746 pRegGov_P1 pgtmp2, rFlagsReg cr)
16747 %{
16748 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16749 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16750 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16751 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16752
16753 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16754 ins_encode %{
16755 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16756 __ string_compare($str1$$Register, $str2$$Register,
16757 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16758 $tmp1$$Register, $tmp2$$Register,
16759 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16760 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16761 StrIntrinsicNode::UL);
16762 %}
16763 ins_pipe(pipe_class_memory);
16764 %}
16765
16766 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16767 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16768 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16769 pRegGov_P1 pgtmp2, rFlagsReg cr)
16770 %{
16771 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16772 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16773 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16774 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16775
16776 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16777 ins_encode %{
16778 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16779 __ string_compare($str1$$Register, $str2$$Register,
16780 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16781 $tmp1$$Register, $tmp2$$Register,
16782 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16783 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16784 StrIntrinsicNode::UU);
16785 %}
16786 ins_pipe(pipe_class_memory);
16787 %}
16788
16789 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16790 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16791 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16792 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16793 %{
16794 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16795 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16796 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16797 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16798 TEMP vtmp0, TEMP vtmp1, KILL cr);
16799 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16800 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16801
16802 ins_encode %{
16803 __ string_indexof($str1$$Register, $str2$$Register,
16804 $cnt1$$Register, $cnt2$$Register,
16805 $tmp1$$Register, $tmp2$$Register,
16806 $tmp3$$Register, $tmp4$$Register,
16807 $tmp5$$Register, $tmp6$$Register,
16808 -1, $result$$Register, StrIntrinsicNode::UU);
16809 %}
16810 ins_pipe(pipe_class_memory);
16811 %}
16812
16813 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16814 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16815 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16816 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16817 %{
16818 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16819 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16820 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16821 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16822 TEMP vtmp0, TEMP vtmp1, KILL cr);
16823 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16824 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16825
16826 ins_encode %{
16827 __ string_indexof($str1$$Register, $str2$$Register,
16828 $cnt1$$Register, $cnt2$$Register,
16829 $tmp1$$Register, $tmp2$$Register,
16830 $tmp3$$Register, $tmp4$$Register,
16831 $tmp5$$Register, $tmp6$$Register,
16832 -1, $result$$Register, StrIntrinsicNode::LL);
16833 %}
16834 ins_pipe(pipe_class_memory);
16835 %}
16836
16837 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16838 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16839 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16840 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16841 %{
16842 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16843 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16844 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16845 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16846 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16847 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16848 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16849
16850 ins_encode %{
16851 __ string_indexof($str1$$Register, $str2$$Register,
16852 $cnt1$$Register, $cnt2$$Register,
16853 $tmp1$$Register, $tmp2$$Register,
16854 $tmp3$$Register, $tmp4$$Register,
16855 $tmp5$$Register, $tmp6$$Register,
16856 -1, $result$$Register, StrIntrinsicNode::UL);
16857 %}
16858 ins_pipe(pipe_class_memory);
16859 %}
16860
16861 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16862 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16863 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16864 %{
16865 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16866 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16867 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16868 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16869 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16870 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16871
16872 ins_encode %{
16873 int icnt2 = (int)$int_cnt2$$constant;
16874 __ string_indexof($str1$$Register, $str2$$Register,
16875 $cnt1$$Register, zr,
16876 $tmp1$$Register, $tmp2$$Register,
16877 $tmp3$$Register, $tmp4$$Register, zr, zr,
16878 icnt2, $result$$Register, StrIntrinsicNode::UU);
16879 %}
16880 ins_pipe(pipe_class_memory);
16881 %}
16882
16883 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16884 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16885 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16886 %{
16887 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16888 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16889 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16890 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16891 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16892 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16893
16894 ins_encode %{
16895 int icnt2 = (int)$int_cnt2$$constant;
16896 __ string_indexof($str1$$Register, $str2$$Register,
16897 $cnt1$$Register, zr,
16898 $tmp1$$Register, $tmp2$$Register,
16899 $tmp3$$Register, $tmp4$$Register, zr, zr,
16900 icnt2, $result$$Register, StrIntrinsicNode::LL);
16901 %}
16902 ins_pipe(pipe_class_memory);
16903 %}
16904
16905 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16906 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16907 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16908 %{
16909 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16910 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16911 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16912 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16913 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16914 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16915
16916 ins_encode %{
16917 int icnt2 = (int)$int_cnt2$$constant;
16918 __ string_indexof($str1$$Register, $str2$$Register,
16919 $cnt1$$Register, zr,
16920 $tmp1$$Register, $tmp2$$Register,
16921 $tmp3$$Register, $tmp4$$Register, zr, zr,
16922 icnt2, $result$$Register, StrIntrinsicNode::UL);
16923 %}
16924 ins_pipe(pipe_class_memory);
16925 %}
16926
16927 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16928 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16929 iRegINoSp tmp3, rFlagsReg cr)
16930 %{
16931 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16932 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16933 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16934 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16935
16936 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16937
16938 ins_encode %{
16939 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16940 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16941 $tmp3$$Register);
16942 %}
16943 ins_pipe(pipe_class_memory);
16944 %}
16945
16946 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16947 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16948 iRegINoSp tmp3, rFlagsReg cr)
16949 %{
16950 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16951 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16952 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16953 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16954
16955 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16956
16957 ins_encode %{
16958 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16959 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16960 $tmp3$$Register);
16961 %}
16962 ins_pipe(pipe_class_memory);
16963 %}
16964
16965 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16966 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16967 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16968 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16969 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16970 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16971 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16972 ins_encode %{
16973 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16974 $result$$Register, $ztmp1$$FloatRegister,
16975 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16976 $ptmp$$PRegister, true /* isL */);
16977 %}
16978 ins_pipe(pipe_class_memory);
16979 %}
16980
16981 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16982 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16983 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16984 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16985 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16986 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16987 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16988 ins_encode %{
16989 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16990 $result$$Register, $ztmp1$$FloatRegister,
16991 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16992 $ptmp$$PRegister, false /* isL */);
16993 %}
16994 ins_pipe(pipe_class_memory);
16995 %}
16996
16997 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16998 iRegI_R0 result, rFlagsReg cr)
16999 %{
17000 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17001 match(Set result (StrEquals (Binary str1 str2) cnt));
17002 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17003
17004 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17005 ins_encode %{
17006 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17007 __ string_equals($str1$$Register, $str2$$Register,
17008 $result$$Register, $cnt$$Register);
17009 %}
17010 ins_pipe(pipe_class_memory);
17011 %}
17012
17013 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17014 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17015 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17016 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17017 iRegP_R10 tmp, rFlagsReg cr)
17018 %{
17019 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17020 match(Set result (AryEq ary1 ary2));
17021 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17022 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17023 TEMP vtmp6, TEMP vtmp7, KILL cr);
17024
17025 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17026 ins_encode %{
17027 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17028 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17029 $result$$Register, $tmp$$Register, 1);
17030 if (tpc == nullptr) {
17031 ciEnv::current()->record_failure("CodeCache is full");
17032 return;
17033 }
17034 %}
17035 ins_pipe(pipe_class_memory);
17036 %}
17037
17038 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17039 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17040 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17041 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17042 iRegP_R10 tmp, rFlagsReg cr)
17043 %{
17044 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17045 match(Set result (AryEq ary1 ary2));
17046 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17047 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17048 TEMP vtmp6, TEMP vtmp7, KILL cr);
17049
17050 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17051 ins_encode %{
17052 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17053 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17054 $result$$Register, $tmp$$Register, 2);
17055 if (tpc == nullptr) {
17056 ciEnv::current()->record_failure("CodeCache is full");
17057 return;
17058 }
17059 %}
17060 ins_pipe(pipe_class_memory);
17061 %}
17062
17063 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17064 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17065 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17066 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17067 %{
17068 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17069 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17070 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17071
17072 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
17073 ins_encode %{
17074 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17075 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17076 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17077 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17078 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17079 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17080 (BasicType)$basic_type$$constant);
17081 if (tpc == nullptr) {
17082 ciEnv::current()->record_failure("CodeCache is full");
17083 return;
17084 }
17085 %}
17086 ins_pipe(pipe_class_memory);
17087 %}
17088
17089 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17090 %{
17091 match(Set result (CountPositives ary1 len));
17092 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17093 format %{ "count positives byte[] $ary1,$len -> $result" %}
17094 ins_encode %{
17095 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17096 if (tpc == nullptr) {
17097 ciEnv::current()->record_failure("CodeCache is full");
17098 return;
17099 }
17100 %}
17101 ins_pipe( pipe_slow );
17102 %}
17103
17104 // fast char[] to byte[] compression
17105 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17106 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17107 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17108 iRegI_R0 result, rFlagsReg cr)
17109 %{
17110 match(Set result (StrCompressedCopy src (Binary dst len)));
17111 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17112 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17113
17114 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17115 ins_encode %{
17116 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17117 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17118 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17119 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17120 %}
17121 ins_pipe(pipe_slow);
17122 %}
17123
17124 // fast byte[] to char[] inflation
17125 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17126 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17127 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17128 %{
17129 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17130 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17131 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17132 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17133
17134 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17135 ins_encode %{
17136 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17137 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17138 $vtmp2$$FloatRegister, $tmp$$Register);
17139 if (tpc == nullptr) {
17140 ciEnv::current()->record_failure("CodeCache is full");
17141 return;
17142 }
17143 %}
17144 ins_pipe(pipe_class_memory);
17145 %}
17146
17147 // encode char[] to byte[] in ISO_8859_1
17148 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17149 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17150 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17151 iRegI_R0 result, rFlagsReg cr)
17152 %{
17153 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17154 match(Set result (EncodeISOArray src (Binary dst len)));
17155 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17156 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17157
17158 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17159 ins_encode %{
17160 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17161 $result$$Register, false,
17162 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17163 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17164 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17165 %}
17166 ins_pipe(pipe_class_memory);
17167 %}
17168
17169 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17170 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17171 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17172 iRegI_R0 result, rFlagsReg cr)
17173 %{
17174 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17175 match(Set result (EncodeISOArray src (Binary dst len)));
17176 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17177 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17178
17179 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17180 ins_encode %{
17181 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17182 $result$$Register, true,
17183 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17184 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17185 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17186 %}
17187 ins_pipe(pipe_class_memory);
17188 %}
17189
17190 //----------------------------- CompressBits/ExpandBits ------------------------
17191
17192 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17193 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17194 match(Set dst (CompressBits src mask));
17195 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17196 format %{ "mov $tsrc, $src\n\t"
17197 "mov $tmask, $mask\n\t"
17198 "bext $tdst, $tsrc, $tmask\n\t"
17199 "mov $dst, $tdst"
17200 %}
17201 ins_encode %{
17202 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17203 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17204 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17205 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17206 %}
17207 ins_pipe(pipe_slow);
17208 %}
17209
17210 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17211 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17212 match(Set dst (CompressBits (LoadI mem) mask));
17213 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17214 format %{ "ldrs $tsrc, $mem\n\t"
17215 "ldrs $tmask, $mask\n\t"
17216 "bext $tdst, $tsrc, $tmask\n\t"
17217 "mov $dst, $tdst"
17218 %}
17219 ins_encode %{
17220 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17221 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17222 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17223 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17224 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17225 %}
17226 ins_pipe(pipe_slow);
17227 %}
17228
17229 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17230 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17231 match(Set dst (CompressBits src mask));
17232 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17233 format %{ "mov $tsrc, $src\n\t"
17234 "mov $tmask, $mask\n\t"
17235 "bext $tdst, $tsrc, $tmask\n\t"
17236 "mov $dst, $tdst"
17237 %}
17238 ins_encode %{
17239 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17240 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17241 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17242 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17243 %}
17244 ins_pipe(pipe_slow);
17245 %}
17246
17247 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17248 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17249 match(Set dst (CompressBits (LoadL mem) mask));
17250 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17251 format %{ "ldrd $tsrc, $mem\n\t"
17252 "ldrd $tmask, $mask\n\t"
17253 "bext $tdst, $tsrc, $tmask\n\t"
17254 "mov $dst, $tdst"
17255 %}
17256 ins_encode %{
17257 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17258 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17259 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17260 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17261 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17262 %}
17263 ins_pipe(pipe_slow);
17264 %}
17265
17266 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17267 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17268 match(Set dst (ExpandBits src mask));
17269 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17270 format %{ "mov $tsrc, $src\n\t"
17271 "mov $tmask, $mask\n\t"
17272 "bdep $tdst, $tsrc, $tmask\n\t"
17273 "mov $dst, $tdst"
17274 %}
17275 ins_encode %{
17276 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17277 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17278 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17279 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17280 %}
17281 ins_pipe(pipe_slow);
17282 %}
17283
17284 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17285 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17286 match(Set dst (ExpandBits (LoadI mem) mask));
17287 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17288 format %{ "ldrs $tsrc, $mem\n\t"
17289 "ldrs $tmask, $mask\n\t"
17290 "bdep $tdst, $tsrc, $tmask\n\t"
17291 "mov $dst, $tdst"
17292 %}
17293 ins_encode %{
17294 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17295 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17296 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17297 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17298 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17299 %}
17300 ins_pipe(pipe_slow);
17301 %}
17302
17303 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17304 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17305 match(Set dst (ExpandBits src mask));
17306 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17307 format %{ "mov $tsrc, $src\n\t"
17308 "mov $tmask, $mask\n\t"
17309 "bdep $tdst, $tsrc, $tmask\n\t"
17310 "mov $dst, $tdst"
17311 %}
17312 ins_encode %{
17313 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17314 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17315 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17316 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17317 %}
17318 ins_pipe(pipe_slow);
17319 %}
17320
17321
17322 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17323 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17324 match(Set dst (ExpandBits (LoadL mem) mask));
17325 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17326 format %{ "ldrd $tsrc, $mem\n\t"
17327 "ldrd $tmask, $mask\n\t"
17328 "bdep $tdst, $tsrc, $tmask\n\t"
17329 "mov $dst, $tdst"
17330 %}
17331 ins_encode %{
17332 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17333 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17334 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17335 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17336 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17337 %}
17338 ins_pipe(pipe_slow);
17339 %}
17340
17341 //----------------------------- Reinterpret ----------------------------------
17342 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17343 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17344 match(Set dst (ReinterpretHF2S src));
17345 format %{ "reinterpretHF2S $dst, $src" %}
17346 ins_encode %{
17347 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17348 %}
17349 ins_pipe(pipe_slow);
17350 %}
17351
17352 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17353 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17354 match(Set dst (ReinterpretS2HF src));
17355 format %{ "reinterpretS2HF $dst, $src" %}
17356 ins_encode %{
17357 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17358 %}
17359 ins_pipe(pipe_slow);
17360 %}
17361
17362 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17363 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17364 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
17365 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
17366 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
17367 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17368 // can be omitted in this pattern, resulting in -
17369 // fcvt $dst, $src // Convert float to half-precision float
17370 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17371 %{
17372 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17373 format %{ "convF2HFAndS2HF $dst, $src" %}
17374 ins_encode %{
17375 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17376 %}
17377 ins_pipe(pipe_slow);
17378 %}
17379
17380 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17381 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17382 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
17383 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17384 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
17385 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17386 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17387 // resulting in -
17388 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
17389 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17390 %{
17391 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17392 format %{ "convHF2SAndHF2F $dst, $src" %}
17393 ins_encode %{
17394 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17395 %}
17396 ins_pipe(pipe_slow);
17397 %}
17398
17399 // ============================================================================
17400 // This name is KNOWN by the ADLC and cannot be changed.
17401 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17402 // for this guy.
17403 instruct tlsLoadP(thread_RegP dst)
17404 %{
17405 match(Set dst (ThreadLocal));
17406
17407 ins_cost(0);
17408
17409 format %{ " -- \t// $dst=Thread::current(), empty" %}
17410
17411 size(0);
17412
17413 ins_encode( /*empty*/ );
17414
17415 ins_pipe(pipe_class_empty);
17416 %}
17417
17418 //----------PEEPHOLE RULES-----------------------------------------------------
17419 // These must follow all instruction definitions as they use the names
17420 // defined in the instructions definitions.
17421 //
17422 // peepmatch ( root_instr_name [preceding_instruction]* );
17423 //
17424 // peepconstraint %{
17425 // (instruction_number.operand_name relational_op instruction_number.operand_name
17426 // [, ...] );
17427 // // instruction numbers are zero-based using left to right order in peepmatch
17428 //
17429 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17430 // // provide an instruction_number.operand_name for each operand that appears
17431 // // in the replacement instruction's match rule
17432 //
17433 // ---------VM FLAGS---------------------------------------------------------
17434 //
17435 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17436 //
17437 // Each peephole rule is given an identifying number starting with zero and
17438 // increasing by one in the order seen by the parser. An individual peephole
17439 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17440 // on the command-line.
17441 //
17442 // ---------CURRENT LIMITATIONS----------------------------------------------
17443 //
17444 // Only match adjacent instructions in same basic block
17445 // Only equality constraints
17446 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17447 // Only one replacement instruction
17448 //
17449 // ---------EXAMPLE----------------------------------------------------------
17450 //
17451 // // pertinent parts of existing instructions in architecture description
17452 // instruct movI(iRegINoSp dst, iRegI src)
17453 // %{
17454 // match(Set dst (CopyI src));
17455 // %}
17456 //
17457 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17458 // %{
17459 // match(Set dst (AddI dst src));
17460 // effect(KILL cr);
17461 // %}
17462 //
17463 // // Change (inc mov) to lea
17464 // peephole %{
17465 // // increment preceded by register-register move
17466 // peepmatch ( incI_iReg movI );
17467 // // require that the destination register of the increment
17468 // // match the destination register of the move
17469 // peepconstraint ( 0.dst == 1.dst );
17470 // // construct a replacement instruction that sets
17471 // // the destination to ( move's source register + one )
17472 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17473 // %}
17474 //
17475
17476 // Implementation no longer uses movX instructions since
17477 // machine-independent system no longer uses CopyX nodes.
17478 //
17479 // peephole
17480 // %{
17481 // peepmatch (incI_iReg movI);
17482 // peepconstraint (0.dst == 1.dst);
17483 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17484 // %}
17485
17486 // peephole
17487 // %{
17488 // peepmatch (decI_iReg movI);
17489 // peepconstraint (0.dst == 1.dst);
17490 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17491 // %}
17492
17493 // peephole
17494 // %{
17495 // peepmatch (addI_iReg_imm movI);
17496 // peepconstraint (0.dst == 1.dst);
17497 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17498 // %}
17499
17500 // peephole
17501 // %{
17502 // peepmatch (incL_iReg movL);
17503 // peepconstraint (0.dst == 1.dst);
17504 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17505 // %}
17506
17507 // peephole
17508 // %{
17509 // peepmatch (decL_iReg movL);
17510 // peepconstraint (0.dst == 1.dst);
17511 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17512 // %}
17513
17514 // peephole
17515 // %{
17516 // peepmatch (addL_iReg_imm movL);
17517 // peepconstraint (0.dst == 1.dst);
17518 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17519 // %}
17520
17521 // peephole
17522 // %{
17523 // peepmatch (addP_iReg_imm movP);
17524 // peepconstraint (0.dst == 1.dst);
17525 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17526 // %}
17527
17528 // // Change load of spilled value to only a spill
17529 // instruct storeI(memory mem, iRegI src)
17530 // %{
17531 // match(Set mem (StoreI mem src));
17532 // %}
17533 //
17534 // instruct loadI(iRegINoSp dst, memory mem)
17535 // %{
17536 // match(Set dst (LoadI mem));
17537 // %}
17538 //
17539
17540 //----------SMARTSPILL RULES---------------------------------------------------
17541 // These must follow all instruction definitions as they use the names
17542 // defined in the instructions definitions.
17543
17544 // Local Variables:
17545 // mode: c++
17546 // End: