1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for vector register V10
885 reg_class v10_veca_reg(
886 V10, V10_H, V10_J, V10_K
887 );
888
889 // Class for vector register V11
890 reg_class v11_veca_reg(
891 V11, V11_H, V11_J, V11_K
892 );
893
894 // Class for vector register V12
895 reg_class v12_veca_reg(
896 V12, V12_H, V12_J, V12_K
897 );
898
899 // Class for vector register V13
900 reg_class v13_veca_reg(
901 V13, V13_H, V13_J, V13_K
902 );
903
904 // Class for vector register V17
905 reg_class v17_veca_reg(
906 V17, V17_H, V17_J, V17_K
907 );
908
909 // Class for vector register V18
910 reg_class v18_veca_reg(
911 V18, V18_H, V18_J, V18_K
912 );
913
914 // Class for vector register V23
915 reg_class v23_veca_reg(
916 V23, V23_H, V23_J, V23_K
917 );
918
919 // Class for vector register V24
920 reg_class v24_veca_reg(
921 V24, V24_H, V24_J, V24_K
922 );
923
924 // Class for 128 bit register v0
925 reg_class v0_reg(
926 V0, V0_H
927 );
928
929 // Class for 128 bit register v1
930 reg_class v1_reg(
931 V1, V1_H
932 );
933
934 // Class for 128 bit register v2
935 reg_class v2_reg(
936 V2, V2_H
937 );
938
939 // Class for 128 bit register v3
940 reg_class v3_reg(
941 V3, V3_H
942 );
943
944 // Class for 128 bit register v4
945 reg_class v4_reg(
946 V4, V4_H
947 );
948
949 // Class for 128 bit register v5
950 reg_class v5_reg(
951 V5, V5_H
952 );
953
954 // Class for 128 bit register v6
955 reg_class v6_reg(
956 V6, V6_H
957 );
958
959 // Class for 128 bit register v7
960 reg_class v7_reg(
961 V7, V7_H
962 );
963
964 // Class for 128 bit register v8
965 reg_class v8_reg(
966 V8, V8_H
967 );
968
969 // Class for 128 bit register v9
970 reg_class v9_reg(
971 V9, V9_H
972 );
973
974 // Class for 128 bit register v10
975 reg_class v10_reg(
976 V10, V10_H
977 );
978
979 // Class for 128 bit register v11
980 reg_class v11_reg(
981 V11, V11_H
982 );
983
984 // Class for 128 bit register v12
985 reg_class v12_reg(
986 V12, V12_H
987 );
988
989 // Class for 128 bit register v13
990 reg_class v13_reg(
991 V13, V13_H
992 );
993
994 // Class for 128 bit register v14
995 reg_class v14_reg(
996 V14, V14_H
997 );
998
999 // Class for 128 bit register v15
1000 reg_class v15_reg(
1001 V15, V15_H
1002 );
1003
1004 // Class for 128 bit register v16
1005 reg_class v16_reg(
1006 V16, V16_H
1007 );
1008
1009 // Class for 128 bit register v17
1010 reg_class v17_reg(
1011 V17, V17_H
1012 );
1013
1014 // Class for 128 bit register v18
1015 reg_class v18_reg(
1016 V18, V18_H
1017 );
1018
1019 // Class for 128 bit register v19
1020 reg_class v19_reg(
1021 V19, V19_H
1022 );
1023
1024 // Class for 128 bit register v20
1025 reg_class v20_reg(
1026 V20, V20_H
1027 );
1028
1029 // Class for 128 bit register v21
1030 reg_class v21_reg(
1031 V21, V21_H
1032 );
1033
1034 // Class for 128 bit register v22
1035 reg_class v22_reg(
1036 V22, V22_H
1037 );
1038
1039 // Class for 128 bit register v23
1040 reg_class v23_reg(
1041 V23, V23_H
1042 );
1043
1044 // Class for 128 bit register v24
1045 reg_class v24_reg(
1046 V24, V24_H
1047 );
1048
1049 // Class for 128 bit register v25
1050 reg_class v25_reg(
1051 V25, V25_H
1052 );
1053
1054 // Class for 128 bit register v26
1055 reg_class v26_reg(
1056 V26, V26_H
1057 );
1058
1059 // Class for 128 bit register v27
1060 reg_class v27_reg(
1061 V27, V27_H
1062 );
1063
1064 // Class for 128 bit register v28
1065 reg_class v28_reg(
1066 V28, V28_H
1067 );
1068
1069 // Class for 128 bit register v29
1070 reg_class v29_reg(
1071 V29, V29_H
1072 );
1073
1074 // Class for 128 bit register v30
1075 reg_class v30_reg(
1076 V30, V30_H
1077 );
1078
1079 // Class for 128 bit register v31
1080 reg_class v31_reg(
1081 V31, V31_H
1082 );
1083
1084 // Class for all SVE predicate registers.
1085 reg_class pr_reg (
1086 P0,
1087 P1,
1088 P2,
1089 P3,
1090 P4,
1091 P5,
1092 P6,
1093 // P7, non-allocatable, preserved with all elements preset to TRUE.
1094 P8,
1095 P9,
1096 P10,
1097 P11,
1098 P12,
1099 P13,
1100 P14,
1101 P15
1102 );
1103
1104 // Class for SVE governing predicate registers, which are used
1105 // to determine the active elements of a predicated instruction.
1106 reg_class gov_pr (
1107 P0,
1108 P1,
1109 P2,
1110 P3,
1111 P4,
1112 P5,
1113 P6,
1114 // P7, non-allocatable, preserved with all elements preset to TRUE.
1115 );
1116
1117 reg_class p0_reg(P0);
1118 reg_class p1_reg(P1);
1119
1120 // Singleton class for condition codes
1121 reg_class int_flags(RFLAGS);
1122
1123 %}
1124
1125 //----------DEFINITION BLOCK---------------------------------------------------
1126 // Define name --> value mappings to inform the ADLC of an integer valued name
1127 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1128 // Format:
1129 // int_def <name> ( <int_value>, <expression>);
1130 // Generated Code in ad_<arch>.hpp
1131 // #define <name> (<expression>)
1132 // // value == <int_value>
1133 // Generated code in ad_<arch>.cpp adlc_verification()
1134 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1135 //
1136
1137 // we follow the ppc-aix port in using a simple cost model which ranks
1138 // register operations as cheap, memory ops as more expensive and
1139 // branches as most expensive. the first two have a low as well as a
1140 // normal cost. huge cost appears to be a way of saying don't do
1141 // something
1142
1143 definitions %{
1144 // The default cost (of a register move instruction).
1145 int_def INSN_COST ( 100, 100);
1146 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1147 int_def CALL_COST ( 200, 2 * INSN_COST);
1148 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1149 %}
1150
1151
1152 //----------SOURCE BLOCK-------------------------------------------------------
1153 // This is a block of C++ code which provides values, functions, and
1154 // definitions necessary in the rest of the architecture description
1155
1156 source_hpp %{
1157
1158 #include "asm/macroAssembler.hpp"
1159 #include "gc/shared/barrierSetAssembler.hpp"
1160 #include "gc/shared/cardTable.hpp"
1161 #include "gc/shared/cardTableBarrierSet.hpp"
1162 #include "gc/shared/collectedHeap.hpp"
1163 #include "opto/addnode.hpp"
1164 #include "opto/convertnode.hpp"
1165 #include "runtime/objectMonitor.hpp"
1166
1167 extern RegMask _ANY_REG32_mask;
1168 extern RegMask _ANY_REG_mask;
1169 extern RegMask _PTR_REG_mask;
1170 extern RegMask _NO_SPECIAL_REG32_mask;
1171 extern RegMask _NO_SPECIAL_REG_mask;
1172 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1173 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1174
1175 class CallStubImpl {
1176
1177 //--------------------------------------------------------------
1178 //---< Used for optimization in Compile::shorten_branches >---
1179 //--------------------------------------------------------------
1180
1181 public:
1182 // Size of call trampoline stub.
1183 static uint size_call_trampoline() {
1184 return 0; // no call trampolines on this platform
1185 }
1186
1187 // number of relocations needed by a call trampoline stub
1188 static uint reloc_call_trampoline() {
1189 return 0; // no call trampolines on this platform
1190 }
1191 };
1192
1193 class HandlerImpl {
1194
1195 public:
1196
1197 static int emit_exception_handler(C2_MacroAssembler *masm);
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_exception_handler() {
1201 return MacroAssembler::far_codestub_branch_size();
1202 }
1203
1204 static uint size_deopt_handler() {
1205 // count one adr and one far branch instruction
1206 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1207 }
1208 };
1209
1210 class Node::PD {
1211 public:
1212 enum NodeFlags {
1213 _last_flag = Node::_last_flag
1214 };
1215 };
1216
1217 bool is_CAS(int opcode, bool maybe_volatile);
1218
1219 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1220
1221 bool unnecessary_acquire(const Node *barrier);
1222 bool needs_acquiring_load(const Node *load);
1223
1224 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1225
1226 bool unnecessary_release(const Node *barrier);
1227 bool unnecessary_volatile(const Node *barrier);
1228 bool needs_releasing_store(const Node *store);
1229
1230 // predicate controlling translation of CompareAndSwapX
1231 bool needs_acquiring_load_exclusive(const Node *load);
1232
1233 // predicate controlling addressing modes
1234 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1235
1236 // Convert BootTest condition to Assembler condition.
1237 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1238 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1239 %}
1240
1241 source %{
1242
1243 // Derived RegMask with conditionally allocatable registers
1244
1245 void PhaseOutput::pd_perform_mach_node_analysis() {
1246 }
1247
1248 int MachNode::pd_alignment_required() const {
1249 return 1;
1250 }
1251
1252 int MachNode::compute_padding(int current_offset) const {
1253 return 0;
1254 }
1255
1256 RegMask _ANY_REG32_mask;
1257 RegMask _ANY_REG_mask;
1258 RegMask _PTR_REG_mask;
1259 RegMask _NO_SPECIAL_REG32_mask;
1260 RegMask _NO_SPECIAL_REG_mask;
1261 RegMask _NO_SPECIAL_PTR_REG_mask;
1262 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1263
1264 void reg_mask_init() {
1265 // We derive below RegMask(s) from the ones which are auto-generated from
1266 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1267 // registers conditionally reserved.
1268
1269 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1270 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1271
1272 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1273
1274 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1275
1276 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1277 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1278
1279 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1280 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1281
1282 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1283 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1284
1285 // r27 is not allocatable when compressed oops is on and heapbase is not
1286 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1287 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1288 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1289 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1290 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1291 }
1292
1293 // r29 is not allocatable when PreserveFramePointer is on
1294 if (PreserveFramePointer) {
1295 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1296 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1297 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1301 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1302 }
1303
1304 // Optimizaton of volatile gets and puts
1305 // -------------------------------------
1306 //
1307 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1308 // use to implement volatile reads and writes. For a volatile read
1309 // we simply need
1310 //
1311 // ldar<x>
1312 //
1313 // and for a volatile write we need
1314 //
1315 // stlr<x>
1316 //
1317 // Alternatively, we can implement them by pairing a normal
1318 // load/store with a memory barrier. For a volatile read we need
1319 //
1320 // ldr<x>
1321 // dmb ishld
1322 //
1323 // for a volatile write
1324 //
1325 // dmb ish
1326 // str<x>
1327 // dmb ish
1328 //
1329 // We can also use ldaxr and stlxr to implement compare and swap CAS
1330 // sequences. These are normally translated to an instruction
1331 // sequence like the following
1332 //
1333 // dmb ish
1334 // retry:
1335 // ldxr<x> rval raddr
1336 // cmp rval rold
1337 // b.ne done
1338 // stlxr<x> rval, rnew, rold
1339 // cbnz rval retry
1340 // done:
1341 // cset r0, eq
1342 // dmb ishld
1343 //
1344 // Note that the exclusive store is already using an stlxr
1345 // instruction. That is required to ensure visibility to other
1346 // threads of the exclusive write (assuming it succeeds) before that
1347 // of any subsequent writes.
1348 //
1349 // The following instruction sequence is an improvement on the above
1350 //
1351 // retry:
1352 // ldaxr<x> rval raddr
1353 // cmp rval rold
1354 // b.ne done
1355 // stlxr<x> rval, rnew, rold
1356 // cbnz rval retry
1357 // done:
1358 // cset r0, eq
1359 //
1360 // We don't need the leading dmb ish since the stlxr guarantees
1361 // visibility of prior writes in the case that the swap is
1362 // successful. Crucially we don't have to worry about the case where
1363 // the swap is not successful since no valid program should be
1364 // relying on visibility of prior changes by the attempting thread
1365 // in the case where the CAS fails.
1366 //
1367 // Similarly, we don't need the trailing dmb ishld if we substitute
1368 // an ldaxr instruction since that will provide all the guarantees we
1369 // require regarding observation of changes made by other threads
1370 // before any change to the CAS address observed by the load.
1371 //
1372 // In order to generate the desired instruction sequence we need to
1373 // be able to identify specific 'signature' ideal graph node
1374 // sequences which i) occur as a translation of a volatile reads or
1375 // writes or CAS operations and ii) do not occur through any other
1376 // translation or graph transformation. We can then provide
1377 // alternative aldc matching rules which translate these node
1378 // sequences to the desired machine code sequences. Selection of the
1379 // alternative rules can be implemented by predicates which identify
1380 // the relevant node sequences.
1381 //
1382 // The ideal graph generator translates a volatile read to the node
1383 // sequence
1384 //
1385 // LoadX[mo_acquire]
1386 // MemBarAcquire
1387 //
1388 // As a special case when using the compressed oops optimization we
1389 // may also see this variant
1390 //
1391 // LoadN[mo_acquire]
1392 // DecodeN
1393 // MemBarAcquire
1394 //
1395 // A volatile write is translated to the node sequence
1396 //
1397 // MemBarRelease
1398 // StoreX[mo_release] {CardMark}-optional
1399 // MemBarVolatile
1400 //
1401 // n.b. the above node patterns are generated with a strict
1402 // 'signature' configuration of input and output dependencies (see
1403 // the predicates below for exact details). The card mark may be as
1404 // simple as a few extra nodes or, in a few GC configurations, may
1405 // include more complex control flow between the leading and
1406 // trailing memory barriers. However, whatever the card mark
1407 // configuration these signatures are unique to translated volatile
1408 // reads/stores -- they will not appear as a result of any other
1409 // bytecode translation or inlining nor as a consequence of
1410 // optimizing transforms.
1411 //
1412 // We also want to catch inlined unsafe volatile gets and puts and
1413 // be able to implement them using either ldar<x>/stlr<x> or some
1414 // combination of ldr<x>/stlr<x> and dmb instructions.
1415 //
1416 // Inlined unsafe volatiles puts manifest as a minor variant of the
1417 // normal volatile put node sequence containing an extra cpuorder
1418 // membar
1419 //
1420 // MemBarRelease
1421 // MemBarCPUOrder
1422 // StoreX[mo_release] {CardMark}-optional
1423 // MemBarCPUOrder
1424 // MemBarVolatile
1425 //
1426 // n.b. as an aside, a cpuorder membar is not itself subject to
1427 // matching and translation by adlc rules. However, the rule
1428 // predicates need to detect its presence in order to correctly
1429 // select the desired adlc rules.
1430 //
1431 // Inlined unsafe volatile gets manifest as a slightly different
1432 // node sequence to a normal volatile get because of the
1433 // introduction of some CPUOrder memory barriers to bracket the
1434 // Load. However, but the same basic skeleton of a LoadX feeding a
1435 // MemBarAcquire, possibly through an optional DecodeN, is still
1436 // present
1437 //
1438 // MemBarCPUOrder
1439 // || \\
1440 // MemBarCPUOrder LoadX[mo_acquire]
1441 // || |
1442 // || {DecodeN} optional
1443 // || /
1444 // MemBarAcquire
1445 //
1446 // In this case the acquire membar does not directly depend on the
1447 // load. However, we can be sure that the load is generated from an
1448 // inlined unsafe volatile get if we see it dependent on this unique
1449 // sequence of membar nodes. Similarly, given an acquire membar we
1450 // can know that it was added because of an inlined unsafe volatile
1451 // get if it is fed and feeds a cpuorder membar and if its feed
1452 // membar also feeds an acquiring load.
1453 //
1454 // Finally an inlined (Unsafe) CAS operation is translated to the
1455 // following ideal graph
1456 //
1457 // MemBarRelease
1458 // MemBarCPUOrder
1459 // CompareAndSwapX {CardMark}-optional
1460 // MemBarCPUOrder
1461 // MemBarAcquire
1462 //
1463 // So, where we can identify these volatile read and write
1464 // signatures we can choose to plant either of the above two code
1465 // sequences. For a volatile read we can simply plant a normal
1466 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1467 // also choose to inhibit translation of the MemBarAcquire and
1468 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1469 //
1470 // When we recognise a volatile store signature we can choose to
1471 // plant at a dmb ish as a translation for the MemBarRelease, a
1472 // normal str<x> and then a dmb ish for the MemBarVolatile.
1473 // Alternatively, we can inhibit translation of the MemBarRelease
1474 // and MemBarVolatile and instead plant a simple stlr<x>
1475 // instruction.
1476 //
1477 // when we recognise a CAS signature we can choose to plant a dmb
1478 // ish as a translation for the MemBarRelease, the conventional
1479 // macro-instruction sequence for the CompareAndSwap node (which
1480 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1481 // Alternatively, we can elide generation of the dmb instructions
1482 // and plant the alternative CompareAndSwap macro-instruction
1483 // sequence (which uses ldaxr<x>).
1484 //
1485 // Of course, the above only applies when we see these signature
1486 // configurations. We still want to plant dmb instructions in any
1487 // other cases where we may see a MemBarAcquire, MemBarRelease or
1488 // MemBarVolatile. For example, at the end of a constructor which
1489 // writes final/volatile fields we will see a MemBarRelease
1490 // instruction and this needs a 'dmb ish' lest we risk the
1491 // constructed object being visible without making the
1492 // final/volatile field writes visible.
1493 //
1494 // n.b. the translation rules below which rely on detection of the
1495 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1496 // If we see anything other than the signature configurations we
1497 // always just translate the loads and stores to ldr<x> and str<x>
1498 // and translate acquire, release and volatile membars to the
1499 // relevant dmb instructions.
1500 //
1501
1502 // is_CAS(int opcode, bool maybe_volatile)
1503 //
1504 // return true if opcode is one of the possible CompareAndSwapX
1505 // values otherwise false.
1506
1507 bool is_CAS(int opcode, bool maybe_volatile)
1508 {
1509 switch(opcode) {
1510 // We handle these
1511 case Op_CompareAndSwapI:
1512 case Op_CompareAndSwapL:
1513 case Op_CompareAndSwapP:
1514 case Op_CompareAndSwapN:
1515 case Op_CompareAndSwapB:
1516 case Op_CompareAndSwapS:
1517 case Op_GetAndSetI:
1518 case Op_GetAndSetL:
1519 case Op_GetAndSetP:
1520 case Op_GetAndSetN:
1521 case Op_GetAndAddI:
1522 case Op_GetAndAddL:
1523 return true;
1524 case Op_CompareAndExchangeI:
1525 case Op_CompareAndExchangeN:
1526 case Op_CompareAndExchangeB:
1527 case Op_CompareAndExchangeS:
1528 case Op_CompareAndExchangeL:
1529 case Op_CompareAndExchangeP:
1530 case Op_WeakCompareAndSwapB:
1531 case Op_WeakCompareAndSwapS:
1532 case Op_WeakCompareAndSwapI:
1533 case Op_WeakCompareAndSwapL:
1534 case Op_WeakCompareAndSwapP:
1535 case Op_WeakCompareAndSwapN:
1536 return maybe_volatile;
1537 default:
1538 return false;
1539 }
1540 }
1541
1542 // helper to determine the maximum number of Phi nodes we may need to
1543 // traverse when searching from a card mark membar for the merge mem
1544 // feeding a trailing membar or vice versa
1545
1546 // predicates controlling emit of ldr<x>/ldar<x>
1547
1548 bool unnecessary_acquire(const Node *barrier)
1549 {
1550 assert(barrier->is_MemBar(), "expecting a membar");
1551
1552 MemBarNode* mb = barrier->as_MemBar();
1553
1554 if (mb->trailing_load()) {
1555 return true;
1556 }
1557
1558 if (mb->trailing_load_store()) {
1559 Node* load_store = mb->in(MemBarNode::Precedent);
1560 assert(load_store->is_LoadStore(), "unexpected graph shape");
1561 return is_CAS(load_store->Opcode(), true);
1562 }
1563
1564 return false;
1565 }
1566
1567 bool needs_acquiring_load(const Node *n)
1568 {
1569 assert(n->is_Load(), "expecting a load");
1570 LoadNode *ld = n->as_Load();
1571 return ld->is_acquire();
1572 }
1573
1574 bool unnecessary_release(const Node *n)
1575 {
1576 assert((n->is_MemBar() &&
1577 n->Opcode() == Op_MemBarRelease),
1578 "expecting a release membar");
1579
1580 MemBarNode *barrier = n->as_MemBar();
1581 if (!barrier->leading()) {
1582 return false;
1583 } else {
1584 Node* trailing = barrier->trailing_membar();
1585 MemBarNode* trailing_mb = trailing->as_MemBar();
1586 assert(trailing_mb->trailing(), "Not a trailing membar?");
1587 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1588
1589 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1590 if (mem->is_Store()) {
1591 assert(mem->as_Store()->is_release(), "");
1592 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1593 return true;
1594 } else {
1595 assert(mem->is_LoadStore(), "");
1596 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1597 return is_CAS(mem->Opcode(), true);
1598 }
1599 }
1600 return false;
1601 }
1602
1603 bool unnecessary_volatile(const Node *n)
1604 {
1605 // assert n->is_MemBar();
1606 MemBarNode *mbvol = n->as_MemBar();
1607
1608 bool release = mbvol->trailing_store();
1609 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1610 #ifdef ASSERT
1611 if (release) {
1612 Node* leading = mbvol->leading_membar();
1613 assert(leading->Opcode() == Op_MemBarRelease, "");
1614 assert(leading->as_MemBar()->leading_store(), "");
1615 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1616 }
1617 #endif
1618
1619 return release;
1620 }
1621
1622 // predicates controlling emit of str<x>/stlr<x>
1623
1624 bool needs_releasing_store(const Node *n)
1625 {
1626 // assert n->is_Store();
1627 StoreNode *st = n->as_Store();
1628 return st->trailing_membar() != nullptr;
1629 }
1630
1631 // predicate controlling translation of CAS
1632 //
1633 // returns true if CAS needs to use an acquiring load otherwise false
1634
1635 bool needs_acquiring_load_exclusive(const Node *n)
1636 {
1637 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1638 LoadStoreNode* ldst = n->as_LoadStore();
1639 if (is_CAS(n->Opcode(), false)) {
1640 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1641 } else {
1642 return ldst->trailing_membar() != nullptr;
1643 }
1644
1645 // so we can just return true here
1646 return true;
1647 }
1648
1649 #define __ masm->
1650
1651 // advance declarations for helper functions to convert register
1652 // indices to register objects
1653
1654 // the ad file has to provide implementations of certain methods
1655 // expected by the generic code
1656 //
1657 // REQUIRED FUNCTIONALITY
1658
1659 //=============================================================================
1660
1661 // !!!!! Special hack to get all types of calls to specify the byte offset
1662 // from the start of the call to the point where the return address
1663 // will point.
1664
1665 int MachCallStaticJavaNode::ret_addr_offset()
1666 {
1667 // call should be a simple bl
1668 int off = 4;
1669 return off;
1670 }
1671
1672 int MachCallDynamicJavaNode::ret_addr_offset()
1673 {
1674 return 16; // movz, movk, movk, bl
1675 }
1676
1677 int MachCallRuntimeNode::ret_addr_offset() {
1678 // for generated stubs the call will be
1679 // bl(addr)
1680 // or with far branches
1681 // bl(trampoline_stub)
1682 // for real runtime callouts it will be six instructions
1683 // see aarch64_enc_java_to_runtime
1684 // adr(rscratch2, retaddr)
1685 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1686 // lea(rscratch1, RuntimeAddress(addr)
1687 // blr(rscratch1)
1688 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1689 if (cb) {
1690 return 1 * NativeInstruction::instruction_size;
1691 } else {
1692 return 6 * NativeInstruction::instruction_size;
1693 }
1694 }
1695
1696 //=============================================================================
1697
1698 #ifndef PRODUCT
1699 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1700 st->print("BREAKPOINT");
1701 }
1702 #endif
1703
1704 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1705 __ brk(0);
1706 }
1707
1708 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1709 return MachNode::size(ra_);
1710 }
1711
1712 //=============================================================================
1713
1714 #ifndef PRODUCT
1715 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1716 st->print("nop \t# %d bytes pad for loops and calls", _count);
1717 }
1718 #endif
1719
1720 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1721 for (int i = 0; i < _count; i++) {
1722 __ nop();
1723 }
1724 }
1725
1726 uint MachNopNode::size(PhaseRegAlloc*) const {
1727 return _count * NativeInstruction::instruction_size;
1728 }
1729
1730 //=============================================================================
1731 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1732
1733 int ConstantTable::calculate_table_base_offset() const {
1734 return 0; // absolute addressing, no offset
1735 }
1736
1737 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1738 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1739 ShouldNotReachHere();
1740 }
1741
1742 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1743 // Empty encoding
1744 }
1745
1746 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1747 return 0;
1748 }
1749
1750 #ifndef PRODUCT
1751 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1752 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1753 }
1754 #endif
1755
1756 #ifndef PRODUCT
1757 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1758 Compile* C = ra_->C;
1759
1760 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1761
1762 if (C->output()->need_stack_bang(framesize))
1763 st->print("# stack bang size=%d\n\t", framesize);
1764
1765 if (VM_Version::use_rop_protection()) {
1766 st->print("ldr zr, [lr]\n\t");
1767 st->print("paciaz\n\t");
1768 }
1769 if (framesize < ((1 << 9) + 2 * wordSize)) {
1770 st->print("sub sp, sp, #%d\n\t", framesize);
1771 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1772 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1773 } else {
1774 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1775 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1776 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1777 st->print("sub sp, sp, rscratch1");
1778 }
1779 if (C->stub_function() == nullptr) {
1780 st->print("\n\t");
1781 st->print("ldr rscratch1, [guard]\n\t");
1782 st->print("dmb ishld\n\t");
1783 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1784 st->print("cmp rscratch1, rscratch2\n\t");
1785 st->print("b.eq skip");
1786 st->print("\n\t");
1787 st->print("blr #nmethod_entry_barrier_stub\n\t");
1788 st->print("b skip\n\t");
1789 st->print("guard: int\n\t");
1790 st->print("\n\t");
1791 st->print("skip:\n\t");
1792 }
1793 }
1794 #endif
1795
1796 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1797 Compile* C = ra_->C;
1798
1799 // n.b. frame size includes space for return pc and rfp
1800 const int framesize = C->output()->frame_size_in_bytes();
1801
1802 if (C->clinit_barrier_on_entry()) {
1803 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1804
1805 Label L_skip_barrier;
1806
1807 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1808 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1809 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1810 __ bind(L_skip_barrier);
1811 }
1812
1813 if (C->max_vector_size() > 0) {
1814 __ reinitialize_ptrue();
1815 }
1816
1817 int bangsize = C->output()->bang_size_in_bytes();
1818 if (C->output()->need_stack_bang(bangsize))
1819 __ generate_stack_overflow_check(bangsize);
1820
1821 __ build_frame(framesize);
1822
1823 if (C->stub_function() == nullptr) {
1824 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1825 // Dummy labels for just measuring the code size
1826 Label dummy_slow_path;
1827 Label dummy_continuation;
1828 Label dummy_guard;
1829 Label* slow_path = &dummy_slow_path;
1830 Label* continuation = &dummy_continuation;
1831 Label* guard = &dummy_guard;
1832 if (!Compile::current()->output()->in_scratch_emit_size()) {
1833 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1834 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1835 Compile::current()->output()->add_stub(stub);
1836 slow_path = &stub->entry();
1837 continuation = &stub->continuation();
1838 guard = &stub->guard();
1839 }
1840 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1841 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1842 }
1843
1844 if (VerifyStackAtCalls) {
1845 Unimplemented();
1846 }
1847
1848 C->output()->set_frame_complete(__ offset());
1849
1850 if (C->has_mach_constant_base_node()) {
1851 // NOTE: We set the table base offset here because users might be
1852 // emitted before MachConstantBaseNode.
1853 ConstantTable& constant_table = C->output()->constant_table();
1854 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1855 }
1856 }
1857
1858 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1859 {
1860 return MachNode::size(ra_); // too many variables; just compute it
1861 // the hard way
1862 }
1863
1864 int MachPrologNode::reloc() const
1865 {
1866 return 0;
1867 }
1868
1869 //=============================================================================
1870
1871 #ifndef PRODUCT
1872 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1873 Compile* C = ra_->C;
1874 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1875
1876 st->print("# pop frame %d\n\t",framesize);
1877
1878 if (framesize == 0) {
1879 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1880 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1881 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1882 st->print("add sp, sp, #%d\n\t", framesize);
1883 } else {
1884 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1885 st->print("add sp, sp, rscratch1\n\t");
1886 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1887 }
1888 if (VM_Version::use_rop_protection()) {
1889 st->print("autiaz\n\t");
1890 st->print("ldr zr, [lr]\n\t");
1891 }
1892
1893 if (do_polling() && C->is_method_compilation()) {
1894 st->print("# test polling word\n\t");
1895 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1896 st->print("cmp sp, rscratch1\n\t");
1897 st->print("bhi #slow_path");
1898 }
1899 }
1900 #endif
1901
1902 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1903 Compile* C = ra_->C;
1904 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1905
1906 __ remove_frame(framesize);
1907
1908 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1909 __ reserved_stack_check();
1910 }
1911
1912 if (do_polling() && C->is_method_compilation()) {
1913 Label dummy_label;
1914 Label* code_stub = &dummy_label;
1915 if (!C->output()->in_scratch_emit_size()) {
1916 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1917 C->output()->add_stub(stub);
1918 code_stub = &stub->entry();
1919 }
1920 __ relocate(relocInfo::poll_return_type);
1921 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1922 }
1923 }
1924
1925 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1926 // Variable size. Determine dynamically.
1927 return MachNode::size(ra_);
1928 }
1929
1930 int MachEpilogNode::reloc() const {
1931 // Return number of relocatable values contained in this instruction.
1932 return 1; // 1 for polling page.
1933 }
1934
1935 const Pipeline * MachEpilogNode::pipeline() const {
1936 return MachNode::pipeline_class();
1937 }
1938
1939 //=============================================================================
1940
1941 static enum RC rc_class(OptoReg::Name reg) {
1942
1943 if (reg == OptoReg::Bad) {
1944 return rc_bad;
1945 }
1946
1947 // we have 32 int registers * 2 halves
1948 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1949
1950 if (reg < slots_of_int_registers) {
1951 return rc_int;
1952 }
1953
1954 // we have 32 float register * 8 halves
1955 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1956 if (reg < slots_of_int_registers + slots_of_float_registers) {
1957 return rc_float;
1958 }
1959
1960 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1961 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1962 return rc_predicate;
1963 }
1964
1965 // Between predicate regs & stack is the flags.
1966 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1967
1968 return rc_stack;
1969 }
1970
1971 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1972 Compile* C = ra_->C;
1973
1974 // Get registers to move.
1975 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1976 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1977 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1978 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1979
1980 enum RC src_hi_rc = rc_class(src_hi);
1981 enum RC src_lo_rc = rc_class(src_lo);
1982 enum RC dst_hi_rc = rc_class(dst_hi);
1983 enum RC dst_lo_rc = rc_class(dst_lo);
1984
1985 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1986
1987 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1988 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1989 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1990 "expected aligned-adjacent pairs");
1991 }
1992
1993 if (src_lo == dst_lo && src_hi == dst_hi) {
1994 return 0; // Self copy, no move.
1995 }
1996
1997 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1998 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1999 int src_offset = ra_->reg2offset(src_lo);
2000 int dst_offset = ra_->reg2offset(dst_lo);
2001
2002 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2003 uint ireg = ideal_reg();
2004 if (ireg == Op_VecA && masm) {
2005 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2006 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2007 // stack->stack
2008 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2009 sve_vector_reg_size_in_bytes);
2010 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2011 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2012 sve_vector_reg_size_in_bytes);
2013 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2014 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2015 sve_vector_reg_size_in_bytes);
2016 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2017 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2018 as_FloatRegister(Matcher::_regEncode[src_lo]),
2019 as_FloatRegister(Matcher::_regEncode[src_lo]));
2020 } else {
2021 ShouldNotReachHere();
2022 }
2023 } else if (masm) {
2024 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2025 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2026 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2027 // stack->stack
2028 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2029 if (ireg == Op_VecD) {
2030 __ unspill(rscratch1, true, src_offset);
2031 __ spill(rscratch1, true, dst_offset);
2032 } else {
2033 __ spill_copy128(src_offset, dst_offset);
2034 }
2035 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2036 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2037 ireg == Op_VecD ? __ T8B : __ T16B,
2038 as_FloatRegister(Matcher::_regEncode[src_lo]));
2039 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2040 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2041 ireg == Op_VecD ? __ D : __ Q,
2042 ra_->reg2offset(dst_lo));
2043 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2044 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2045 ireg == Op_VecD ? __ D : __ Q,
2046 ra_->reg2offset(src_lo));
2047 } else {
2048 ShouldNotReachHere();
2049 }
2050 }
2051 } else if (masm) {
2052 switch (src_lo_rc) {
2053 case rc_int:
2054 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2055 if (is64) {
2056 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2057 as_Register(Matcher::_regEncode[src_lo]));
2058 } else {
2059 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2060 as_Register(Matcher::_regEncode[src_lo]));
2061 }
2062 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2063 if (is64) {
2064 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2065 as_Register(Matcher::_regEncode[src_lo]));
2066 } else {
2067 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2068 as_Register(Matcher::_regEncode[src_lo]));
2069 }
2070 } else { // gpr --> stack spill
2071 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2072 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2073 }
2074 break;
2075 case rc_float:
2076 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2077 if (is64) {
2078 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2079 as_FloatRegister(Matcher::_regEncode[src_lo]));
2080 } else {
2081 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2082 as_FloatRegister(Matcher::_regEncode[src_lo]));
2083 }
2084 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2085 if (is64) {
2086 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2087 as_FloatRegister(Matcher::_regEncode[src_lo]));
2088 } else {
2089 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2090 as_FloatRegister(Matcher::_regEncode[src_lo]));
2091 }
2092 } else { // fpr --> stack spill
2093 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2094 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2095 is64 ? __ D : __ S, dst_offset);
2096 }
2097 break;
2098 case rc_stack:
2099 if (dst_lo_rc == rc_int) { // stack --> gpr load
2100 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2101 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2102 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2103 is64 ? __ D : __ S, src_offset);
2104 } else if (dst_lo_rc == rc_predicate) {
2105 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2106 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2107 } else { // stack --> stack copy
2108 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2109 if (ideal_reg() == Op_RegVectMask) {
2110 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2111 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2112 } else {
2113 __ unspill(rscratch1, is64, src_offset);
2114 __ spill(rscratch1, is64, dst_offset);
2115 }
2116 }
2117 break;
2118 case rc_predicate:
2119 if (dst_lo_rc == rc_predicate) {
2120 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2121 } else if (dst_lo_rc == rc_stack) {
2122 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2123 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2124 } else {
2125 assert(false, "bad src and dst rc_class combination.");
2126 ShouldNotReachHere();
2127 }
2128 break;
2129 default:
2130 assert(false, "bad rc_class for spill");
2131 ShouldNotReachHere();
2132 }
2133 }
2134
2135 if (st) {
2136 st->print("spill ");
2137 if (src_lo_rc == rc_stack) {
2138 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2139 } else {
2140 st->print("%s -> ", Matcher::regName[src_lo]);
2141 }
2142 if (dst_lo_rc == rc_stack) {
2143 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2144 } else {
2145 st->print("%s", Matcher::regName[dst_lo]);
2146 }
2147 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2148 int vsize = 0;
2149 switch (ideal_reg()) {
2150 case Op_VecD:
2151 vsize = 64;
2152 break;
2153 case Op_VecX:
2154 vsize = 128;
2155 break;
2156 case Op_VecA:
2157 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2158 break;
2159 default:
2160 assert(false, "bad register type for spill");
2161 ShouldNotReachHere();
2162 }
2163 st->print("\t# vector spill size = %d", vsize);
2164 } else if (ideal_reg() == Op_RegVectMask) {
2165 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2166 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2167 st->print("\t# predicate spill size = %d", vsize);
2168 } else {
2169 st->print("\t# spill size = %d", is64 ? 64 : 32);
2170 }
2171 }
2172
2173 return 0;
2174
2175 }
2176
2177 #ifndef PRODUCT
2178 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2179 if (!ra_)
2180 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2181 else
2182 implementation(nullptr, ra_, false, st);
2183 }
2184 #endif
2185
2186 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2187 implementation(masm, ra_, false, nullptr);
2188 }
2189
2190 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2191 return MachNode::size(ra_);
2192 }
2193
2194 //=============================================================================
2195
2196 #ifndef PRODUCT
2197 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2198 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2199 int reg = ra_->get_reg_first(this);
2200 st->print("add %s, rsp, #%d]\t# box lock",
2201 Matcher::regName[reg], offset);
2202 }
2203 #endif
2204
2205 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2206 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2207 int reg = ra_->get_encode(this);
2208
2209 // This add will handle any 24-bit signed offset. 24 bits allows an
2210 // 8 megabyte stack frame.
2211 __ add(as_Register(reg), sp, offset);
2212 }
2213
2214 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2215 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2216 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2217
2218 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2219 return NativeInstruction::instruction_size;
2220 } else {
2221 return 2 * NativeInstruction::instruction_size;
2222 }
2223 }
2224
2225 //=============================================================================
2226
2227 #ifndef PRODUCT
2228 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2229 {
2230 st->print_cr("# MachUEPNode");
2231 if (UseCompressedClassPointers) {
2232 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2233 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2234 st->print_cr("\tcmpw rscratch1, r10");
2235 } else {
2236 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2237 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2238 st->print_cr("\tcmp rscratch1, r10");
2239 }
2240 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2241 }
2242 #endif
2243
2244 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2245 {
2246 __ ic_check(InteriorEntryAlignment);
2247 }
2248
2249 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2250 {
2251 return MachNode::size(ra_);
2252 }
2253
2254 // REQUIRED EMIT CODE
2255
2256 //=============================================================================
2257
2258 // Emit exception handler code.
2259 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2260 {
2261 // mov rscratch1 #exception_blob_entry_point
2262 // br rscratch1
2263 // Note that the code buffer's insts_mark is always relative to insts.
2264 // That's why we must use the macroassembler to generate a handler.
2265 address base = __ start_a_stub(size_exception_handler());
2266 if (base == nullptr) {
2267 ciEnv::current()->record_failure("CodeCache is full");
2268 return 0; // CodeBuffer::expand failed
2269 }
2270 int offset = __ offset();
2271 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2272 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2273 __ end_a_stub();
2274 return offset;
2275 }
2276
2277 // Emit deopt handler code.
2278 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2279 {
2280 // Note that the code buffer's insts_mark is always relative to insts.
2281 // That's why we must use the macroassembler to generate a handler.
2282 address base = __ start_a_stub(size_deopt_handler());
2283 if (base == nullptr) {
2284 ciEnv::current()->record_failure("CodeCache is full");
2285 return 0; // CodeBuffer::expand failed
2286 }
2287 int offset = __ offset();
2288
2289 __ adr(lr, __ pc());
2290 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2291
2292 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2293 __ end_a_stub();
2294 return offset;
2295 }
2296
2297 // REQUIRED MATCHER CODE
2298
2299 //=============================================================================
2300
2301 bool Matcher::match_rule_supported(int opcode) {
2302 if (!has_match_rule(opcode))
2303 return false;
2304
2305 switch (opcode) {
2306 case Op_OnSpinWait:
2307 return VM_Version::supports_on_spin_wait();
2308 case Op_CacheWB:
2309 case Op_CacheWBPreSync:
2310 case Op_CacheWBPostSync:
2311 if (!VM_Version::supports_data_cache_line_flush()) {
2312 return false;
2313 }
2314 break;
2315 case Op_ExpandBits:
2316 case Op_CompressBits:
2317 if (!VM_Version::supports_svebitperm()) {
2318 return false;
2319 }
2320 break;
2321 case Op_FmaF:
2322 case Op_FmaD:
2323 case Op_FmaVF:
2324 case Op_FmaVD:
2325 if (!UseFMA) {
2326 return false;
2327 }
2328 break;
2329 case Op_FmaHF:
2330 // UseFMA flag also needs to be checked along with FEAT_FP16
2331 if (!UseFMA || !is_feat_fp16_supported()) {
2332 return false;
2333 }
2334 break;
2335 case Op_AddHF:
2336 case Op_SubHF:
2337 case Op_MulHF:
2338 case Op_DivHF:
2339 case Op_MinHF:
2340 case Op_MaxHF:
2341 case Op_SqrtHF:
2342 // Half-precision floating point scalar operations require FEAT_FP16
2343 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2344 // features are supported.
2345 if (!is_feat_fp16_supported()) {
2346 return false;
2347 }
2348 break;
2349 }
2350
2351 return true; // Per default match rules are supported.
2352 }
2353
2354 const RegMask* Matcher::predicate_reg_mask(void) {
2355 return &_PR_REG_mask;
2356 }
2357
2358 bool Matcher::supports_vector_calling_convention(void) {
2359 return EnableVectorSupport;
2360 }
2361
2362 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2363 assert(EnableVectorSupport, "sanity");
2364 int lo = V0_num;
2365 int hi = V0_H_num;
2366 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2367 hi = V0_K_num;
2368 }
2369 return OptoRegPair(hi, lo);
2370 }
2371
2372 // Is this branch offset short enough that a short branch can be used?
2373 //
2374 // NOTE: If the platform does not provide any short branch variants, then
2375 // this method should return false for offset 0.
2376 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2377 // The passed offset is relative to address of the branch.
2378
2379 return (-32768 <= offset && offset < 32768);
2380 }
2381
2382 // Vector width in bytes.
2383 int Matcher::vector_width_in_bytes(BasicType bt) {
2384 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2385 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2386 // Minimum 2 values in vector
2387 if (size < 2*type2aelembytes(bt)) size = 0;
2388 // But never < 4
2389 if (size < 4) size = 0;
2390 return size;
2391 }
2392
2393 // Limits on vector size (number of elements) loaded into vector.
2394 int Matcher::max_vector_size(const BasicType bt) {
2395 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2396 }
2397
2398 int Matcher::min_vector_size(const BasicType bt) {
2399 // Usually, the shortest vector length supported by AArch64 ISA and
2400 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2401 // vectors in a few special cases.
2402 int size;
2403 switch(bt) {
2404 case T_BOOLEAN:
2405 // Load/store a vector mask with only 2 elements for vector types
2406 // such as "2I/2F/2L/2D".
2407 size = 2;
2408 break;
2409 case T_BYTE:
2410 // Generate a "4B" vector, to support vector cast between "8B/16B"
2411 // and "4S/4I/4L/4F/4D".
2412 size = 4;
2413 break;
2414 case T_SHORT:
2415 // Generate a "2S" vector, to support vector cast between "4S/8S"
2416 // and "2I/2L/2F/2D".
2417 size = 2;
2418 break;
2419 default:
2420 // Limit the min vector length to 64-bit.
2421 size = 8 / type2aelembytes(bt);
2422 // The number of elements in a vector should be at least 2.
2423 size = MAX2(size, 2);
2424 }
2425
2426 int max_size = max_vector_size(bt);
2427 return MIN2(size, max_size);
2428 }
2429
2430 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2431 return Matcher::max_vector_size(bt);
2432 }
2433
2434 // Actual max scalable vector register length.
2435 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2436 return Matcher::max_vector_size(bt);
2437 }
2438
2439 // Vector ideal reg.
2440 uint Matcher::vector_ideal_reg(int len) {
2441 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2442 return Op_VecA;
2443 }
2444 switch(len) {
2445 // For 16-bit/32-bit mask vector, reuse VecD.
2446 case 2:
2447 case 4:
2448 case 8: return Op_VecD;
2449 case 16: return Op_VecX;
2450 }
2451 ShouldNotReachHere();
2452 return 0;
2453 }
2454
2455 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2456 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2457 switch (ideal_reg) {
2458 case Op_VecA: return new vecAOper();
2459 case Op_VecD: return new vecDOper();
2460 case Op_VecX: return new vecXOper();
2461 }
2462 ShouldNotReachHere();
2463 return nullptr;
2464 }
2465
2466 bool Matcher::is_reg2reg_move(MachNode* m) {
2467 return false;
2468 }
2469
2470 bool Matcher::is_generic_vector(MachOper* opnd) {
2471 return opnd->opcode() == VREG;
2472 }
2473
2474 // Return whether or not this register is ever used as an argument.
2475 // This function is used on startup to build the trampoline stubs in
2476 // generateOptoStub. Registers not mentioned will be killed by the VM
2477 // call in the trampoline, and arguments in those registers not be
2478 // available to the callee.
2479 bool Matcher::can_be_java_arg(int reg)
2480 {
2481 return
2482 reg == R0_num || reg == R0_H_num ||
2483 reg == R1_num || reg == R1_H_num ||
2484 reg == R2_num || reg == R2_H_num ||
2485 reg == R3_num || reg == R3_H_num ||
2486 reg == R4_num || reg == R4_H_num ||
2487 reg == R5_num || reg == R5_H_num ||
2488 reg == R6_num || reg == R6_H_num ||
2489 reg == R7_num || reg == R7_H_num ||
2490 reg == V0_num || reg == V0_H_num ||
2491 reg == V1_num || reg == V1_H_num ||
2492 reg == V2_num || reg == V2_H_num ||
2493 reg == V3_num || reg == V3_H_num ||
2494 reg == V4_num || reg == V4_H_num ||
2495 reg == V5_num || reg == V5_H_num ||
2496 reg == V6_num || reg == V6_H_num ||
2497 reg == V7_num || reg == V7_H_num;
2498 }
2499
2500 bool Matcher::is_spillable_arg(int reg)
2501 {
2502 return can_be_java_arg(reg);
2503 }
2504
2505 uint Matcher::int_pressure_limit()
2506 {
2507 // JDK-8183543: When taking the number of available registers as int
2508 // register pressure threshold, the jtreg test:
2509 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2510 // failed due to C2 compilation failure with
2511 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2512 //
2513 // A derived pointer is live at CallNode and then is flagged by RA
2514 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2515 // derived pointers and lastly fail to spill after reaching maximum
2516 // number of iterations. Lowering the default pressure threshold to
2517 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2518 // a high register pressure area of the code so that split_DEF can
2519 // generate DefinitionSpillCopy for the derived pointer.
2520 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2521 if (!PreserveFramePointer) {
2522 // When PreserveFramePointer is off, frame pointer is allocatable,
2523 // but different from other SOC registers, it is excluded from
2524 // fatproj's mask because its save type is No-Save. Decrease 1 to
2525 // ensure high pressure at fatproj when PreserveFramePointer is off.
2526 // See check_pressure_at_fatproj().
2527 default_int_pressure_threshold--;
2528 }
2529 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2530 }
2531
2532 uint Matcher::float_pressure_limit()
2533 {
2534 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2535 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2536 }
2537
2538 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2539 return false;
2540 }
2541
2542 const RegMask& Matcher::divI_proj_mask() {
2543 ShouldNotReachHere();
2544 return RegMask::EMPTY;
2545 }
2546
2547 // Register for MODI projection of divmodI.
2548 const RegMask& Matcher::modI_proj_mask() {
2549 ShouldNotReachHere();
2550 return RegMask::EMPTY;
2551 }
2552
2553 // Register for DIVL projection of divmodL.
2554 const RegMask& Matcher::divL_proj_mask() {
2555 ShouldNotReachHere();
2556 return RegMask::EMPTY;
2557 }
2558
2559 // Register for MODL projection of divmodL.
2560 const RegMask& Matcher::modL_proj_mask() {
2561 ShouldNotReachHere();
2562 return RegMask::EMPTY;
2563 }
2564
2565 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2566 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2567 Node* u = addp->fast_out(i);
2568 if (u->is_LoadStore()) {
2569 // On AArch64, LoadStoreNodes (i.e. compare and swap
2570 // instructions) only take register indirect as an operand, so
2571 // any attempt to use an AddPNode as an input to a LoadStoreNode
2572 // must fail.
2573 return false;
2574 }
2575 if (u->is_Mem()) {
2576 int opsize = u->as_Mem()->memory_size();
2577 assert(opsize > 0, "unexpected memory operand size");
2578 if (u->as_Mem()->memory_size() != (1<<shift)) {
2579 return false;
2580 }
2581 }
2582 }
2583 return true;
2584 }
2585
2586 // Convert BootTest condition to Assembler condition.
2587 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2588 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2589 Assembler::Condition result;
2590 switch(cond) {
2591 case BoolTest::eq:
2592 result = Assembler::EQ; break;
2593 case BoolTest::ne:
2594 result = Assembler::NE; break;
2595 case BoolTest::le:
2596 result = Assembler::LE; break;
2597 case BoolTest::ge:
2598 result = Assembler::GE; break;
2599 case BoolTest::lt:
2600 result = Assembler::LT; break;
2601 case BoolTest::gt:
2602 result = Assembler::GT; break;
2603 case BoolTest::ule:
2604 result = Assembler::LS; break;
2605 case BoolTest::uge:
2606 result = Assembler::HS; break;
2607 case BoolTest::ult:
2608 result = Assembler::LO; break;
2609 case BoolTest::ugt:
2610 result = Assembler::HI; break;
2611 case BoolTest::overflow:
2612 result = Assembler::VS; break;
2613 case BoolTest::no_overflow:
2614 result = Assembler::VC; break;
2615 default:
2616 ShouldNotReachHere();
2617 return Assembler::Condition(-1);
2618 }
2619
2620 // Check conversion
2621 if (cond & BoolTest::unsigned_compare) {
2622 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2623 } else {
2624 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2625 }
2626
2627 return result;
2628 }
2629
2630 // Binary src (Replicate con)
2631 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2632 if (n == nullptr || m == nullptr) {
2633 return false;
2634 }
2635
2636 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2637 return false;
2638 }
2639
2640 Node* imm_node = m->in(1);
2641 if (!imm_node->is_Con()) {
2642 return false;
2643 }
2644
2645 const Type* t = imm_node->bottom_type();
2646 if (!(t->isa_int() || t->isa_long())) {
2647 return false;
2648 }
2649
2650 switch (n->Opcode()) {
2651 case Op_AndV:
2652 case Op_OrV:
2653 case Op_XorV: {
2654 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2655 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2656 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2657 }
2658 case Op_AddVB:
2659 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2660 case Op_AddVS:
2661 case Op_AddVI:
2662 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2663 case Op_AddVL:
2664 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2665 default:
2666 return false;
2667 }
2668 }
2669
2670 // (XorV src (Replicate m1))
2671 // (XorVMask src (MaskAll m1))
2672 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2673 if (n != nullptr && m != nullptr) {
2674 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2675 VectorNode::is_all_ones_vector(m);
2676 }
2677 return false;
2678 }
2679
2680 // Should the matcher clone input 'm' of node 'n'?
2681 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2682 if (is_vshift_con_pattern(n, m) ||
2683 is_vector_bitwise_not_pattern(n, m) ||
2684 is_valid_sve_arith_imm_pattern(n, m) ||
2685 is_encode_and_store_pattern(n, m)) {
2686 mstack.push(m, Visit);
2687 return true;
2688 }
2689 return false;
2690 }
2691
2692 // Should the Matcher clone shifts on addressing modes, expecting them
2693 // to be subsumed into complex addressing expressions or compute them
2694 // into registers?
2695 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2696
2697 // Loads and stores with indirect memory input (e.g., volatile loads and
2698 // stores) do not subsume the input into complex addressing expressions. If
2699 // the addressing expression is input to at least one such load or store, do
2700 // not clone the addressing expression. Query needs_acquiring_load and
2701 // needs_releasing_store as a proxy for indirect memory input, as it is not
2702 // possible to directly query for indirect memory input at this stage.
2703 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2704 Node* n = m->fast_out(i);
2705 if (n->is_Load() && needs_acquiring_load(n)) {
2706 return false;
2707 }
2708 if (n->is_Store() && needs_releasing_store(n)) {
2709 return false;
2710 }
2711 }
2712
2713 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2714 return true;
2715 }
2716
2717 Node *off = m->in(AddPNode::Offset);
2718 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2719 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2720 // Are there other uses besides address expressions?
2721 !is_visited(off)) {
2722 address_visited.set(off->_idx); // Flag as address_visited
2723 mstack.push(off->in(2), Visit);
2724 Node *conv = off->in(1);
2725 if (conv->Opcode() == Op_ConvI2L &&
2726 // Are there other uses besides address expressions?
2727 !is_visited(conv)) {
2728 address_visited.set(conv->_idx); // Flag as address_visited
2729 mstack.push(conv->in(1), Pre_Visit);
2730 } else {
2731 mstack.push(conv, Pre_Visit);
2732 }
2733 address_visited.test_set(m->_idx); // Flag as address_visited
2734 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2735 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2736 return true;
2737 } else if (off->Opcode() == Op_ConvI2L &&
2738 // Are there other uses besides address expressions?
2739 !is_visited(off)) {
2740 address_visited.test_set(m->_idx); // Flag as address_visited
2741 address_visited.set(off->_idx); // Flag as address_visited
2742 mstack.push(off->in(1), Pre_Visit);
2743 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2744 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2745 return true;
2746 }
2747 return false;
2748 }
2749
2750 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2751 { \
2752 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2753 guarantee(DISP == 0, "mode not permitted for volatile"); \
2754 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2755 __ INSN(REG, as_Register(BASE)); \
2756 }
2757
2758
2759 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2760 {
2761 Address::extend scale;
2762
2763 // Hooboy, this is fugly. We need a way to communicate to the
2764 // encoder that the index needs to be sign extended, so we have to
2765 // enumerate all the cases.
2766 switch (opcode) {
2767 case INDINDEXSCALEDI2L:
2768 case INDINDEXSCALEDI2LN:
2769 case INDINDEXI2L:
2770 case INDINDEXI2LN:
2771 scale = Address::sxtw(size);
2772 break;
2773 default:
2774 scale = Address::lsl(size);
2775 }
2776
2777 if (index == -1) {
2778 return Address(base, disp);
2779 } else {
2780 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2781 return Address(base, as_Register(index), scale);
2782 }
2783 }
2784
2785
2786 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2787 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2788 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2789 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2790 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2791
2792 // Used for all non-volatile memory accesses. The use of
2793 // $mem->opcode() to discover whether this pattern uses sign-extended
2794 // offsets is something of a kludge.
2795 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2796 Register reg, int opcode,
2797 Register base, int index, int scale, int disp,
2798 int size_in_memory)
2799 {
2800 Address addr = mem2address(opcode, base, index, scale, disp);
2801 if (addr.getMode() == Address::base_plus_offset) {
2802 /* Fix up any out-of-range offsets. */
2803 assert_different_registers(rscratch1, base);
2804 assert_different_registers(rscratch1, reg);
2805 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2806 }
2807 (masm->*insn)(reg, addr);
2808 }
2809
2810 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2811 FloatRegister reg, int opcode,
2812 Register base, int index, int size, int disp,
2813 int size_in_memory)
2814 {
2815 Address::extend scale;
2816
2817 switch (opcode) {
2818 case INDINDEXSCALEDI2L:
2819 case INDINDEXSCALEDI2LN:
2820 scale = Address::sxtw(size);
2821 break;
2822 default:
2823 scale = Address::lsl(size);
2824 }
2825
2826 if (index == -1) {
2827 // Fix up any out-of-range offsets.
2828 assert_different_registers(rscratch1, base);
2829 Address addr = Address(base, disp);
2830 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2831 (masm->*insn)(reg, addr);
2832 } else {
2833 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2834 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2835 }
2836 }
2837
2838 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2839 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2840 int opcode, Register base, int index, int size, int disp)
2841 {
2842 if (index == -1) {
2843 (masm->*insn)(reg, T, Address(base, disp));
2844 } else {
2845 assert(disp == 0, "unsupported address mode");
2846 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2847 }
2848 }
2849
2850 %}
2851
2852
2853
2854 //----------ENCODING BLOCK-----------------------------------------------------
2855 // This block specifies the encoding classes used by the compiler to
2856 // output byte streams. Encoding classes are parameterized macros
2857 // used by Machine Instruction Nodes in order to generate the bit
2858 // encoding of the instruction. Operands specify their base encoding
2859 // interface with the interface keyword. There are currently
2860 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2861 // COND_INTER. REG_INTER causes an operand to generate a function
2862 // which returns its register number when queried. CONST_INTER causes
2863 // an operand to generate a function which returns the value of the
2864 // constant when queried. MEMORY_INTER causes an operand to generate
2865 // four functions which return the Base Register, the Index Register,
2866 // the Scale Value, and the Offset Value of the operand when queried.
2867 // COND_INTER causes an operand to generate six functions which return
2868 // the encoding code (ie - encoding bits for the instruction)
2869 // associated with each basic boolean condition for a conditional
2870 // instruction.
2871 //
2872 // Instructions specify two basic values for encoding. Again, a
2873 // function is available to check if the constant displacement is an
2874 // oop. They use the ins_encode keyword to specify their encoding
2875 // classes (which must be a sequence of enc_class names, and their
2876 // parameters, specified in the encoding block), and they use the
2877 // opcode keyword to specify, in order, their primary, secondary, and
2878 // tertiary opcode. Only the opcode sections which a particular
2879 // instruction needs for encoding need to be specified.
2880 encode %{
2881 // Build emit functions for each basic byte or larger field in the
2882 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2883 // from C++ code in the enc_class source block. Emit functions will
2884 // live in the main source block for now. In future, we can
2885 // generalize this by adding a syntax that specifies the sizes of
2886 // fields in an order, so that the adlc can build the emit functions
2887 // automagically
2888
2889 // catch all for unimplemented encodings
2890 enc_class enc_unimplemented %{
2891 __ unimplemented("C2 catch all");
2892 %}
2893
2894 // BEGIN Non-volatile memory access
2895
2896 // This encoding class is generated automatically from ad_encode.m4.
2897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2898 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2899 Register dst_reg = as_Register($dst$$reg);
2900 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2901 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2902 %}
2903
2904 // This encoding class is generated automatically from ad_encode.m4.
2905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2906 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2907 Register dst_reg = as_Register($dst$$reg);
2908 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2909 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2910 %}
2911
2912 // This encoding class is generated automatically from ad_encode.m4.
2913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2914 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2915 Register dst_reg = as_Register($dst$$reg);
2916 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2917 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2918 %}
2919
2920 // This encoding class is generated automatically from ad_encode.m4.
2921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2922 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2923 Register dst_reg = as_Register($dst$$reg);
2924 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2925 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2926 %}
2927
2928 // This encoding class is generated automatically from ad_encode.m4.
2929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2930 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2931 Register dst_reg = as_Register($dst$$reg);
2932 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2933 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2934 %}
2935
2936 // This encoding class is generated automatically from ad_encode.m4.
2937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2938 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2939 Register dst_reg = as_Register($dst$$reg);
2940 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2941 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2942 %}
2943
2944 // This encoding class is generated automatically from ad_encode.m4.
2945 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2946 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2947 Register dst_reg = as_Register($dst$$reg);
2948 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2949 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2950 %}
2951
2952 // This encoding class is generated automatically from ad_encode.m4.
2953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2954 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2955 Register dst_reg = as_Register($dst$$reg);
2956 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2957 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2958 %}
2959
2960 // This encoding class is generated automatically from ad_encode.m4.
2961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2962 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2963 Register dst_reg = as_Register($dst$$reg);
2964 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2965 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2966 %}
2967
2968 // This encoding class is generated automatically from ad_encode.m4.
2969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2970 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2971 Register dst_reg = as_Register($dst$$reg);
2972 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2973 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2974 %}
2975
2976 // This encoding class is generated automatically from ad_encode.m4.
2977 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2978 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2979 Register dst_reg = as_Register($dst$$reg);
2980 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2981 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2982 %}
2983
2984 // This encoding class is generated automatically from ad_encode.m4.
2985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2986 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2987 Register dst_reg = as_Register($dst$$reg);
2988 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2989 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2990 %}
2991
2992 // This encoding class is generated automatically from ad_encode.m4.
2993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2994 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2995 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2996 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2997 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2998 %}
2999
3000 // This encoding class is generated automatically from ad_encode.m4.
3001 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3002 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
3003 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3004 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
3005 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3006 %}
3007
3008 // This encoding class is generated automatically from ad_encode.m4.
3009 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3010 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3011 Register src_reg = as_Register($src$$reg);
3012 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3013 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3014 %}
3015
3016 // This encoding class is generated automatically from ad_encode.m4.
3017 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3018 enc_class aarch64_enc_strb0(memory1 mem) %{
3019 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3020 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3021 %}
3022
3023 // This encoding class is generated automatically from ad_encode.m4.
3024 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3025 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3026 Register src_reg = as_Register($src$$reg);
3027 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3028 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3029 %}
3030
3031 // This encoding class is generated automatically from ad_encode.m4.
3032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3033 enc_class aarch64_enc_strh0(memory2 mem) %{
3034 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3035 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3036 %}
3037
3038 // This encoding class is generated automatically from ad_encode.m4.
3039 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3040 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3041 Register src_reg = as_Register($src$$reg);
3042 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3043 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3044 %}
3045
3046 // This encoding class is generated automatically from ad_encode.m4.
3047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3048 enc_class aarch64_enc_strw0(memory4 mem) %{
3049 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3050 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3051 %}
3052
3053 // This encoding class is generated automatically from ad_encode.m4.
3054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3055 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3056 Register src_reg = as_Register($src$$reg);
3057 // we sometimes get asked to store the stack pointer into the
3058 // current thread -- we cannot do that directly on AArch64
3059 if (src_reg == r31_sp) {
3060 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3061 __ mov(rscratch2, sp);
3062 src_reg = rscratch2;
3063 }
3064 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3065 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3066 %}
3067
3068 // This encoding class is generated automatically from ad_encode.m4.
3069 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3070 enc_class aarch64_enc_str0(memory8 mem) %{
3071 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3072 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3073 %}
3074
3075 // This encoding class is generated automatically from ad_encode.m4.
3076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3077 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3078 FloatRegister src_reg = as_FloatRegister($src$$reg);
3079 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3080 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3081 %}
3082
3083 // This encoding class is generated automatically from ad_encode.m4.
3084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3085 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3086 FloatRegister src_reg = as_FloatRegister($src$$reg);
3087 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3088 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3089 %}
3090
3091 // This encoding class is generated automatically from ad_encode.m4.
3092 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3093 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3094 __ membar(Assembler::StoreStore);
3095 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3096 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3097 %}
3098
3099 // END Non-volatile memory access
3100
3101 // Vector loads and stores
3102 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3103 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3104 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3105 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3106 %}
3107
3108 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3109 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3110 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3111 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3112 %}
3113
3114 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3115 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3116 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3117 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3118 %}
3119
3120 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3121 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3122 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3123 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3124 %}
3125
3126 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3127 FloatRegister src_reg = as_FloatRegister($src$$reg);
3128 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3129 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3130 %}
3131
3132 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3133 FloatRegister src_reg = as_FloatRegister($src$$reg);
3134 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3135 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3136 %}
3137
3138 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3139 FloatRegister src_reg = as_FloatRegister($src$$reg);
3140 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3141 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3142 %}
3143
3144 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3145 FloatRegister src_reg = as_FloatRegister($src$$reg);
3146 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3147 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3148 %}
3149
3150 // volatile loads and stores
3151
3152 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3153 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3154 rscratch1, stlrb);
3155 %}
3156
3157 enc_class aarch64_enc_stlrb0(memory mem) %{
3158 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3159 rscratch1, stlrb);
3160 %}
3161
3162 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3163 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, stlrh);
3165 %}
3166
3167 enc_class aarch64_enc_stlrh0(memory mem) %{
3168 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3169 rscratch1, stlrh);
3170 %}
3171
3172 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3173 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3174 rscratch1, stlrw);
3175 %}
3176
3177 enc_class aarch64_enc_stlrw0(memory mem) %{
3178 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3179 rscratch1, stlrw);
3180 %}
3181
3182 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3183 Register dst_reg = as_Register($dst$$reg);
3184 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3185 rscratch1, ldarb);
3186 __ sxtbw(dst_reg, dst_reg);
3187 %}
3188
3189 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3190 Register dst_reg = as_Register($dst$$reg);
3191 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3192 rscratch1, ldarb);
3193 __ sxtb(dst_reg, dst_reg);
3194 %}
3195
3196 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3197 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3198 rscratch1, ldarb);
3199 %}
3200
3201 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3202 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3203 rscratch1, ldarb);
3204 %}
3205
3206 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3207 Register dst_reg = as_Register($dst$$reg);
3208 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3209 rscratch1, ldarh);
3210 __ sxthw(dst_reg, dst_reg);
3211 %}
3212
3213 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3214 Register dst_reg = as_Register($dst$$reg);
3215 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3216 rscratch1, ldarh);
3217 __ sxth(dst_reg, dst_reg);
3218 %}
3219
3220 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3221 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3222 rscratch1, ldarh);
3223 %}
3224
3225 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3226 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3227 rscratch1, ldarh);
3228 %}
3229
3230 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3231 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3232 rscratch1, ldarw);
3233 %}
3234
3235 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3236 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3237 rscratch1, ldarw);
3238 %}
3239
3240 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3241 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3242 rscratch1, ldar);
3243 %}
3244
3245 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3246 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3247 rscratch1, ldarw);
3248 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3249 %}
3250
3251 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3252 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3253 rscratch1, ldar);
3254 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3255 %}
3256
3257 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3258 Register src_reg = as_Register($src$$reg);
3259 // we sometimes get asked to store the stack pointer into the
3260 // current thread -- we cannot do that directly on AArch64
3261 if (src_reg == r31_sp) {
3262 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3263 __ mov(rscratch2, sp);
3264 src_reg = rscratch2;
3265 }
3266 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3267 rscratch1, stlr);
3268 %}
3269
3270 enc_class aarch64_enc_stlr0(memory mem) %{
3271 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3272 rscratch1, stlr);
3273 %}
3274
3275 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3276 {
3277 FloatRegister src_reg = as_FloatRegister($src$$reg);
3278 __ fmovs(rscratch2, src_reg);
3279 }
3280 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3281 rscratch1, stlrw);
3282 %}
3283
3284 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3285 {
3286 FloatRegister src_reg = as_FloatRegister($src$$reg);
3287 __ fmovd(rscratch2, src_reg);
3288 }
3289 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3290 rscratch1, stlr);
3291 %}
3292
3293 // synchronized read/update encodings
3294
3295 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3296 Register dst_reg = as_Register($dst$$reg);
3297 Register base = as_Register($mem$$base);
3298 int index = $mem$$index;
3299 int scale = $mem$$scale;
3300 int disp = $mem$$disp;
3301 if (index == -1) {
3302 if (disp != 0) {
3303 __ lea(rscratch1, Address(base, disp));
3304 __ ldaxr(dst_reg, rscratch1);
3305 } else {
3306 // TODO
3307 // should we ever get anything other than this case?
3308 __ ldaxr(dst_reg, base);
3309 }
3310 } else {
3311 Register index_reg = as_Register(index);
3312 if (disp == 0) {
3313 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3314 __ ldaxr(dst_reg, rscratch1);
3315 } else {
3316 __ lea(rscratch1, Address(base, disp));
3317 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3318 __ ldaxr(dst_reg, rscratch1);
3319 }
3320 }
3321 %}
3322
3323 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3324 Register src_reg = as_Register($src$$reg);
3325 Register base = as_Register($mem$$base);
3326 int index = $mem$$index;
3327 int scale = $mem$$scale;
3328 int disp = $mem$$disp;
3329 if (index == -1) {
3330 if (disp != 0) {
3331 __ lea(rscratch2, Address(base, disp));
3332 __ stlxr(rscratch1, src_reg, rscratch2);
3333 } else {
3334 // TODO
3335 // should we ever get anything other than this case?
3336 __ stlxr(rscratch1, src_reg, base);
3337 }
3338 } else {
3339 Register index_reg = as_Register(index);
3340 if (disp == 0) {
3341 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3342 __ stlxr(rscratch1, src_reg, rscratch2);
3343 } else {
3344 __ lea(rscratch2, Address(base, disp));
3345 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3346 __ stlxr(rscratch1, src_reg, rscratch2);
3347 }
3348 }
3349 __ cmpw(rscratch1, zr);
3350 %}
3351
3352 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3353 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3354 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3355 Assembler::xword, /*acquire*/ false, /*release*/ true,
3356 /*weak*/ false, noreg);
3357 %}
3358
3359 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3360 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3361 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3362 Assembler::word, /*acquire*/ false, /*release*/ true,
3363 /*weak*/ false, noreg);
3364 %}
3365
3366 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3367 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3368 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3369 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3370 /*weak*/ false, noreg);
3371 %}
3372
3373 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3374 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3375 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3376 Assembler::byte, /*acquire*/ false, /*release*/ true,
3377 /*weak*/ false, noreg);
3378 %}
3379
3380
3381 // The only difference between aarch64_enc_cmpxchg and
3382 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3383 // CompareAndSwap sequence to serve as a barrier on acquiring a
3384 // lock.
3385 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3386 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3387 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3388 Assembler::xword, /*acquire*/ true, /*release*/ true,
3389 /*weak*/ false, noreg);
3390 %}
3391
3392 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3393 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3394 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3395 Assembler::word, /*acquire*/ true, /*release*/ true,
3396 /*weak*/ false, noreg);
3397 %}
3398
3399 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3400 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3401 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3402 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3403 /*weak*/ false, noreg);
3404 %}
3405
3406 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3407 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3408 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3409 Assembler::byte, /*acquire*/ true, /*release*/ true,
3410 /*weak*/ false, noreg);
3411 %}
3412
3413 // auxiliary used for CompareAndSwapX to set result register
3414 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3415 Register res_reg = as_Register($res$$reg);
3416 __ cset(res_reg, Assembler::EQ);
3417 %}
3418
3419 // prefetch encodings
3420
3421 enc_class aarch64_enc_prefetchw(memory mem) %{
3422 Register base = as_Register($mem$$base);
3423 int index = $mem$$index;
3424 int scale = $mem$$scale;
3425 int disp = $mem$$disp;
3426 if (index == -1) {
3427 // Fix up any out-of-range offsets.
3428 assert_different_registers(rscratch1, base);
3429 Address addr = Address(base, disp);
3430 addr = __ legitimize_address(addr, 8, rscratch1);
3431 __ prfm(addr, PSTL1KEEP);
3432 } else {
3433 Register index_reg = as_Register(index);
3434 if (disp == 0) {
3435 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3436 } else {
3437 __ lea(rscratch1, Address(base, disp));
3438 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3439 }
3440 }
3441 %}
3442
3443 // mov encodings
3444
3445 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3446 uint32_t con = (uint32_t)$src$$constant;
3447 Register dst_reg = as_Register($dst$$reg);
3448 if (con == 0) {
3449 __ movw(dst_reg, zr);
3450 } else {
3451 __ movw(dst_reg, con);
3452 }
3453 %}
3454
3455 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3456 Register dst_reg = as_Register($dst$$reg);
3457 uint64_t con = (uint64_t)$src$$constant;
3458 if (con == 0) {
3459 __ mov(dst_reg, zr);
3460 } else {
3461 __ mov(dst_reg, con);
3462 }
3463 %}
3464
3465 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3466 Register dst_reg = as_Register($dst$$reg);
3467 address con = (address)$src$$constant;
3468 if (con == nullptr || con == (address)1) {
3469 ShouldNotReachHere();
3470 } else {
3471 relocInfo::relocType rtype = $src->constant_reloc();
3472 if (rtype == relocInfo::oop_type) {
3473 __ movoop(dst_reg, (jobject)con);
3474 } else if (rtype == relocInfo::metadata_type) {
3475 __ mov_metadata(dst_reg, (Metadata*)con);
3476 } else {
3477 assert(rtype == relocInfo::none, "unexpected reloc type");
3478 if (! __ is_valid_AArch64_address(con) ||
3479 con < (address)(uintptr_t)os::vm_page_size()) {
3480 __ mov(dst_reg, con);
3481 } else {
3482 uint64_t offset;
3483 __ adrp(dst_reg, con, offset);
3484 __ add(dst_reg, dst_reg, offset);
3485 }
3486 }
3487 }
3488 %}
3489
3490 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3491 Register dst_reg = as_Register($dst$$reg);
3492 __ mov(dst_reg, zr);
3493 %}
3494
3495 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3496 Register dst_reg = as_Register($dst$$reg);
3497 __ mov(dst_reg, (uint64_t)1);
3498 %}
3499
3500 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3501 Register dst_reg = as_Register($dst$$reg);
3502 address con = (address)$src$$constant;
3503 if (con == nullptr) {
3504 ShouldNotReachHere();
3505 } else {
3506 relocInfo::relocType rtype = $src->constant_reloc();
3507 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3508 __ set_narrow_oop(dst_reg, (jobject)con);
3509 }
3510 %}
3511
3512 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3513 Register dst_reg = as_Register($dst$$reg);
3514 __ mov(dst_reg, zr);
3515 %}
3516
3517 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3518 Register dst_reg = as_Register($dst$$reg);
3519 address con = (address)$src$$constant;
3520 if (con == nullptr) {
3521 ShouldNotReachHere();
3522 } else {
3523 relocInfo::relocType rtype = $src->constant_reloc();
3524 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3525 __ set_narrow_klass(dst_reg, (Klass *)con);
3526 }
3527 %}
3528
3529 // arithmetic encodings
3530
3531 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3532 Register dst_reg = as_Register($dst$$reg);
3533 Register src_reg = as_Register($src1$$reg);
3534 int32_t con = (int32_t)$src2$$constant;
3535 // add has primary == 0, subtract has primary == 1
3536 if ($primary) { con = -con; }
3537 if (con < 0) {
3538 __ subw(dst_reg, src_reg, -con);
3539 } else {
3540 __ addw(dst_reg, src_reg, con);
3541 }
3542 %}
3543
3544 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3545 Register dst_reg = as_Register($dst$$reg);
3546 Register src_reg = as_Register($src1$$reg);
3547 int32_t con = (int32_t)$src2$$constant;
3548 // add has primary == 0, subtract has primary == 1
3549 if ($primary) { con = -con; }
3550 if (con < 0) {
3551 __ sub(dst_reg, src_reg, -con);
3552 } else {
3553 __ add(dst_reg, src_reg, con);
3554 }
3555 %}
3556
3557 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3558 Register dst_reg = as_Register($dst$$reg);
3559 Register src1_reg = as_Register($src1$$reg);
3560 Register src2_reg = as_Register($src2$$reg);
3561 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3562 %}
3563
3564 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3565 Register dst_reg = as_Register($dst$$reg);
3566 Register src1_reg = as_Register($src1$$reg);
3567 Register src2_reg = as_Register($src2$$reg);
3568 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3569 %}
3570
3571 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3572 Register dst_reg = as_Register($dst$$reg);
3573 Register src1_reg = as_Register($src1$$reg);
3574 Register src2_reg = as_Register($src2$$reg);
3575 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3576 %}
3577
3578 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3579 Register dst_reg = as_Register($dst$$reg);
3580 Register src1_reg = as_Register($src1$$reg);
3581 Register src2_reg = as_Register($src2$$reg);
3582 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3583 %}
3584
3585 // compare instruction encodings
3586
3587 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3588 Register reg1 = as_Register($src1$$reg);
3589 Register reg2 = as_Register($src2$$reg);
3590 __ cmpw(reg1, reg2);
3591 %}
3592
3593 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3594 Register reg = as_Register($src1$$reg);
3595 int32_t val = $src2$$constant;
3596 if (val >= 0) {
3597 __ subsw(zr, reg, val);
3598 } else {
3599 __ addsw(zr, reg, -val);
3600 }
3601 %}
3602
3603 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3604 Register reg1 = as_Register($src1$$reg);
3605 uint32_t val = (uint32_t)$src2$$constant;
3606 __ movw(rscratch1, val);
3607 __ cmpw(reg1, rscratch1);
3608 %}
3609
3610 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3611 Register reg1 = as_Register($src1$$reg);
3612 Register reg2 = as_Register($src2$$reg);
3613 __ cmp(reg1, reg2);
3614 %}
3615
3616 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3617 Register reg = as_Register($src1$$reg);
3618 int64_t val = $src2$$constant;
3619 if (val >= 0) {
3620 __ subs(zr, reg, val);
3621 } else if (val != -val) {
3622 __ adds(zr, reg, -val);
3623 } else {
3624 // aargh, Long.MIN_VALUE is a special case
3625 __ orr(rscratch1, zr, (uint64_t)val);
3626 __ subs(zr, reg, rscratch1);
3627 }
3628 %}
3629
3630 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3631 Register reg1 = as_Register($src1$$reg);
3632 uint64_t val = (uint64_t)$src2$$constant;
3633 __ mov(rscratch1, val);
3634 __ cmp(reg1, rscratch1);
3635 %}
3636
3637 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3638 Register reg1 = as_Register($src1$$reg);
3639 Register reg2 = as_Register($src2$$reg);
3640 __ cmp(reg1, reg2);
3641 %}
3642
3643 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3644 Register reg1 = as_Register($src1$$reg);
3645 Register reg2 = as_Register($src2$$reg);
3646 __ cmpw(reg1, reg2);
3647 %}
3648
3649 enc_class aarch64_enc_testp(iRegP src) %{
3650 Register reg = as_Register($src$$reg);
3651 __ cmp(reg, zr);
3652 %}
3653
3654 enc_class aarch64_enc_testn(iRegN src) %{
3655 Register reg = as_Register($src$$reg);
3656 __ cmpw(reg, zr);
3657 %}
3658
3659 enc_class aarch64_enc_b(label lbl) %{
3660 Label *L = $lbl$$label;
3661 __ b(*L);
3662 %}
3663
3664 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3665 Label *L = $lbl$$label;
3666 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3667 %}
3668
3669 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3670 Label *L = $lbl$$label;
3671 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3672 %}
3673
3674 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3675 %{
3676 Register sub_reg = as_Register($sub$$reg);
3677 Register super_reg = as_Register($super$$reg);
3678 Register temp_reg = as_Register($temp$$reg);
3679 Register result_reg = as_Register($result$$reg);
3680
3681 Label miss;
3682 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3683 nullptr, &miss,
3684 /*set_cond_codes:*/ true);
3685 if ($primary) {
3686 __ mov(result_reg, zr);
3687 }
3688 __ bind(miss);
3689 %}
3690
3691 enc_class aarch64_enc_java_static_call(method meth) %{
3692 address addr = (address)$meth$$method;
3693 address call;
3694 if (!_method) {
3695 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3696 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3697 if (call == nullptr) {
3698 ciEnv::current()->record_failure("CodeCache is full");
3699 return;
3700 }
3701 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3702 // The NOP here is purely to ensure that eliding a call to
3703 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3704 __ nop();
3705 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3706 } else {
3707 int method_index = resolved_method_index(masm);
3708 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3709 : static_call_Relocation::spec(method_index);
3710 call = __ trampoline_call(Address(addr, rspec));
3711 if (call == nullptr) {
3712 ciEnv::current()->record_failure("CodeCache is full");
3713 return;
3714 }
3715 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3716 // Calls of the same statically bound method can share
3717 // a stub to the interpreter.
3718 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3719 } else {
3720 // Emit stub for static call
3721 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3722 if (stub == nullptr) {
3723 ciEnv::current()->record_failure("CodeCache is full");
3724 return;
3725 }
3726 }
3727 }
3728
3729 __ post_call_nop();
3730
3731 // Only non uncommon_trap calls need to reinitialize ptrue.
3732 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3733 __ reinitialize_ptrue();
3734 }
3735 %}
3736
3737 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3738 int method_index = resolved_method_index(masm);
3739 address call = __ ic_call((address)$meth$$method, method_index);
3740 if (call == nullptr) {
3741 ciEnv::current()->record_failure("CodeCache is full");
3742 return;
3743 }
3744 __ post_call_nop();
3745 if (Compile::current()->max_vector_size() > 0) {
3746 __ reinitialize_ptrue();
3747 }
3748 %}
3749
3750 enc_class aarch64_enc_call_epilog() %{
3751 if (VerifyStackAtCalls) {
3752 // Check that stack depth is unchanged: find majik cookie on stack
3753 __ call_Unimplemented();
3754 }
3755 %}
3756
3757 enc_class aarch64_enc_java_to_runtime(method meth) %{
3758 // some calls to generated routines (arraycopy code) are scheduled
3759 // by C2 as runtime calls. if so we can call them using a br (they
3760 // will be in a reachable segment) otherwise we have to use a blr
3761 // which loads the absolute address into a register.
3762 address entry = (address)$meth$$method;
3763 CodeBlob *cb = CodeCache::find_blob(entry);
3764 if (cb) {
3765 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3766 if (call == nullptr) {
3767 ciEnv::current()->record_failure("CodeCache is full");
3768 return;
3769 }
3770 __ post_call_nop();
3771 } else {
3772 Label retaddr;
3773 // Make the anchor frame walkable
3774 __ adr(rscratch2, retaddr);
3775 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3776 __ lea(rscratch1, RuntimeAddress(entry));
3777 __ blr(rscratch1);
3778 __ bind(retaddr);
3779 __ post_call_nop();
3780 }
3781 if (Compile::current()->max_vector_size() > 0) {
3782 __ reinitialize_ptrue();
3783 }
3784 %}
3785
3786 enc_class aarch64_enc_rethrow() %{
3787 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3788 %}
3789
3790 enc_class aarch64_enc_ret() %{
3791 #ifdef ASSERT
3792 if (Compile::current()->max_vector_size() > 0) {
3793 __ verify_ptrue();
3794 }
3795 #endif
3796 __ ret(lr);
3797 %}
3798
3799 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3800 Register target_reg = as_Register($jump_target$$reg);
3801 __ br(target_reg);
3802 %}
3803
3804 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3805 Register target_reg = as_Register($jump_target$$reg);
3806 // exception oop should be in r0
3807 // ret addr has been popped into lr
3808 // callee expects it in r3
3809 __ mov(r3, lr);
3810 __ br(target_reg);
3811 %}
3812
3813 %}
3814
3815 //----------FRAME--------------------------------------------------------------
3816 // Definition of frame structure and management information.
3817 //
3818 // S T A C K L A Y O U T Allocators stack-slot number
3819 // | (to get allocators register number
3820 // G Owned by | | v add OptoReg::stack0())
3821 // r CALLER | |
3822 // o | +--------+ pad to even-align allocators stack-slot
3823 // w V | pad0 | numbers; owned by CALLER
3824 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3825 // h ^ | in | 5
3826 // | | args | 4 Holes in incoming args owned by SELF
3827 // | | | | 3
3828 // | | +--------+
3829 // V | | old out| Empty on Intel, window on Sparc
3830 // | old |preserve| Must be even aligned.
3831 // | SP-+--------+----> Matcher::_old_SP, even aligned
3832 // | | in | 3 area for Intel ret address
3833 // Owned by |preserve| Empty on Sparc.
3834 // SELF +--------+
3835 // | | pad2 | 2 pad to align old SP
3836 // | +--------+ 1
3837 // | | locks | 0
3838 // | +--------+----> OptoReg::stack0(), even aligned
3839 // | | pad1 | 11 pad to align new SP
3840 // | +--------+
3841 // | | | 10
3842 // | | spills | 9 spills
3843 // V | | 8 (pad0 slot for callee)
3844 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3845 // ^ | out | 7
3846 // | | args | 6 Holes in outgoing args owned by CALLEE
3847 // Owned by +--------+
3848 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3849 // | new |preserve| Must be even-aligned.
3850 // | SP-+--------+----> Matcher::_new_SP, even aligned
3851 // | | |
3852 //
3853 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3854 // known from SELF's arguments and the Java calling convention.
3855 // Region 6-7 is determined per call site.
3856 // Note 2: If the calling convention leaves holes in the incoming argument
3857 // area, those holes are owned by SELF. Holes in the outgoing area
3858 // are owned by the CALLEE. Holes should not be necessary in the
3859 // incoming area, as the Java calling convention is completely under
3860 // the control of the AD file. Doubles can be sorted and packed to
3861 // avoid holes. Holes in the outgoing arguments may be necessary for
3862 // varargs C calling conventions.
3863 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3864 // even aligned with pad0 as needed.
3865 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3866 // (the latter is true on Intel but is it false on AArch64?)
3867 // region 6-11 is even aligned; it may be padded out more so that
3868 // the region from SP to FP meets the minimum stack alignment.
3869 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3870 // alignment. Region 11, pad1, may be dynamically extended so that
3871 // SP meets the minimum alignment.
3872
3873 frame %{
3874 // These three registers define part of the calling convention
3875 // between compiled code and the interpreter.
3876
3877 // Inline Cache Register or Method for I2C.
3878 inline_cache_reg(R12);
3879
3880 // Number of stack slots consumed by locking an object
3881 sync_stack_slots(2);
3882
3883 // Compiled code's Frame Pointer
3884 frame_pointer(R31);
3885
3886 // Interpreter stores its frame pointer in a register which is
3887 // stored to the stack by I2CAdaptors.
3888 // I2CAdaptors convert from interpreted java to compiled java.
3889 interpreter_frame_pointer(R29);
3890
3891 // Stack alignment requirement
3892 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3893
3894 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3895 // for calls to C. Supports the var-args backing area for register parms.
3896 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3897
3898 // The after-PROLOG location of the return address. Location of
3899 // return address specifies a type (REG or STACK) and a number
3900 // representing the register number (i.e. - use a register name) or
3901 // stack slot.
3902 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3903 // Otherwise, it is above the locks and verification slot and alignment word
3904 // TODO this may well be correct but need to check why that - 2 is there
3905 // ppc port uses 0 but we definitely need to allow for fixed_slots
3906 // which folds in the space used for monitors
3907 return_addr(STACK - 2 +
3908 align_up((Compile::current()->in_preserve_stack_slots() +
3909 Compile::current()->fixed_slots()),
3910 stack_alignment_in_slots()));
3911
3912 // Location of compiled Java return values. Same as C for now.
3913 return_value
3914 %{
3915 // TODO do we allow ideal_reg == Op_RegN???
3916 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3917 "only return normal values");
3918
3919 static const int lo[Op_RegL + 1] = { // enum name
3920 0, // Op_Node
3921 0, // Op_Set
3922 R0_num, // Op_RegN
3923 R0_num, // Op_RegI
3924 R0_num, // Op_RegP
3925 V0_num, // Op_RegF
3926 V0_num, // Op_RegD
3927 R0_num // Op_RegL
3928 };
3929
3930 static const int hi[Op_RegL + 1] = { // enum name
3931 0, // Op_Node
3932 0, // Op_Set
3933 OptoReg::Bad, // Op_RegN
3934 OptoReg::Bad, // Op_RegI
3935 R0_H_num, // Op_RegP
3936 OptoReg::Bad, // Op_RegF
3937 V0_H_num, // Op_RegD
3938 R0_H_num // Op_RegL
3939 };
3940
3941 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3942 %}
3943 %}
3944
3945 //----------ATTRIBUTES---------------------------------------------------------
3946 //----------Operand Attributes-------------------------------------------------
3947 op_attrib op_cost(1); // Required cost attribute
3948
3949 //----------Instruction Attributes---------------------------------------------
3950 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3951 ins_attrib ins_size(32); // Required size attribute (in bits)
3952 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3953 // a non-matching short branch variant
3954 // of some long branch?
3955 ins_attrib ins_alignment(4); // Required alignment attribute (must
3956 // be a power of 2) specifies the
3957 // alignment that some part of the
3958 // instruction (not necessarily the
3959 // start) requires. If > 1, a
3960 // compute_padding() function must be
3961 // provided for the instruction
3962
3963 // Whether this node is expanded during code emission into a sequence of
3964 // instructions and the first instruction can perform an implicit null check.
3965 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3966
3967 //----------OPERANDS-----------------------------------------------------------
3968 // Operand definitions must precede instruction definitions for correct parsing
3969 // in the ADLC because operands constitute user defined types which are used in
3970 // instruction definitions.
3971
3972 //----------Simple Operands----------------------------------------------------
3973
3974 // Integer operands 32 bit
3975 // 32 bit immediate
3976 operand immI()
3977 %{
3978 match(ConI);
3979
3980 op_cost(0);
3981 format %{ %}
3982 interface(CONST_INTER);
3983 %}
3984
3985 // 32 bit zero
3986 operand immI0()
3987 %{
3988 predicate(n->get_int() == 0);
3989 match(ConI);
3990
3991 op_cost(0);
3992 format %{ %}
3993 interface(CONST_INTER);
3994 %}
3995
3996 // 32 bit unit increment
3997 operand immI_1()
3998 %{
3999 predicate(n->get_int() == 1);
4000 match(ConI);
4001
4002 op_cost(0);
4003 format %{ %}
4004 interface(CONST_INTER);
4005 %}
4006
4007 // 32 bit unit decrement
4008 operand immI_M1()
4009 %{
4010 predicate(n->get_int() == -1);
4011 match(ConI);
4012
4013 op_cost(0);
4014 format %{ %}
4015 interface(CONST_INTER);
4016 %}
4017
4018 // Shift values for add/sub extension shift
4019 operand immIExt()
4020 %{
4021 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4022 match(ConI);
4023
4024 op_cost(0);
4025 format %{ %}
4026 interface(CONST_INTER);
4027 %}
4028
4029 operand immI_gt_1()
4030 %{
4031 predicate(n->get_int() > 1);
4032 match(ConI);
4033
4034 op_cost(0);
4035 format %{ %}
4036 interface(CONST_INTER);
4037 %}
4038
4039 operand immI_le_4()
4040 %{
4041 predicate(n->get_int() <= 4);
4042 match(ConI);
4043
4044 op_cost(0);
4045 format %{ %}
4046 interface(CONST_INTER);
4047 %}
4048
4049 operand immI_16()
4050 %{
4051 predicate(n->get_int() == 16);
4052 match(ConI);
4053
4054 op_cost(0);
4055 format %{ %}
4056 interface(CONST_INTER);
4057 %}
4058
4059 operand immI_24()
4060 %{
4061 predicate(n->get_int() == 24);
4062 match(ConI);
4063
4064 op_cost(0);
4065 format %{ %}
4066 interface(CONST_INTER);
4067 %}
4068
4069 operand immI_32()
4070 %{
4071 predicate(n->get_int() == 32);
4072 match(ConI);
4073
4074 op_cost(0);
4075 format %{ %}
4076 interface(CONST_INTER);
4077 %}
4078
4079 operand immI_48()
4080 %{
4081 predicate(n->get_int() == 48);
4082 match(ConI);
4083
4084 op_cost(0);
4085 format %{ %}
4086 interface(CONST_INTER);
4087 %}
4088
4089 operand immI_56()
4090 %{
4091 predicate(n->get_int() == 56);
4092 match(ConI);
4093
4094 op_cost(0);
4095 format %{ %}
4096 interface(CONST_INTER);
4097 %}
4098
4099 operand immI_255()
4100 %{
4101 predicate(n->get_int() == 255);
4102 match(ConI);
4103
4104 op_cost(0);
4105 format %{ %}
4106 interface(CONST_INTER);
4107 %}
4108
4109 operand immI_65535()
4110 %{
4111 predicate(n->get_int() == 65535);
4112 match(ConI);
4113
4114 op_cost(0);
4115 format %{ %}
4116 interface(CONST_INTER);
4117 %}
4118
4119 operand immI_positive()
4120 %{
4121 predicate(n->get_int() > 0);
4122 match(ConI);
4123
4124 op_cost(0);
4125 format %{ %}
4126 interface(CONST_INTER);
4127 %}
4128
4129 // BoolTest condition for signed compare
4130 operand immI_cmp_cond()
4131 %{
4132 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4133 match(ConI);
4134
4135 op_cost(0);
4136 format %{ %}
4137 interface(CONST_INTER);
4138 %}
4139
4140 // BoolTest condition for unsigned compare
4141 operand immI_cmpU_cond()
4142 %{
4143 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4144 match(ConI);
4145
4146 op_cost(0);
4147 format %{ %}
4148 interface(CONST_INTER);
4149 %}
4150
4151 operand immL_255()
4152 %{
4153 predicate(n->get_long() == 255L);
4154 match(ConL);
4155
4156 op_cost(0);
4157 format %{ %}
4158 interface(CONST_INTER);
4159 %}
4160
4161 operand immL_65535()
4162 %{
4163 predicate(n->get_long() == 65535L);
4164 match(ConL);
4165
4166 op_cost(0);
4167 format %{ %}
4168 interface(CONST_INTER);
4169 %}
4170
4171 operand immL_4294967295()
4172 %{
4173 predicate(n->get_long() == 4294967295L);
4174 match(ConL);
4175
4176 op_cost(0);
4177 format %{ %}
4178 interface(CONST_INTER);
4179 %}
4180
4181 operand immL_bitmask()
4182 %{
4183 predicate((n->get_long() != 0)
4184 && ((n->get_long() & 0xc000000000000000l) == 0)
4185 && is_power_of_2(n->get_long() + 1));
4186 match(ConL);
4187
4188 op_cost(0);
4189 format %{ %}
4190 interface(CONST_INTER);
4191 %}
4192
4193 operand immI_bitmask()
4194 %{
4195 predicate((n->get_int() != 0)
4196 && ((n->get_int() & 0xc0000000) == 0)
4197 && is_power_of_2(n->get_int() + 1));
4198 match(ConI);
4199
4200 op_cost(0);
4201 format %{ %}
4202 interface(CONST_INTER);
4203 %}
4204
4205 operand immL_positive_bitmaskI()
4206 %{
4207 predicate((n->get_long() != 0)
4208 && ((julong)n->get_long() < 0x80000000ULL)
4209 && is_power_of_2(n->get_long() + 1));
4210 match(ConL);
4211
4212 op_cost(0);
4213 format %{ %}
4214 interface(CONST_INTER);
4215 %}
4216
4217 // Scale values for scaled offset addressing modes (up to long but not quad)
4218 operand immIScale()
4219 %{
4220 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4221 match(ConI);
4222
4223 op_cost(0);
4224 format %{ %}
4225 interface(CONST_INTER);
4226 %}
4227
4228 // 5 bit signed integer
4229 operand immI5()
4230 %{
4231 predicate(Assembler::is_simm(n->get_int(), 5));
4232 match(ConI);
4233
4234 op_cost(0);
4235 format %{ %}
4236 interface(CONST_INTER);
4237 %}
4238
4239 // 7 bit unsigned integer
4240 operand immIU7()
4241 %{
4242 predicate(Assembler::is_uimm(n->get_int(), 7));
4243 match(ConI);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 // Offset for scaled or unscaled immediate loads and stores
4251 operand immIOffset()
4252 %{
4253 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4254 match(ConI);
4255
4256 op_cost(0);
4257 format %{ %}
4258 interface(CONST_INTER);
4259 %}
4260
4261 operand immIOffset1()
4262 %{
4263 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4264 match(ConI);
4265
4266 op_cost(0);
4267 format %{ %}
4268 interface(CONST_INTER);
4269 %}
4270
4271 operand immIOffset2()
4272 %{
4273 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4274 match(ConI);
4275
4276 op_cost(0);
4277 format %{ %}
4278 interface(CONST_INTER);
4279 %}
4280
4281 operand immIOffset4()
4282 %{
4283 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4284 match(ConI);
4285
4286 op_cost(0);
4287 format %{ %}
4288 interface(CONST_INTER);
4289 %}
4290
4291 operand immIOffset8()
4292 %{
4293 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4294 match(ConI);
4295
4296 op_cost(0);
4297 format %{ %}
4298 interface(CONST_INTER);
4299 %}
4300
4301 operand immIOffset16()
4302 %{
4303 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4304 match(ConI);
4305
4306 op_cost(0);
4307 format %{ %}
4308 interface(CONST_INTER);
4309 %}
4310
4311 operand immLOffset()
4312 %{
4313 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4314 match(ConL);
4315
4316 op_cost(0);
4317 format %{ %}
4318 interface(CONST_INTER);
4319 %}
4320
4321 operand immLoffset1()
4322 %{
4323 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4324 match(ConL);
4325
4326 op_cost(0);
4327 format %{ %}
4328 interface(CONST_INTER);
4329 %}
4330
4331 operand immLoffset2()
4332 %{
4333 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4334 match(ConL);
4335
4336 op_cost(0);
4337 format %{ %}
4338 interface(CONST_INTER);
4339 %}
4340
4341 operand immLoffset4()
4342 %{
4343 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4344 match(ConL);
4345
4346 op_cost(0);
4347 format %{ %}
4348 interface(CONST_INTER);
4349 %}
4350
4351 operand immLoffset8()
4352 %{
4353 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4354 match(ConL);
4355
4356 op_cost(0);
4357 format %{ %}
4358 interface(CONST_INTER);
4359 %}
4360
4361 operand immLoffset16()
4362 %{
4363 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4364 match(ConL);
4365
4366 op_cost(0);
4367 format %{ %}
4368 interface(CONST_INTER);
4369 %}
4370
4371 // 5 bit signed long integer
4372 operand immL5()
4373 %{
4374 predicate(Assembler::is_simm(n->get_long(), 5));
4375 match(ConL);
4376
4377 op_cost(0);
4378 format %{ %}
4379 interface(CONST_INTER);
4380 %}
4381
4382 // 7 bit unsigned long integer
4383 operand immLU7()
4384 %{
4385 predicate(Assembler::is_uimm(n->get_long(), 7));
4386 match(ConL);
4387
4388 op_cost(0);
4389 format %{ %}
4390 interface(CONST_INTER);
4391 %}
4392
4393 // 8 bit signed value.
4394 operand immI8()
4395 %{
4396 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4397 match(ConI);
4398
4399 op_cost(0);
4400 format %{ %}
4401 interface(CONST_INTER);
4402 %}
4403
4404 // 8 bit signed value (simm8), or #simm8 LSL 8.
4405 operand immIDupV()
4406 %{
4407 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4408 match(ConI);
4409
4410 op_cost(0);
4411 format %{ %}
4412 interface(CONST_INTER);
4413 %}
4414
4415 // 8 bit signed value (simm8), or #simm8 LSL 8.
4416 operand immLDupV()
4417 %{
4418 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4419 match(ConL);
4420
4421 op_cost(0);
4422 format %{ %}
4423 interface(CONST_INTER);
4424 %}
4425
4426 // 8 bit signed value (simm8), or #simm8 LSL 8.
4427 operand immHDupV()
4428 %{
4429 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4430 match(ConH);
4431
4432 op_cost(0);
4433 format %{ %}
4434 interface(CONST_INTER);
4435 %}
4436
4437 // 8 bit integer valid for vector add sub immediate
4438 operand immBAddSubV()
4439 %{
4440 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4441 match(ConI);
4442
4443 op_cost(0);
4444 format %{ %}
4445 interface(CONST_INTER);
4446 %}
4447
4448 // 32 bit integer valid for add sub immediate
4449 operand immIAddSub()
4450 %{
4451 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4452 match(ConI);
4453 op_cost(0);
4454 format %{ %}
4455 interface(CONST_INTER);
4456 %}
4457
4458 // 32 bit integer valid for vector add sub immediate
4459 operand immIAddSubV()
4460 %{
4461 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4462 match(ConI);
4463
4464 op_cost(0);
4465 format %{ %}
4466 interface(CONST_INTER);
4467 %}
4468
4469 // 32 bit unsigned integer valid for logical immediate
4470
4471 operand immBLog()
4472 %{
4473 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4474 match(ConI);
4475
4476 op_cost(0);
4477 format %{ %}
4478 interface(CONST_INTER);
4479 %}
4480
4481 operand immSLog()
4482 %{
4483 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4484 match(ConI);
4485
4486 op_cost(0);
4487 format %{ %}
4488 interface(CONST_INTER);
4489 %}
4490
4491 operand immILog()
4492 %{
4493 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4494 match(ConI);
4495
4496 op_cost(0);
4497 format %{ %}
4498 interface(CONST_INTER);
4499 %}
4500
4501 // Integer operands 64 bit
4502 // 64 bit immediate
4503 operand immL()
4504 %{
4505 match(ConL);
4506
4507 op_cost(0);
4508 format %{ %}
4509 interface(CONST_INTER);
4510 %}
4511
4512 // 64 bit zero
4513 operand immL0()
4514 %{
4515 predicate(n->get_long() == 0);
4516 match(ConL);
4517
4518 op_cost(0);
4519 format %{ %}
4520 interface(CONST_INTER);
4521 %}
4522
4523 // 64 bit unit decrement
4524 operand immL_M1()
4525 %{
4526 predicate(n->get_long() == -1);
4527 match(ConL);
4528
4529 op_cost(0);
4530 format %{ %}
4531 interface(CONST_INTER);
4532 %}
4533
4534 // 64 bit integer valid for add sub immediate
4535 operand immLAddSub()
4536 %{
4537 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4538 match(ConL);
4539 op_cost(0);
4540 format %{ %}
4541 interface(CONST_INTER);
4542 %}
4543
4544 // 64 bit integer valid for addv subv immediate
4545 operand immLAddSubV()
4546 %{
4547 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4548 match(ConL);
4549
4550 op_cost(0);
4551 format %{ %}
4552 interface(CONST_INTER);
4553 %}
4554
4555 // 64 bit integer valid for logical immediate
4556 operand immLLog()
4557 %{
4558 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4559 match(ConL);
4560 op_cost(0);
4561 format %{ %}
4562 interface(CONST_INTER);
4563 %}
4564
4565 // Long Immediate: low 32-bit mask
4566 operand immL_32bits()
4567 %{
4568 predicate(n->get_long() == 0xFFFFFFFFL);
4569 match(ConL);
4570 op_cost(0);
4571 format %{ %}
4572 interface(CONST_INTER);
4573 %}
4574
4575 // Pointer operands
4576 // Pointer Immediate
4577 operand immP()
4578 %{
4579 match(ConP);
4580
4581 op_cost(0);
4582 format %{ %}
4583 interface(CONST_INTER);
4584 %}
4585
4586 // nullptr Pointer Immediate
4587 operand immP0()
4588 %{
4589 predicate(n->get_ptr() == 0);
4590 match(ConP);
4591
4592 op_cost(0);
4593 format %{ %}
4594 interface(CONST_INTER);
4595 %}
4596
4597 // Pointer Immediate One
4598 // this is used in object initialization (initial object header)
4599 operand immP_1()
4600 %{
4601 predicate(n->get_ptr() == 1);
4602 match(ConP);
4603
4604 op_cost(0);
4605 format %{ %}
4606 interface(CONST_INTER);
4607 %}
4608
4609 // Float and Double operands
4610 // Double Immediate
4611 operand immD()
4612 %{
4613 match(ConD);
4614 op_cost(0);
4615 format %{ %}
4616 interface(CONST_INTER);
4617 %}
4618
4619 // Double Immediate: +0.0d
4620 operand immD0()
4621 %{
4622 predicate(jlong_cast(n->getd()) == 0);
4623 match(ConD);
4624
4625 op_cost(0);
4626 format %{ %}
4627 interface(CONST_INTER);
4628 %}
4629
4630 // constant 'double +0.0'.
4631 operand immDPacked()
4632 %{
4633 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4634 match(ConD);
4635 op_cost(0);
4636 format %{ %}
4637 interface(CONST_INTER);
4638 %}
4639
4640 // Float Immediate
4641 operand immF()
4642 %{
4643 match(ConF);
4644 op_cost(0);
4645 format %{ %}
4646 interface(CONST_INTER);
4647 %}
4648
4649 // Float Immediate: +0.0f.
4650 operand immF0()
4651 %{
4652 predicate(jint_cast(n->getf()) == 0);
4653 match(ConF);
4654
4655 op_cost(0);
4656 format %{ %}
4657 interface(CONST_INTER);
4658 %}
4659
4660 // Half Float (FP16) Immediate
4661 operand immH()
4662 %{
4663 match(ConH);
4664 op_cost(0);
4665 format %{ %}
4666 interface(CONST_INTER);
4667 %}
4668
4669 //
4670 operand immFPacked()
4671 %{
4672 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4673 match(ConF);
4674 op_cost(0);
4675 format %{ %}
4676 interface(CONST_INTER);
4677 %}
4678
4679 // Narrow pointer operands
4680 // Narrow Pointer Immediate
4681 operand immN()
4682 %{
4683 match(ConN);
4684
4685 op_cost(0);
4686 format %{ %}
4687 interface(CONST_INTER);
4688 %}
4689
4690 // Narrow nullptr Pointer Immediate
4691 operand immN0()
4692 %{
4693 predicate(n->get_narrowcon() == 0);
4694 match(ConN);
4695
4696 op_cost(0);
4697 format %{ %}
4698 interface(CONST_INTER);
4699 %}
4700
4701 operand immNKlass()
4702 %{
4703 match(ConNKlass);
4704
4705 op_cost(0);
4706 format %{ %}
4707 interface(CONST_INTER);
4708 %}
4709
4710 // Integer 32 bit Register Operands
4711 // Integer 32 bitRegister (excludes SP)
4712 operand iRegI()
4713 %{
4714 constraint(ALLOC_IN_RC(any_reg32));
4715 match(RegI);
4716 match(iRegINoSp);
4717 op_cost(0);
4718 format %{ %}
4719 interface(REG_INTER);
4720 %}
4721
4722 // Integer 32 bit Register not Special
4723 operand iRegINoSp()
4724 %{
4725 constraint(ALLOC_IN_RC(no_special_reg32));
4726 match(RegI);
4727 op_cost(0);
4728 format %{ %}
4729 interface(REG_INTER);
4730 %}
4731
4732 // Integer 64 bit Register Operands
4733 // Integer 64 bit Register (includes SP)
4734 operand iRegL()
4735 %{
4736 constraint(ALLOC_IN_RC(any_reg));
4737 match(RegL);
4738 match(iRegLNoSp);
4739 op_cost(0);
4740 format %{ %}
4741 interface(REG_INTER);
4742 %}
4743
4744 // Integer 64 bit Register not Special
4745 operand iRegLNoSp()
4746 %{
4747 constraint(ALLOC_IN_RC(no_special_reg));
4748 match(RegL);
4749 match(iRegL_R0);
4750 format %{ %}
4751 interface(REG_INTER);
4752 %}
4753
4754 // Pointer Register Operands
4755 // Pointer Register
4756 operand iRegP()
4757 %{
4758 constraint(ALLOC_IN_RC(ptr_reg));
4759 match(RegP);
4760 match(iRegPNoSp);
4761 match(iRegP_R0);
4762 //match(iRegP_R2);
4763 //match(iRegP_R4);
4764 match(iRegP_R5);
4765 match(thread_RegP);
4766 op_cost(0);
4767 format %{ %}
4768 interface(REG_INTER);
4769 %}
4770
4771 // Pointer 64 bit Register not Special
4772 operand iRegPNoSp()
4773 %{
4774 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4775 match(RegP);
4776 // match(iRegP);
4777 // match(iRegP_R0);
4778 // match(iRegP_R2);
4779 // match(iRegP_R4);
4780 // match(iRegP_R5);
4781 // match(thread_RegP);
4782 op_cost(0);
4783 format %{ %}
4784 interface(REG_INTER);
4785 %}
4786
4787 // This operand is not allowed to use rfp even if
4788 // rfp is not used to hold the frame pointer.
4789 operand iRegPNoSpNoRfp()
4790 %{
4791 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4792 match(RegP);
4793 match(iRegPNoSp);
4794 op_cost(0);
4795 format %{ %}
4796 interface(REG_INTER);
4797 %}
4798
4799 // Pointer 64 bit Register R0 only
4800 operand iRegP_R0()
4801 %{
4802 constraint(ALLOC_IN_RC(r0_reg));
4803 match(RegP);
4804 // match(iRegP);
4805 match(iRegPNoSp);
4806 op_cost(0);
4807 format %{ %}
4808 interface(REG_INTER);
4809 %}
4810
4811 // Pointer 64 bit Register R1 only
4812 operand iRegP_R1()
4813 %{
4814 constraint(ALLOC_IN_RC(r1_reg));
4815 match(RegP);
4816 // match(iRegP);
4817 match(iRegPNoSp);
4818 op_cost(0);
4819 format %{ %}
4820 interface(REG_INTER);
4821 %}
4822
4823 // Pointer 64 bit Register R2 only
4824 operand iRegP_R2()
4825 %{
4826 constraint(ALLOC_IN_RC(r2_reg));
4827 match(RegP);
4828 // match(iRegP);
4829 match(iRegPNoSp);
4830 op_cost(0);
4831 format %{ %}
4832 interface(REG_INTER);
4833 %}
4834
4835 // Pointer 64 bit Register R3 only
4836 operand iRegP_R3()
4837 %{
4838 constraint(ALLOC_IN_RC(r3_reg));
4839 match(RegP);
4840 // match(iRegP);
4841 match(iRegPNoSp);
4842 op_cost(0);
4843 format %{ %}
4844 interface(REG_INTER);
4845 %}
4846
4847 // Pointer 64 bit Register R4 only
4848 operand iRegP_R4()
4849 %{
4850 constraint(ALLOC_IN_RC(r4_reg));
4851 match(RegP);
4852 // match(iRegP);
4853 match(iRegPNoSp);
4854 op_cost(0);
4855 format %{ %}
4856 interface(REG_INTER);
4857 %}
4858
4859 // Pointer 64 bit Register R5 only
4860 operand iRegP_R5()
4861 %{
4862 constraint(ALLOC_IN_RC(r5_reg));
4863 match(RegP);
4864 // match(iRegP);
4865 match(iRegPNoSp);
4866 op_cost(0);
4867 format %{ %}
4868 interface(REG_INTER);
4869 %}
4870
4871 // Pointer 64 bit Register R10 only
4872 operand iRegP_R10()
4873 %{
4874 constraint(ALLOC_IN_RC(r10_reg));
4875 match(RegP);
4876 // match(iRegP);
4877 match(iRegPNoSp);
4878 op_cost(0);
4879 format %{ %}
4880 interface(REG_INTER);
4881 %}
4882
4883 // Long 64 bit Register R0 only
4884 operand iRegL_R0()
4885 %{
4886 constraint(ALLOC_IN_RC(r0_reg));
4887 match(RegL);
4888 match(iRegLNoSp);
4889 op_cost(0);
4890 format %{ %}
4891 interface(REG_INTER);
4892 %}
4893
4894 // Long 64 bit Register R11 only
4895 operand iRegL_R11()
4896 %{
4897 constraint(ALLOC_IN_RC(r11_reg));
4898 match(RegL);
4899 match(iRegLNoSp);
4900 op_cost(0);
4901 format %{ %}
4902 interface(REG_INTER);
4903 %}
4904
4905 // Register R0 only
4906 operand iRegI_R0()
4907 %{
4908 constraint(ALLOC_IN_RC(int_r0_reg));
4909 match(RegI);
4910 match(iRegINoSp);
4911 op_cost(0);
4912 format %{ %}
4913 interface(REG_INTER);
4914 %}
4915
4916 // Register R2 only
4917 operand iRegI_R2()
4918 %{
4919 constraint(ALLOC_IN_RC(int_r2_reg));
4920 match(RegI);
4921 match(iRegINoSp);
4922 op_cost(0);
4923 format %{ %}
4924 interface(REG_INTER);
4925 %}
4926
4927 // Register R3 only
4928 operand iRegI_R3()
4929 %{
4930 constraint(ALLOC_IN_RC(int_r3_reg));
4931 match(RegI);
4932 match(iRegINoSp);
4933 op_cost(0);
4934 format %{ %}
4935 interface(REG_INTER);
4936 %}
4937
4938
4939 // Register R4 only
4940 operand iRegI_R4()
4941 %{
4942 constraint(ALLOC_IN_RC(int_r4_reg));
4943 match(RegI);
4944 match(iRegINoSp);
4945 op_cost(0);
4946 format %{ %}
4947 interface(REG_INTER);
4948 %}
4949
4950
4951 // Pointer Register Operands
4952 // Narrow Pointer Register
4953 operand iRegN()
4954 %{
4955 constraint(ALLOC_IN_RC(any_reg32));
4956 match(RegN);
4957 match(iRegNNoSp);
4958 op_cost(0);
4959 format %{ %}
4960 interface(REG_INTER);
4961 %}
4962
4963 // Integer 64 bit Register not Special
4964 operand iRegNNoSp()
4965 %{
4966 constraint(ALLOC_IN_RC(no_special_reg32));
4967 match(RegN);
4968 op_cost(0);
4969 format %{ %}
4970 interface(REG_INTER);
4971 %}
4972
4973 // Float Register
4974 // Float register operands
4975 operand vRegF()
4976 %{
4977 constraint(ALLOC_IN_RC(float_reg));
4978 match(RegF);
4979
4980 op_cost(0);
4981 format %{ %}
4982 interface(REG_INTER);
4983 %}
4984
4985 // Double Register
4986 // Double register operands
4987 operand vRegD()
4988 %{
4989 constraint(ALLOC_IN_RC(double_reg));
4990 match(RegD);
4991
4992 op_cost(0);
4993 format %{ %}
4994 interface(REG_INTER);
4995 %}
4996
4997 // Generic vector class. This will be used for
4998 // all vector operands, including NEON and SVE.
4999 operand vReg()
5000 %{
5001 constraint(ALLOC_IN_RC(dynamic));
5002 match(VecA);
5003 match(VecD);
5004 match(VecX);
5005
5006 op_cost(0);
5007 format %{ %}
5008 interface(REG_INTER);
5009 %}
5010
5011 operand vReg_V10()
5012 %{
5013 constraint(ALLOC_IN_RC(v10_veca_reg));
5014 match(vReg);
5015
5016 op_cost(0);
5017 format %{ %}
5018 interface(REG_INTER);
5019 %}
5020
5021 operand vReg_V11()
5022 %{
5023 constraint(ALLOC_IN_RC(v11_veca_reg));
5024 match(vReg);
5025
5026 op_cost(0);
5027 format %{ %}
5028 interface(REG_INTER);
5029 %}
5030
5031 operand vReg_V12()
5032 %{
5033 constraint(ALLOC_IN_RC(v12_veca_reg));
5034 match(vReg);
5035
5036 op_cost(0);
5037 format %{ %}
5038 interface(REG_INTER);
5039 %}
5040
5041 operand vReg_V13()
5042 %{
5043 constraint(ALLOC_IN_RC(v13_veca_reg));
5044 match(vReg);
5045
5046 op_cost(0);
5047 format %{ %}
5048 interface(REG_INTER);
5049 %}
5050
5051 operand vReg_V17()
5052 %{
5053 constraint(ALLOC_IN_RC(v17_veca_reg));
5054 match(vReg);
5055
5056 op_cost(0);
5057 format %{ %}
5058 interface(REG_INTER);
5059 %}
5060
5061 operand vReg_V18()
5062 %{
5063 constraint(ALLOC_IN_RC(v18_veca_reg));
5064 match(vReg);
5065
5066 op_cost(0);
5067 format %{ %}
5068 interface(REG_INTER);
5069 %}
5070
5071 operand vReg_V23()
5072 %{
5073 constraint(ALLOC_IN_RC(v23_veca_reg));
5074 match(vReg);
5075
5076 op_cost(0);
5077 format %{ %}
5078 interface(REG_INTER);
5079 %}
5080
5081 operand vReg_V24()
5082 %{
5083 constraint(ALLOC_IN_RC(v24_veca_reg));
5084 match(vReg);
5085
5086 op_cost(0);
5087 format %{ %}
5088 interface(REG_INTER);
5089 %}
5090
5091 operand vecA()
5092 %{
5093 constraint(ALLOC_IN_RC(vectora_reg));
5094 match(VecA);
5095
5096 op_cost(0);
5097 format %{ %}
5098 interface(REG_INTER);
5099 %}
5100
5101 operand vecD()
5102 %{
5103 constraint(ALLOC_IN_RC(vectord_reg));
5104 match(VecD);
5105
5106 op_cost(0);
5107 format %{ %}
5108 interface(REG_INTER);
5109 %}
5110
5111 operand vecX()
5112 %{
5113 constraint(ALLOC_IN_RC(vectorx_reg));
5114 match(VecX);
5115
5116 op_cost(0);
5117 format %{ %}
5118 interface(REG_INTER);
5119 %}
5120
5121 operand vRegD_V0()
5122 %{
5123 constraint(ALLOC_IN_RC(v0_reg));
5124 match(RegD);
5125 op_cost(0);
5126 format %{ %}
5127 interface(REG_INTER);
5128 %}
5129
5130 operand vRegD_V1()
5131 %{
5132 constraint(ALLOC_IN_RC(v1_reg));
5133 match(RegD);
5134 op_cost(0);
5135 format %{ %}
5136 interface(REG_INTER);
5137 %}
5138
5139 operand vRegD_V2()
5140 %{
5141 constraint(ALLOC_IN_RC(v2_reg));
5142 match(RegD);
5143 op_cost(0);
5144 format %{ %}
5145 interface(REG_INTER);
5146 %}
5147
5148 operand vRegD_V3()
5149 %{
5150 constraint(ALLOC_IN_RC(v3_reg));
5151 match(RegD);
5152 op_cost(0);
5153 format %{ %}
5154 interface(REG_INTER);
5155 %}
5156
5157 operand vRegD_V4()
5158 %{
5159 constraint(ALLOC_IN_RC(v4_reg));
5160 match(RegD);
5161 op_cost(0);
5162 format %{ %}
5163 interface(REG_INTER);
5164 %}
5165
5166 operand vRegD_V5()
5167 %{
5168 constraint(ALLOC_IN_RC(v5_reg));
5169 match(RegD);
5170 op_cost(0);
5171 format %{ %}
5172 interface(REG_INTER);
5173 %}
5174
5175 operand vRegD_V6()
5176 %{
5177 constraint(ALLOC_IN_RC(v6_reg));
5178 match(RegD);
5179 op_cost(0);
5180 format %{ %}
5181 interface(REG_INTER);
5182 %}
5183
5184 operand vRegD_V7()
5185 %{
5186 constraint(ALLOC_IN_RC(v7_reg));
5187 match(RegD);
5188 op_cost(0);
5189 format %{ %}
5190 interface(REG_INTER);
5191 %}
5192
5193 operand vRegD_V12()
5194 %{
5195 constraint(ALLOC_IN_RC(v12_reg));
5196 match(RegD);
5197 op_cost(0);
5198 format %{ %}
5199 interface(REG_INTER);
5200 %}
5201
5202 operand vRegD_V13()
5203 %{
5204 constraint(ALLOC_IN_RC(v13_reg));
5205 match(RegD);
5206 op_cost(0);
5207 format %{ %}
5208 interface(REG_INTER);
5209 %}
5210
5211 operand pReg()
5212 %{
5213 constraint(ALLOC_IN_RC(pr_reg));
5214 match(RegVectMask);
5215 match(pRegGov);
5216 op_cost(0);
5217 format %{ %}
5218 interface(REG_INTER);
5219 %}
5220
5221 operand pRegGov()
5222 %{
5223 constraint(ALLOC_IN_RC(gov_pr));
5224 match(RegVectMask);
5225 match(pReg);
5226 op_cost(0);
5227 format %{ %}
5228 interface(REG_INTER);
5229 %}
5230
5231 operand pRegGov_P0()
5232 %{
5233 constraint(ALLOC_IN_RC(p0_reg));
5234 match(RegVectMask);
5235 op_cost(0);
5236 format %{ %}
5237 interface(REG_INTER);
5238 %}
5239
5240 operand pRegGov_P1()
5241 %{
5242 constraint(ALLOC_IN_RC(p1_reg));
5243 match(RegVectMask);
5244 op_cost(0);
5245 format %{ %}
5246 interface(REG_INTER);
5247 %}
5248
5249 // Flags register, used as output of signed compare instructions
5250
5251 // note that on AArch64 we also use this register as the output for
5252 // for floating point compare instructions (CmpF CmpD). this ensures
5253 // that ordered inequality tests use GT, GE, LT or LE none of which
5254 // pass through cases where the result is unordered i.e. one or both
5255 // inputs to the compare is a NaN. this means that the ideal code can
5256 // replace e.g. a GT with an LE and not end up capturing the NaN case
5257 // (where the comparison should always fail). EQ and NE tests are
5258 // always generated in ideal code so that unordered folds into the NE
5259 // case, matching the behaviour of AArch64 NE.
5260 //
5261 // This differs from x86 where the outputs of FP compares use a
5262 // special FP flags registers and where compares based on this
5263 // register are distinguished into ordered inequalities (cmpOpUCF) and
5264 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5265 // to explicitly handle the unordered case in branches. x86 also has
5266 // to include extra CMoveX rules to accept a cmpOpUCF input.
5267
5268 operand rFlagsReg()
5269 %{
5270 constraint(ALLOC_IN_RC(int_flags));
5271 match(RegFlags);
5272
5273 op_cost(0);
5274 format %{ "RFLAGS" %}
5275 interface(REG_INTER);
5276 %}
5277
5278 // Flags register, used as output of unsigned compare instructions
5279 operand rFlagsRegU()
5280 %{
5281 constraint(ALLOC_IN_RC(int_flags));
5282 match(RegFlags);
5283
5284 op_cost(0);
5285 format %{ "RFLAGSU" %}
5286 interface(REG_INTER);
5287 %}
5288
5289 // Special Registers
5290
5291 // Method Register
5292 operand inline_cache_RegP(iRegP reg)
5293 %{
5294 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5295 match(reg);
5296 match(iRegPNoSp);
5297 op_cost(0);
5298 format %{ %}
5299 interface(REG_INTER);
5300 %}
5301
5302 // Thread Register
5303 operand thread_RegP(iRegP reg)
5304 %{
5305 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5306 match(reg);
5307 op_cost(0);
5308 format %{ %}
5309 interface(REG_INTER);
5310 %}
5311
5312 //----------Memory Operands----------------------------------------------------
5313
5314 operand indirect(iRegP reg)
5315 %{
5316 constraint(ALLOC_IN_RC(ptr_reg));
5317 match(reg);
5318 op_cost(0);
5319 format %{ "[$reg]" %}
5320 interface(MEMORY_INTER) %{
5321 base($reg);
5322 index(0xffffffff);
5323 scale(0x0);
5324 disp(0x0);
5325 %}
5326 %}
5327
5328 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5329 %{
5330 constraint(ALLOC_IN_RC(ptr_reg));
5331 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5332 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5333 op_cost(0);
5334 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5335 interface(MEMORY_INTER) %{
5336 base($reg);
5337 index($ireg);
5338 scale($scale);
5339 disp(0x0);
5340 %}
5341 %}
5342
5343 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5344 %{
5345 constraint(ALLOC_IN_RC(ptr_reg));
5346 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5347 match(AddP reg (LShiftL lreg scale));
5348 op_cost(0);
5349 format %{ "$reg, $lreg lsl($scale)" %}
5350 interface(MEMORY_INTER) %{
5351 base($reg);
5352 index($lreg);
5353 scale($scale);
5354 disp(0x0);
5355 %}
5356 %}
5357
5358 operand indIndexI2L(iRegP reg, iRegI ireg)
5359 %{
5360 constraint(ALLOC_IN_RC(ptr_reg));
5361 match(AddP reg (ConvI2L ireg));
5362 op_cost(0);
5363 format %{ "$reg, $ireg, 0, I2L" %}
5364 interface(MEMORY_INTER) %{
5365 base($reg);
5366 index($ireg);
5367 scale(0x0);
5368 disp(0x0);
5369 %}
5370 %}
5371
5372 operand indIndex(iRegP reg, iRegL lreg)
5373 %{
5374 constraint(ALLOC_IN_RC(ptr_reg));
5375 match(AddP reg lreg);
5376 op_cost(0);
5377 format %{ "$reg, $lreg" %}
5378 interface(MEMORY_INTER) %{
5379 base($reg);
5380 index($lreg);
5381 scale(0x0);
5382 disp(0x0);
5383 %}
5384 %}
5385
5386 operand indOffI1(iRegP reg, immIOffset1 off)
5387 %{
5388 constraint(ALLOC_IN_RC(ptr_reg));
5389 match(AddP reg off);
5390 op_cost(0);
5391 format %{ "[$reg, $off]" %}
5392 interface(MEMORY_INTER) %{
5393 base($reg);
5394 index(0xffffffff);
5395 scale(0x0);
5396 disp($off);
5397 %}
5398 %}
5399
5400 operand indOffI2(iRegP reg, immIOffset2 off)
5401 %{
5402 constraint(ALLOC_IN_RC(ptr_reg));
5403 match(AddP reg off);
5404 op_cost(0);
5405 format %{ "[$reg, $off]" %}
5406 interface(MEMORY_INTER) %{
5407 base($reg);
5408 index(0xffffffff);
5409 scale(0x0);
5410 disp($off);
5411 %}
5412 %}
5413
5414 operand indOffI4(iRegP reg, immIOffset4 off)
5415 %{
5416 constraint(ALLOC_IN_RC(ptr_reg));
5417 match(AddP reg off);
5418 op_cost(0);
5419 format %{ "[$reg, $off]" %}
5420 interface(MEMORY_INTER) %{
5421 base($reg);
5422 index(0xffffffff);
5423 scale(0x0);
5424 disp($off);
5425 %}
5426 %}
5427
5428 operand indOffI8(iRegP reg, immIOffset8 off)
5429 %{
5430 constraint(ALLOC_IN_RC(ptr_reg));
5431 match(AddP reg off);
5432 op_cost(0);
5433 format %{ "[$reg, $off]" %}
5434 interface(MEMORY_INTER) %{
5435 base($reg);
5436 index(0xffffffff);
5437 scale(0x0);
5438 disp($off);
5439 %}
5440 %}
5441
5442 operand indOffI16(iRegP reg, immIOffset16 off)
5443 %{
5444 constraint(ALLOC_IN_RC(ptr_reg));
5445 match(AddP reg off);
5446 op_cost(0);
5447 format %{ "[$reg, $off]" %}
5448 interface(MEMORY_INTER) %{
5449 base($reg);
5450 index(0xffffffff);
5451 scale(0x0);
5452 disp($off);
5453 %}
5454 %}
5455
5456 operand indOffL1(iRegP reg, immLoffset1 off)
5457 %{
5458 constraint(ALLOC_IN_RC(ptr_reg));
5459 match(AddP reg off);
5460 op_cost(0);
5461 format %{ "[$reg, $off]" %}
5462 interface(MEMORY_INTER) %{
5463 base($reg);
5464 index(0xffffffff);
5465 scale(0x0);
5466 disp($off);
5467 %}
5468 %}
5469
5470 operand indOffL2(iRegP reg, immLoffset2 off)
5471 %{
5472 constraint(ALLOC_IN_RC(ptr_reg));
5473 match(AddP reg off);
5474 op_cost(0);
5475 format %{ "[$reg, $off]" %}
5476 interface(MEMORY_INTER) %{
5477 base($reg);
5478 index(0xffffffff);
5479 scale(0x0);
5480 disp($off);
5481 %}
5482 %}
5483
5484 operand indOffL4(iRegP reg, immLoffset4 off)
5485 %{
5486 constraint(ALLOC_IN_RC(ptr_reg));
5487 match(AddP reg off);
5488 op_cost(0);
5489 format %{ "[$reg, $off]" %}
5490 interface(MEMORY_INTER) %{
5491 base($reg);
5492 index(0xffffffff);
5493 scale(0x0);
5494 disp($off);
5495 %}
5496 %}
5497
5498 operand indOffL8(iRegP reg, immLoffset8 off)
5499 %{
5500 constraint(ALLOC_IN_RC(ptr_reg));
5501 match(AddP reg off);
5502 op_cost(0);
5503 format %{ "[$reg, $off]" %}
5504 interface(MEMORY_INTER) %{
5505 base($reg);
5506 index(0xffffffff);
5507 scale(0x0);
5508 disp($off);
5509 %}
5510 %}
5511
5512 operand indOffL16(iRegP reg, immLoffset16 off)
5513 %{
5514 constraint(ALLOC_IN_RC(ptr_reg));
5515 match(AddP reg off);
5516 op_cost(0);
5517 format %{ "[$reg, $off]" %}
5518 interface(MEMORY_INTER) %{
5519 base($reg);
5520 index(0xffffffff);
5521 scale(0x0);
5522 disp($off);
5523 %}
5524 %}
5525
5526 operand indirectX2P(iRegL reg)
5527 %{
5528 constraint(ALLOC_IN_RC(ptr_reg));
5529 match(CastX2P reg);
5530 op_cost(0);
5531 format %{ "[$reg]\t# long -> ptr" %}
5532 interface(MEMORY_INTER) %{
5533 base($reg);
5534 index(0xffffffff);
5535 scale(0x0);
5536 disp(0x0);
5537 %}
5538 %}
5539
5540 operand indOffX2P(iRegL reg, immLOffset off)
5541 %{
5542 constraint(ALLOC_IN_RC(ptr_reg));
5543 match(AddP (CastX2P reg) off);
5544 op_cost(0);
5545 format %{ "[$reg, $off]\t# long -> ptr" %}
5546 interface(MEMORY_INTER) %{
5547 base($reg);
5548 index(0xffffffff);
5549 scale(0x0);
5550 disp($off);
5551 %}
5552 %}
5553
5554 operand indirectN(iRegN reg)
5555 %{
5556 predicate(CompressedOops::shift() == 0);
5557 constraint(ALLOC_IN_RC(ptr_reg));
5558 match(DecodeN reg);
5559 op_cost(0);
5560 format %{ "[$reg]\t# narrow" %}
5561 interface(MEMORY_INTER) %{
5562 base($reg);
5563 index(0xffffffff);
5564 scale(0x0);
5565 disp(0x0);
5566 %}
5567 %}
5568
5569 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5570 %{
5571 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5572 constraint(ALLOC_IN_RC(ptr_reg));
5573 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5574 op_cost(0);
5575 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5576 interface(MEMORY_INTER) %{
5577 base($reg);
5578 index($ireg);
5579 scale($scale);
5580 disp(0x0);
5581 %}
5582 %}
5583
5584 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5585 %{
5586 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5587 constraint(ALLOC_IN_RC(ptr_reg));
5588 match(AddP (DecodeN reg) (LShiftL lreg scale));
5589 op_cost(0);
5590 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5591 interface(MEMORY_INTER) %{
5592 base($reg);
5593 index($lreg);
5594 scale($scale);
5595 disp(0x0);
5596 %}
5597 %}
5598
5599 operand indIndexI2LN(iRegN reg, iRegI ireg)
5600 %{
5601 predicate(CompressedOops::shift() == 0);
5602 constraint(ALLOC_IN_RC(ptr_reg));
5603 match(AddP (DecodeN reg) (ConvI2L ireg));
5604 op_cost(0);
5605 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5606 interface(MEMORY_INTER) %{
5607 base($reg);
5608 index($ireg);
5609 scale(0x0);
5610 disp(0x0);
5611 %}
5612 %}
5613
5614 operand indIndexN(iRegN reg, iRegL lreg)
5615 %{
5616 predicate(CompressedOops::shift() == 0);
5617 constraint(ALLOC_IN_RC(ptr_reg));
5618 match(AddP (DecodeN reg) lreg);
5619 op_cost(0);
5620 format %{ "$reg, $lreg\t# narrow" %}
5621 interface(MEMORY_INTER) %{
5622 base($reg);
5623 index($lreg);
5624 scale(0x0);
5625 disp(0x0);
5626 %}
5627 %}
5628
5629 operand indOffIN(iRegN reg, immIOffset off)
5630 %{
5631 predicate(CompressedOops::shift() == 0);
5632 constraint(ALLOC_IN_RC(ptr_reg));
5633 match(AddP (DecodeN reg) off);
5634 op_cost(0);
5635 format %{ "[$reg, $off]\t# narrow" %}
5636 interface(MEMORY_INTER) %{
5637 base($reg);
5638 index(0xffffffff);
5639 scale(0x0);
5640 disp($off);
5641 %}
5642 %}
5643
5644 operand indOffLN(iRegN reg, immLOffset off)
5645 %{
5646 predicate(CompressedOops::shift() == 0);
5647 constraint(ALLOC_IN_RC(ptr_reg));
5648 match(AddP (DecodeN reg) off);
5649 op_cost(0);
5650 format %{ "[$reg, $off]\t# narrow" %}
5651 interface(MEMORY_INTER) %{
5652 base($reg);
5653 index(0xffffffff);
5654 scale(0x0);
5655 disp($off);
5656 %}
5657 %}
5658
5659
5660 //----------Special Memory Operands--------------------------------------------
5661 // Stack Slot Operand - This operand is used for loading and storing temporary
5662 // values on the stack where a match requires a value to
5663 // flow through memory.
5664 operand stackSlotP(sRegP reg)
5665 %{
5666 constraint(ALLOC_IN_RC(stack_slots));
5667 op_cost(100);
5668 // No match rule because this operand is only generated in matching
5669 // match(RegP);
5670 format %{ "[$reg]" %}
5671 interface(MEMORY_INTER) %{
5672 base(0x1e); // RSP
5673 index(0x0); // No Index
5674 scale(0x0); // No Scale
5675 disp($reg); // Stack Offset
5676 %}
5677 %}
5678
5679 operand stackSlotI(sRegI reg)
5680 %{
5681 constraint(ALLOC_IN_RC(stack_slots));
5682 // No match rule because this operand is only generated in matching
5683 // match(RegI);
5684 format %{ "[$reg]" %}
5685 interface(MEMORY_INTER) %{
5686 base(0x1e); // RSP
5687 index(0x0); // No Index
5688 scale(0x0); // No Scale
5689 disp($reg); // Stack Offset
5690 %}
5691 %}
5692
5693 operand stackSlotF(sRegF reg)
5694 %{
5695 constraint(ALLOC_IN_RC(stack_slots));
5696 // No match rule because this operand is only generated in matching
5697 // match(RegF);
5698 format %{ "[$reg]" %}
5699 interface(MEMORY_INTER) %{
5700 base(0x1e); // RSP
5701 index(0x0); // No Index
5702 scale(0x0); // No Scale
5703 disp($reg); // Stack Offset
5704 %}
5705 %}
5706
5707 operand stackSlotD(sRegD reg)
5708 %{
5709 constraint(ALLOC_IN_RC(stack_slots));
5710 // No match rule because this operand is only generated in matching
5711 // match(RegD);
5712 format %{ "[$reg]" %}
5713 interface(MEMORY_INTER) %{
5714 base(0x1e); // RSP
5715 index(0x0); // No Index
5716 scale(0x0); // No Scale
5717 disp($reg); // Stack Offset
5718 %}
5719 %}
5720
5721 operand stackSlotL(sRegL reg)
5722 %{
5723 constraint(ALLOC_IN_RC(stack_slots));
5724 // No match rule because this operand is only generated in matching
5725 // match(RegL);
5726 format %{ "[$reg]" %}
5727 interface(MEMORY_INTER) %{
5728 base(0x1e); // RSP
5729 index(0x0); // No Index
5730 scale(0x0); // No Scale
5731 disp($reg); // Stack Offset
5732 %}
5733 %}
5734
5735 // Operands for expressing Control Flow
5736 // NOTE: Label is a predefined operand which should not be redefined in
5737 // the AD file. It is generically handled within the ADLC.
5738
5739 //----------Conditional Branch Operands----------------------------------------
5740 // Comparison Op - This is the operation of the comparison, and is limited to
5741 // the following set of codes:
5742 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5743 //
5744 // Other attributes of the comparison, such as unsignedness, are specified
5745 // by the comparison instruction that sets a condition code flags register.
5746 // That result is represented by a flags operand whose subtype is appropriate
5747 // to the unsignedness (etc.) of the comparison.
5748 //
5749 // Later, the instruction which matches both the Comparison Op (a Bool) and
5750 // the flags (produced by the Cmp) specifies the coding of the comparison op
5751 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5752
5753 // used for signed integral comparisons and fp comparisons
5754
5755 operand cmpOp()
5756 %{
5757 match(Bool);
5758
5759 format %{ "" %}
5760 interface(COND_INTER) %{
5761 equal(0x0, "eq");
5762 not_equal(0x1, "ne");
5763 less(0xb, "lt");
5764 greater_equal(0xa, "ge");
5765 less_equal(0xd, "le");
5766 greater(0xc, "gt");
5767 overflow(0x6, "vs");
5768 no_overflow(0x7, "vc");
5769 %}
5770 %}
5771
5772 // used for unsigned integral comparisons
5773
5774 operand cmpOpU()
5775 %{
5776 match(Bool);
5777
5778 format %{ "" %}
5779 interface(COND_INTER) %{
5780 equal(0x0, "eq");
5781 not_equal(0x1, "ne");
5782 less(0x3, "lo");
5783 greater_equal(0x2, "hs");
5784 less_equal(0x9, "ls");
5785 greater(0x8, "hi");
5786 overflow(0x6, "vs");
5787 no_overflow(0x7, "vc");
5788 %}
5789 %}
5790
5791 // used for certain integral comparisons which can be
5792 // converted to cbxx or tbxx instructions
5793
5794 operand cmpOpEqNe()
5795 %{
5796 match(Bool);
5797 op_cost(0);
5798 predicate(n->as_Bool()->_test._test == BoolTest::ne
5799 || n->as_Bool()->_test._test == BoolTest::eq);
5800
5801 format %{ "" %}
5802 interface(COND_INTER) %{
5803 equal(0x0, "eq");
5804 not_equal(0x1, "ne");
5805 less(0xb, "lt");
5806 greater_equal(0xa, "ge");
5807 less_equal(0xd, "le");
5808 greater(0xc, "gt");
5809 overflow(0x6, "vs");
5810 no_overflow(0x7, "vc");
5811 %}
5812 %}
5813
5814 // used for certain integral comparisons which can be
5815 // converted to cbxx or tbxx instructions
5816
5817 operand cmpOpLtGe()
5818 %{
5819 match(Bool);
5820 op_cost(0);
5821
5822 predicate(n->as_Bool()->_test._test == BoolTest::lt
5823 || n->as_Bool()->_test._test == BoolTest::ge);
5824
5825 format %{ "" %}
5826 interface(COND_INTER) %{
5827 equal(0x0, "eq");
5828 not_equal(0x1, "ne");
5829 less(0xb, "lt");
5830 greater_equal(0xa, "ge");
5831 less_equal(0xd, "le");
5832 greater(0xc, "gt");
5833 overflow(0x6, "vs");
5834 no_overflow(0x7, "vc");
5835 %}
5836 %}
5837
5838 // used for certain unsigned integral comparisons which can be
5839 // converted to cbxx or tbxx instructions
5840
5841 operand cmpOpUEqNeLeGt()
5842 %{
5843 match(Bool);
5844 op_cost(0);
5845
5846 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5847 n->as_Bool()->_test._test == BoolTest::ne ||
5848 n->as_Bool()->_test._test == BoolTest::le ||
5849 n->as_Bool()->_test._test == BoolTest::gt);
5850
5851 format %{ "" %}
5852 interface(COND_INTER) %{
5853 equal(0x0, "eq");
5854 not_equal(0x1, "ne");
5855 less(0x3, "lo");
5856 greater_equal(0x2, "hs");
5857 less_equal(0x9, "ls");
5858 greater(0x8, "hi");
5859 overflow(0x6, "vs");
5860 no_overflow(0x7, "vc");
5861 %}
5862 %}
5863
5864 // Special operand allowing long args to int ops to be truncated for free
5865
5866 operand iRegL2I(iRegL reg) %{
5867
5868 op_cost(0);
5869
5870 match(ConvL2I reg);
5871
5872 format %{ "l2i($reg)" %}
5873
5874 interface(REG_INTER)
5875 %}
5876
5877 operand iRegL2P(iRegL reg) %{
5878
5879 op_cost(0);
5880
5881 match(CastX2P reg);
5882
5883 format %{ "l2p($reg)" %}
5884
5885 interface(REG_INTER)
5886 %}
5887
5888 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5889 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5890 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5891 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5892
5893 //----------OPERAND CLASSES----------------------------------------------------
5894 // Operand Classes are groups of operands that are used as to simplify
5895 // instruction definitions by not requiring the AD writer to specify
5896 // separate instructions for every form of operand when the
5897 // instruction accepts multiple operand types with the same basic
5898 // encoding and format. The classic case of this is memory operands.
5899
5900 // memory is used to define read/write location for load/store
5901 // instruction defs. we can turn a memory op into an Address
5902
5903 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5904 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5905
5906 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5907 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5908
5909 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5910 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5911
5912 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5913 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5914
5915 // All of the memory operands. For the pipeline description.
5916 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5917 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5918 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5919
5920
5921 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5922 // operations. it allows the src to be either an iRegI or a (ConvL2I
5923 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5924 // can be elided because the 32-bit instruction will just employ the
5925 // lower 32 bits anyway.
5926 //
5927 // n.b. this does not elide all L2I conversions. if the truncated
5928 // value is consumed by more than one operation then the ConvL2I
5929 // cannot be bundled into the consuming nodes so an l2i gets planted
5930 // (actually a movw $dst $src) and the downstream instructions consume
5931 // the result of the l2i as an iRegI input. That's a shame since the
5932 // movw is actually redundant but its not too costly.
5933
5934 opclass iRegIorL2I(iRegI, iRegL2I);
5935 opclass iRegPorL2P(iRegP, iRegL2P);
5936
5937 //----------PIPELINE-----------------------------------------------------------
5938 // Rules which define the behavior of the target architectures pipeline.
5939
5940 // For specific pipelines, eg A53, define the stages of that pipeline
5941 //pipe_desc(ISS, EX1, EX2, WR);
5942 #define ISS S0
5943 #define EX1 S1
5944 #define EX2 S2
5945 #define WR S3
5946
5947 // Integer ALU reg operation
5948 pipeline %{
5949
5950 attributes %{
5951 // ARM instructions are of fixed length
5952 fixed_size_instructions; // Fixed size instructions TODO does
5953 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5954 // ARM instructions come in 32-bit word units
5955 instruction_unit_size = 4; // An instruction is 4 bytes long
5956 instruction_fetch_unit_size = 64; // The processor fetches one line
5957 instruction_fetch_units = 1; // of 64 bytes
5958 %}
5959
5960 // We don't use an actual pipeline model so don't care about resources
5961 // or description. we do use pipeline classes to introduce fixed
5962 // latencies
5963
5964 //----------RESOURCES----------------------------------------------------------
5965 // Resources are the functional units available to the machine
5966
5967 resources( INS0, INS1, INS01 = INS0 | INS1,
5968 ALU0, ALU1, ALU = ALU0 | ALU1,
5969 MAC,
5970 DIV,
5971 BRANCH,
5972 LDST,
5973 NEON_FP);
5974
5975 //----------PIPELINE DESCRIPTION-----------------------------------------------
5976 // Pipeline Description specifies the stages in the machine's pipeline
5977
5978 // Define the pipeline as a generic 6 stage pipeline
5979 pipe_desc(S0, S1, S2, S3, S4, S5);
5980
5981 //----------PIPELINE CLASSES---------------------------------------------------
5982 // Pipeline Classes describe the stages in which input and output are
5983 // referenced by the hardware pipeline.
5984
5985 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5986 %{
5987 single_instruction;
5988 src1 : S1(read);
5989 src2 : S2(read);
5990 dst : S5(write);
5991 INS01 : ISS;
5992 NEON_FP : S5;
5993 %}
5994
5995 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5996 %{
5997 single_instruction;
5998 src1 : S1(read);
5999 src2 : S2(read);
6000 dst : S5(write);
6001 INS01 : ISS;
6002 NEON_FP : S5;
6003 %}
6004
6005 pipe_class fp_uop_s(vRegF dst, vRegF src)
6006 %{
6007 single_instruction;
6008 src : S1(read);
6009 dst : S5(write);
6010 INS01 : ISS;
6011 NEON_FP : S5;
6012 %}
6013
6014 pipe_class fp_uop_d(vRegD dst, vRegD src)
6015 %{
6016 single_instruction;
6017 src : S1(read);
6018 dst : S5(write);
6019 INS01 : ISS;
6020 NEON_FP : S5;
6021 %}
6022
6023 pipe_class fp_d2f(vRegF dst, vRegD src)
6024 %{
6025 single_instruction;
6026 src : S1(read);
6027 dst : S5(write);
6028 INS01 : ISS;
6029 NEON_FP : S5;
6030 %}
6031
6032 pipe_class fp_f2d(vRegD dst, vRegF src)
6033 %{
6034 single_instruction;
6035 src : S1(read);
6036 dst : S5(write);
6037 INS01 : ISS;
6038 NEON_FP : S5;
6039 %}
6040
6041 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6042 %{
6043 single_instruction;
6044 src : S1(read);
6045 dst : S5(write);
6046 INS01 : ISS;
6047 NEON_FP : S5;
6048 %}
6049
6050 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6051 %{
6052 single_instruction;
6053 src : S1(read);
6054 dst : S5(write);
6055 INS01 : ISS;
6056 NEON_FP : S5;
6057 %}
6058
6059 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6060 %{
6061 single_instruction;
6062 src : S1(read);
6063 dst : S5(write);
6064 INS01 : ISS;
6065 NEON_FP : S5;
6066 %}
6067
6068 pipe_class fp_l2f(vRegF dst, iRegL src)
6069 %{
6070 single_instruction;
6071 src : S1(read);
6072 dst : S5(write);
6073 INS01 : ISS;
6074 NEON_FP : S5;
6075 %}
6076
6077 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6078 %{
6079 single_instruction;
6080 src : S1(read);
6081 dst : S5(write);
6082 INS01 : ISS;
6083 NEON_FP : S5;
6084 %}
6085
6086 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6087 %{
6088 single_instruction;
6089 src : S1(read);
6090 dst : S5(write);
6091 INS01 : ISS;
6092 NEON_FP : S5;
6093 %}
6094
6095 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6096 %{
6097 single_instruction;
6098 src : S1(read);
6099 dst : S5(write);
6100 INS01 : ISS;
6101 NEON_FP : S5;
6102 %}
6103
6104 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6105 %{
6106 single_instruction;
6107 src : S1(read);
6108 dst : S5(write);
6109 INS01 : ISS;
6110 NEON_FP : S5;
6111 %}
6112
6113 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6114 %{
6115 single_instruction;
6116 src1 : S1(read);
6117 src2 : S2(read);
6118 dst : S5(write);
6119 INS0 : ISS;
6120 NEON_FP : S5;
6121 %}
6122
6123 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6124 %{
6125 single_instruction;
6126 src1 : S1(read);
6127 src2 : S2(read);
6128 dst : S5(write);
6129 INS0 : ISS;
6130 NEON_FP : S5;
6131 %}
6132
6133 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6134 %{
6135 single_instruction;
6136 cr : S1(read);
6137 src1 : S1(read);
6138 src2 : S1(read);
6139 dst : S3(write);
6140 INS01 : ISS;
6141 NEON_FP : S3;
6142 %}
6143
6144 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6145 %{
6146 single_instruction;
6147 cr : S1(read);
6148 src1 : S1(read);
6149 src2 : S1(read);
6150 dst : S3(write);
6151 INS01 : ISS;
6152 NEON_FP : S3;
6153 %}
6154
6155 pipe_class fp_imm_s(vRegF dst)
6156 %{
6157 single_instruction;
6158 dst : S3(write);
6159 INS01 : ISS;
6160 NEON_FP : S3;
6161 %}
6162
6163 pipe_class fp_imm_d(vRegD dst)
6164 %{
6165 single_instruction;
6166 dst : S3(write);
6167 INS01 : ISS;
6168 NEON_FP : S3;
6169 %}
6170
6171 pipe_class fp_load_constant_s(vRegF dst)
6172 %{
6173 single_instruction;
6174 dst : S4(write);
6175 INS01 : ISS;
6176 NEON_FP : S4;
6177 %}
6178
6179 pipe_class fp_load_constant_d(vRegD dst)
6180 %{
6181 single_instruction;
6182 dst : S4(write);
6183 INS01 : ISS;
6184 NEON_FP : S4;
6185 %}
6186
6187 //------- Integer ALU operations --------------------------
6188
6189 // Integer ALU reg-reg operation
6190 // Operands needed in EX1, result generated in EX2
6191 // Eg. ADD x0, x1, x2
6192 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6193 %{
6194 single_instruction;
6195 dst : EX2(write);
6196 src1 : EX1(read);
6197 src2 : EX1(read);
6198 INS01 : ISS; // Dual issue as instruction 0 or 1
6199 ALU : EX2;
6200 %}
6201
6202 // Integer ALU reg-reg operation with constant shift
6203 // Shifted register must be available in LATE_ISS instead of EX1
6204 // Eg. ADD x0, x1, x2, LSL #2
6205 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6206 %{
6207 single_instruction;
6208 dst : EX2(write);
6209 src1 : EX1(read);
6210 src2 : ISS(read);
6211 INS01 : ISS;
6212 ALU : EX2;
6213 %}
6214
6215 // Integer ALU reg operation with constant shift
6216 // Eg. LSL x0, x1, #shift
6217 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6218 %{
6219 single_instruction;
6220 dst : EX2(write);
6221 src1 : ISS(read);
6222 INS01 : ISS;
6223 ALU : EX2;
6224 %}
6225
6226 // Integer ALU reg-reg operation with variable shift
6227 // Both operands must be available in LATE_ISS instead of EX1
6228 // Result is available in EX1 instead of EX2
6229 // Eg. LSLV x0, x1, x2
6230 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6231 %{
6232 single_instruction;
6233 dst : EX1(write);
6234 src1 : ISS(read);
6235 src2 : ISS(read);
6236 INS01 : ISS;
6237 ALU : EX1;
6238 %}
6239
6240 // Integer ALU reg-reg operation with extract
6241 // As for _vshift above, but result generated in EX2
6242 // Eg. EXTR x0, x1, x2, #N
6243 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6244 %{
6245 single_instruction;
6246 dst : EX2(write);
6247 src1 : ISS(read);
6248 src2 : ISS(read);
6249 INS1 : ISS; // Can only dual issue as Instruction 1
6250 ALU : EX1;
6251 %}
6252
6253 // Integer ALU reg operation
6254 // Eg. NEG x0, x1
6255 pipe_class ialu_reg(iRegI dst, iRegI src)
6256 %{
6257 single_instruction;
6258 dst : EX2(write);
6259 src : EX1(read);
6260 INS01 : ISS;
6261 ALU : EX2;
6262 %}
6263
6264 // Integer ALU reg mmediate operation
6265 // Eg. ADD x0, x1, #N
6266 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6267 %{
6268 single_instruction;
6269 dst : EX2(write);
6270 src1 : EX1(read);
6271 INS01 : ISS;
6272 ALU : EX2;
6273 %}
6274
6275 // Integer ALU immediate operation (no source operands)
6276 // Eg. MOV x0, #N
6277 pipe_class ialu_imm(iRegI dst)
6278 %{
6279 single_instruction;
6280 dst : EX1(write);
6281 INS01 : ISS;
6282 ALU : EX1;
6283 %}
6284
6285 //------- Compare operation -------------------------------
6286
6287 // Compare reg-reg
6288 // Eg. CMP x0, x1
6289 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6290 %{
6291 single_instruction;
6292 // fixed_latency(16);
6293 cr : EX2(write);
6294 op1 : EX1(read);
6295 op2 : EX1(read);
6296 INS01 : ISS;
6297 ALU : EX2;
6298 %}
6299
6300 // Compare reg-reg
6301 // Eg. CMP x0, #N
6302 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6303 %{
6304 single_instruction;
6305 // fixed_latency(16);
6306 cr : EX2(write);
6307 op1 : EX1(read);
6308 INS01 : ISS;
6309 ALU : EX2;
6310 %}
6311
6312 //------- Conditional instructions ------------------------
6313
6314 // Conditional no operands
6315 // Eg. CSINC x0, zr, zr, <cond>
6316 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6317 %{
6318 single_instruction;
6319 cr : EX1(read);
6320 dst : EX2(write);
6321 INS01 : ISS;
6322 ALU : EX2;
6323 %}
6324
6325 // Conditional 2 operand
6326 // EG. CSEL X0, X1, X2, <cond>
6327 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6328 %{
6329 single_instruction;
6330 cr : EX1(read);
6331 src1 : EX1(read);
6332 src2 : EX1(read);
6333 dst : EX2(write);
6334 INS01 : ISS;
6335 ALU : EX2;
6336 %}
6337
6338 // Conditional 2 operand
6339 // EG. CSEL X0, X1, X2, <cond>
6340 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6341 %{
6342 single_instruction;
6343 cr : EX1(read);
6344 src : EX1(read);
6345 dst : EX2(write);
6346 INS01 : ISS;
6347 ALU : EX2;
6348 %}
6349
6350 //------- Multiply pipeline operations --------------------
6351
6352 // Multiply reg-reg
6353 // Eg. MUL w0, w1, w2
6354 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6355 %{
6356 single_instruction;
6357 dst : WR(write);
6358 src1 : ISS(read);
6359 src2 : ISS(read);
6360 INS01 : ISS;
6361 MAC : WR;
6362 %}
6363
6364 // Multiply accumulate
6365 // Eg. MADD w0, w1, w2, w3
6366 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6367 %{
6368 single_instruction;
6369 dst : WR(write);
6370 src1 : ISS(read);
6371 src2 : ISS(read);
6372 src3 : ISS(read);
6373 INS01 : ISS;
6374 MAC : WR;
6375 %}
6376
6377 // Eg. MUL w0, w1, w2
6378 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6379 %{
6380 single_instruction;
6381 fixed_latency(3); // Maximum latency for 64 bit mul
6382 dst : WR(write);
6383 src1 : ISS(read);
6384 src2 : ISS(read);
6385 INS01 : ISS;
6386 MAC : WR;
6387 %}
6388
6389 // Multiply accumulate
6390 // Eg. MADD w0, w1, w2, w3
6391 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6392 %{
6393 single_instruction;
6394 fixed_latency(3); // Maximum latency for 64 bit mul
6395 dst : WR(write);
6396 src1 : ISS(read);
6397 src2 : ISS(read);
6398 src3 : ISS(read);
6399 INS01 : ISS;
6400 MAC : WR;
6401 %}
6402
6403 //------- Divide pipeline operations --------------------
6404
6405 // Eg. SDIV w0, w1, w2
6406 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6407 %{
6408 single_instruction;
6409 fixed_latency(8); // Maximum latency for 32 bit divide
6410 dst : WR(write);
6411 src1 : ISS(read);
6412 src2 : ISS(read);
6413 INS0 : ISS; // Can only dual issue as instruction 0
6414 DIV : WR;
6415 %}
6416
6417 // Eg. SDIV x0, x1, x2
6418 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6419 %{
6420 single_instruction;
6421 fixed_latency(16); // Maximum latency for 64 bit divide
6422 dst : WR(write);
6423 src1 : ISS(read);
6424 src2 : ISS(read);
6425 INS0 : ISS; // Can only dual issue as instruction 0
6426 DIV : WR;
6427 %}
6428
6429 //------- Load pipeline operations ------------------------
6430
6431 // Load - prefetch
6432 // Eg. PFRM <mem>
6433 pipe_class iload_prefetch(memory mem)
6434 %{
6435 single_instruction;
6436 mem : ISS(read);
6437 INS01 : ISS;
6438 LDST : WR;
6439 %}
6440
6441 // Load - reg, mem
6442 // Eg. LDR x0, <mem>
6443 pipe_class iload_reg_mem(iRegI dst, memory mem)
6444 %{
6445 single_instruction;
6446 dst : WR(write);
6447 mem : ISS(read);
6448 INS01 : ISS;
6449 LDST : WR;
6450 %}
6451
6452 // Load - reg, reg
6453 // Eg. LDR x0, [sp, x1]
6454 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6455 %{
6456 single_instruction;
6457 dst : WR(write);
6458 src : ISS(read);
6459 INS01 : ISS;
6460 LDST : WR;
6461 %}
6462
6463 //------- Store pipeline operations -----------------------
6464
6465 // Store - zr, mem
6466 // Eg. STR zr, <mem>
6467 pipe_class istore_mem(memory mem)
6468 %{
6469 single_instruction;
6470 mem : ISS(read);
6471 INS01 : ISS;
6472 LDST : WR;
6473 %}
6474
6475 // Store - reg, mem
6476 // Eg. STR x0, <mem>
6477 pipe_class istore_reg_mem(iRegI src, memory mem)
6478 %{
6479 single_instruction;
6480 mem : ISS(read);
6481 src : EX2(read);
6482 INS01 : ISS;
6483 LDST : WR;
6484 %}
6485
6486 // Store - reg, reg
6487 // Eg. STR x0, [sp, x1]
6488 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6489 %{
6490 single_instruction;
6491 dst : ISS(read);
6492 src : EX2(read);
6493 INS01 : ISS;
6494 LDST : WR;
6495 %}
6496
6497 //------- Store pipeline operations -----------------------
6498
6499 // Branch
6500 pipe_class pipe_branch()
6501 %{
6502 single_instruction;
6503 INS01 : ISS;
6504 BRANCH : EX1;
6505 %}
6506
6507 // Conditional branch
6508 pipe_class pipe_branch_cond(rFlagsReg cr)
6509 %{
6510 single_instruction;
6511 cr : EX1(read);
6512 INS01 : ISS;
6513 BRANCH : EX1;
6514 %}
6515
6516 // Compare & Branch
6517 // EG. CBZ/CBNZ
6518 pipe_class pipe_cmp_branch(iRegI op1)
6519 %{
6520 single_instruction;
6521 op1 : EX1(read);
6522 INS01 : ISS;
6523 BRANCH : EX1;
6524 %}
6525
6526 //------- Synchronisation operations ----------------------
6527
6528 // Any operation requiring serialization.
6529 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6530 pipe_class pipe_serial()
6531 %{
6532 single_instruction;
6533 force_serialization;
6534 fixed_latency(16);
6535 INS01 : ISS(2); // Cannot dual issue with any other instruction
6536 LDST : WR;
6537 %}
6538
6539 // Generic big/slow expanded idiom - also serialized
6540 pipe_class pipe_slow()
6541 %{
6542 instruction_count(10);
6543 multiple_bundles;
6544 force_serialization;
6545 fixed_latency(16);
6546 INS01 : ISS(2); // Cannot dual issue with any other instruction
6547 LDST : WR;
6548 %}
6549
6550 // Empty pipeline class
6551 pipe_class pipe_class_empty()
6552 %{
6553 single_instruction;
6554 fixed_latency(0);
6555 %}
6556
6557 // Default pipeline class.
6558 pipe_class pipe_class_default()
6559 %{
6560 single_instruction;
6561 fixed_latency(2);
6562 %}
6563
6564 // Pipeline class for compares.
6565 pipe_class pipe_class_compare()
6566 %{
6567 single_instruction;
6568 fixed_latency(16);
6569 %}
6570
6571 // Pipeline class for memory operations.
6572 pipe_class pipe_class_memory()
6573 %{
6574 single_instruction;
6575 fixed_latency(16);
6576 %}
6577
6578 // Pipeline class for call.
6579 pipe_class pipe_class_call()
6580 %{
6581 single_instruction;
6582 fixed_latency(100);
6583 %}
6584
6585 // Define the class for the Nop node.
6586 define %{
6587 MachNop = pipe_class_empty;
6588 %}
6589
6590 %}
6591 //----------INSTRUCTIONS-------------------------------------------------------
6592 //
6593 // match -- States which machine-independent subtree may be replaced
6594 // by this instruction.
6595 // ins_cost -- The estimated cost of this instruction is used by instruction
6596 // selection to identify a minimum cost tree of machine
6597 // instructions that matches a tree of machine-independent
6598 // instructions.
6599 // format -- A string providing the disassembly for this instruction.
6600 // The value of an instruction's operand may be inserted
6601 // by referring to it with a '$' prefix.
6602 // opcode -- Three instruction opcodes may be provided. These are referred
6603 // to within an encode class as $primary, $secondary, and $tertiary
6604 // rrspectively. The primary opcode is commonly used to
6605 // indicate the type of machine instruction, while secondary
6606 // and tertiary are often used for prefix options or addressing
6607 // modes.
6608 // ins_encode -- A list of encode classes with parameters. The encode class
6609 // name must have been defined in an 'enc_class' specification
6610 // in the encode section of the architecture description.
6611
6612 // ============================================================================
6613 // Memory (Load/Store) Instructions
6614
6615 // Load Instructions
6616
6617 // Load Byte (8 bit signed)
6618 instruct loadB(iRegINoSp dst, memory1 mem)
6619 %{
6620 match(Set dst (LoadB mem));
6621 predicate(!needs_acquiring_load(n));
6622
6623 ins_cost(4 * INSN_COST);
6624 format %{ "ldrsbw $dst, $mem\t# byte" %}
6625
6626 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6627
6628 ins_pipe(iload_reg_mem);
6629 %}
6630
6631 // Load Byte (8 bit signed) into long
6632 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6633 %{
6634 match(Set dst (ConvI2L (LoadB mem)));
6635 predicate(!needs_acquiring_load(n->in(1)));
6636
6637 ins_cost(4 * INSN_COST);
6638 format %{ "ldrsb $dst, $mem\t# byte" %}
6639
6640 ins_encode(aarch64_enc_ldrsb(dst, mem));
6641
6642 ins_pipe(iload_reg_mem);
6643 %}
6644
6645 // Load Byte (8 bit unsigned)
6646 instruct loadUB(iRegINoSp dst, memory1 mem)
6647 %{
6648 match(Set dst (LoadUB mem));
6649 predicate(!needs_acquiring_load(n));
6650
6651 ins_cost(4 * INSN_COST);
6652 format %{ "ldrbw $dst, $mem\t# byte" %}
6653
6654 ins_encode(aarch64_enc_ldrb(dst, mem));
6655
6656 ins_pipe(iload_reg_mem);
6657 %}
6658
6659 // Load Byte (8 bit unsigned) into long
6660 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6661 %{
6662 match(Set dst (ConvI2L (LoadUB mem)));
6663 predicate(!needs_acquiring_load(n->in(1)));
6664
6665 ins_cost(4 * INSN_COST);
6666 format %{ "ldrb $dst, $mem\t# byte" %}
6667
6668 ins_encode(aarch64_enc_ldrb(dst, mem));
6669
6670 ins_pipe(iload_reg_mem);
6671 %}
6672
6673 // Load Short (16 bit signed)
6674 instruct loadS(iRegINoSp dst, memory2 mem)
6675 %{
6676 match(Set dst (LoadS mem));
6677 predicate(!needs_acquiring_load(n));
6678
6679 ins_cost(4 * INSN_COST);
6680 format %{ "ldrshw $dst, $mem\t# short" %}
6681
6682 ins_encode(aarch64_enc_ldrshw(dst, mem));
6683
6684 ins_pipe(iload_reg_mem);
6685 %}
6686
6687 // Load Short (16 bit signed) into long
6688 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6689 %{
6690 match(Set dst (ConvI2L (LoadS mem)));
6691 predicate(!needs_acquiring_load(n->in(1)));
6692
6693 ins_cost(4 * INSN_COST);
6694 format %{ "ldrsh $dst, $mem\t# short" %}
6695
6696 ins_encode(aarch64_enc_ldrsh(dst, mem));
6697
6698 ins_pipe(iload_reg_mem);
6699 %}
6700
6701 // Load Char (16 bit unsigned)
6702 instruct loadUS(iRegINoSp dst, memory2 mem)
6703 %{
6704 match(Set dst (LoadUS mem));
6705 predicate(!needs_acquiring_load(n));
6706
6707 ins_cost(4 * INSN_COST);
6708 format %{ "ldrh $dst, $mem\t# short" %}
6709
6710 ins_encode(aarch64_enc_ldrh(dst, mem));
6711
6712 ins_pipe(iload_reg_mem);
6713 %}
6714
6715 // Load Short/Char (16 bit unsigned) into long
6716 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6717 %{
6718 match(Set dst (ConvI2L (LoadUS mem)));
6719 predicate(!needs_acquiring_load(n->in(1)));
6720
6721 ins_cost(4 * INSN_COST);
6722 format %{ "ldrh $dst, $mem\t# short" %}
6723
6724 ins_encode(aarch64_enc_ldrh(dst, mem));
6725
6726 ins_pipe(iload_reg_mem);
6727 %}
6728
6729 // Load Integer (32 bit signed)
6730 instruct loadI(iRegINoSp dst, memory4 mem)
6731 %{
6732 match(Set dst (LoadI mem));
6733 predicate(!needs_acquiring_load(n));
6734
6735 ins_cost(4 * INSN_COST);
6736 format %{ "ldrw $dst, $mem\t# int" %}
6737
6738 ins_encode(aarch64_enc_ldrw(dst, mem));
6739
6740 ins_pipe(iload_reg_mem);
6741 %}
6742
6743 // Load Integer (32 bit signed) into long
6744 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6745 %{
6746 match(Set dst (ConvI2L (LoadI mem)));
6747 predicate(!needs_acquiring_load(n->in(1)));
6748
6749 ins_cost(4 * INSN_COST);
6750 format %{ "ldrsw $dst, $mem\t# int" %}
6751
6752 ins_encode(aarch64_enc_ldrsw(dst, mem));
6753
6754 ins_pipe(iload_reg_mem);
6755 %}
6756
6757 // Load Integer (32 bit unsigned) into long
6758 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6759 %{
6760 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6761 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6762
6763 ins_cost(4 * INSN_COST);
6764 format %{ "ldrw $dst, $mem\t# int" %}
6765
6766 ins_encode(aarch64_enc_ldrw(dst, mem));
6767
6768 ins_pipe(iload_reg_mem);
6769 %}
6770
6771 // Load Long (64 bit signed)
6772 instruct loadL(iRegLNoSp dst, memory8 mem)
6773 %{
6774 match(Set dst (LoadL mem));
6775 predicate(!needs_acquiring_load(n));
6776
6777 ins_cost(4 * INSN_COST);
6778 format %{ "ldr $dst, $mem\t# int" %}
6779
6780 ins_encode(aarch64_enc_ldr(dst, mem));
6781
6782 ins_pipe(iload_reg_mem);
6783 %}
6784
6785 // Load Range
6786 instruct loadRange(iRegINoSp dst, memory4 mem)
6787 %{
6788 match(Set dst (LoadRange mem));
6789
6790 ins_cost(4 * INSN_COST);
6791 format %{ "ldrw $dst, $mem\t# range" %}
6792
6793 ins_encode(aarch64_enc_ldrw(dst, mem));
6794
6795 ins_pipe(iload_reg_mem);
6796 %}
6797
6798 // Load Pointer
6799 instruct loadP(iRegPNoSp dst, memory8 mem)
6800 %{
6801 match(Set dst (LoadP mem));
6802 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6803
6804 ins_cost(4 * INSN_COST);
6805 format %{ "ldr $dst, $mem\t# ptr" %}
6806
6807 ins_encode(aarch64_enc_ldr(dst, mem));
6808
6809 ins_pipe(iload_reg_mem);
6810 %}
6811
6812 // Load Compressed Pointer
6813 instruct loadN(iRegNNoSp dst, memory4 mem)
6814 %{
6815 match(Set dst (LoadN mem));
6816 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6817
6818 ins_cost(4 * INSN_COST);
6819 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6820
6821 ins_encode(aarch64_enc_ldrw(dst, mem));
6822
6823 ins_pipe(iload_reg_mem);
6824 %}
6825
6826 // Load Klass Pointer
6827 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6828 %{
6829 match(Set dst (LoadKlass mem));
6830 predicate(!needs_acquiring_load(n));
6831
6832 ins_cost(4 * INSN_COST);
6833 format %{ "ldr $dst, $mem\t# class" %}
6834
6835 ins_encode(aarch64_enc_ldr(dst, mem));
6836
6837 ins_pipe(iload_reg_mem);
6838 %}
6839
6840 // Load Narrow Klass Pointer
6841 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6842 %{
6843 match(Set dst (LoadNKlass mem));
6844 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6845
6846 ins_cost(4 * INSN_COST);
6847 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6848
6849 ins_encode(aarch64_enc_ldrw(dst, mem));
6850
6851 ins_pipe(iload_reg_mem);
6852 %}
6853
6854 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6855 %{
6856 match(Set dst (LoadNKlass mem));
6857 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6858
6859 ins_cost(4 * INSN_COST);
6860 format %{
6861 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6862 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6863 %}
6864 ins_encode %{
6865 // inlined aarch64_enc_ldrw
6866 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6867 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6868 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6869 %}
6870 ins_pipe(iload_reg_mem);
6871 %}
6872
6873 // Load Float
6874 instruct loadF(vRegF dst, memory4 mem)
6875 %{
6876 match(Set dst (LoadF mem));
6877 predicate(!needs_acquiring_load(n));
6878
6879 ins_cost(4 * INSN_COST);
6880 format %{ "ldrs $dst, $mem\t# float" %}
6881
6882 ins_encode( aarch64_enc_ldrs(dst, mem) );
6883
6884 ins_pipe(pipe_class_memory);
6885 %}
6886
6887 // Load Double
6888 instruct loadD(vRegD dst, memory8 mem)
6889 %{
6890 match(Set dst (LoadD mem));
6891 predicate(!needs_acquiring_load(n));
6892
6893 ins_cost(4 * INSN_COST);
6894 format %{ "ldrd $dst, $mem\t# double" %}
6895
6896 ins_encode( aarch64_enc_ldrd(dst, mem) );
6897
6898 ins_pipe(pipe_class_memory);
6899 %}
6900
6901
6902 // Load Int Constant
6903 instruct loadConI(iRegINoSp dst, immI src)
6904 %{
6905 match(Set dst src);
6906
6907 ins_cost(INSN_COST);
6908 format %{ "mov $dst, $src\t# int" %}
6909
6910 ins_encode( aarch64_enc_movw_imm(dst, src) );
6911
6912 ins_pipe(ialu_imm);
6913 %}
6914
6915 // Load Long Constant
6916 instruct loadConL(iRegLNoSp dst, immL src)
6917 %{
6918 match(Set dst src);
6919
6920 ins_cost(INSN_COST);
6921 format %{ "mov $dst, $src\t# long" %}
6922
6923 ins_encode( aarch64_enc_mov_imm(dst, src) );
6924
6925 ins_pipe(ialu_imm);
6926 %}
6927
6928 // Load Pointer Constant
6929
6930 instruct loadConP(iRegPNoSp dst, immP con)
6931 %{
6932 match(Set dst con);
6933
6934 ins_cost(INSN_COST * 4);
6935 format %{
6936 "mov $dst, $con\t# ptr\n\t"
6937 %}
6938
6939 ins_encode(aarch64_enc_mov_p(dst, con));
6940
6941 ins_pipe(ialu_imm);
6942 %}
6943
6944 // Load Null Pointer Constant
6945
6946 instruct loadConP0(iRegPNoSp dst, immP0 con)
6947 %{
6948 match(Set dst con);
6949
6950 ins_cost(INSN_COST);
6951 format %{ "mov $dst, $con\t# nullptr ptr" %}
6952
6953 ins_encode(aarch64_enc_mov_p0(dst, con));
6954
6955 ins_pipe(ialu_imm);
6956 %}
6957
6958 // Load Pointer Constant One
6959
6960 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6961 %{
6962 match(Set dst con);
6963
6964 ins_cost(INSN_COST);
6965 format %{ "mov $dst, $con\t# nullptr ptr" %}
6966
6967 ins_encode(aarch64_enc_mov_p1(dst, con));
6968
6969 ins_pipe(ialu_imm);
6970 %}
6971
6972 // Load Narrow Pointer Constant
6973
6974 instruct loadConN(iRegNNoSp dst, immN con)
6975 %{
6976 match(Set dst con);
6977
6978 ins_cost(INSN_COST * 4);
6979 format %{ "mov $dst, $con\t# compressed ptr" %}
6980
6981 ins_encode(aarch64_enc_mov_n(dst, con));
6982
6983 ins_pipe(ialu_imm);
6984 %}
6985
6986 // Load Narrow Null Pointer Constant
6987
6988 instruct loadConN0(iRegNNoSp dst, immN0 con)
6989 %{
6990 match(Set dst con);
6991
6992 ins_cost(INSN_COST);
6993 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6994
6995 ins_encode(aarch64_enc_mov_n0(dst, con));
6996
6997 ins_pipe(ialu_imm);
6998 %}
6999
7000 // Load Narrow Klass Constant
7001
7002 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7003 %{
7004 match(Set dst con);
7005
7006 ins_cost(INSN_COST);
7007 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7008
7009 ins_encode(aarch64_enc_mov_nk(dst, con));
7010
7011 ins_pipe(ialu_imm);
7012 %}
7013
7014 // Load Packed Float Constant
7015
7016 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7017 match(Set dst con);
7018 ins_cost(INSN_COST * 4);
7019 format %{ "fmovs $dst, $con"%}
7020 ins_encode %{
7021 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7022 %}
7023
7024 ins_pipe(fp_imm_s);
7025 %}
7026
7027 // Load Float Constant
7028
7029 instruct loadConF(vRegF dst, immF con) %{
7030 match(Set dst con);
7031
7032 ins_cost(INSN_COST * 4);
7033
7034 format %{
7035 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7036 %}
7037
7038 ins_encode %{
7039 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7040 %}
7041
7042 ins_pipe(fp_load_constant_s);
7043 %}
7044
7045 // Load Packed Double Constant
7046
7047 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7048 match(Set dst con);
7049 ins_cost(INSN_COST);
7050 format %{ "fmovd $dst, $con"%}
7051 ins_encode %{
7052 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7053 %}
7054
7055 ins_pipe(fp_imm_d);
7056 %}
7057
7058 // Load Double Constant
7059
7060 instruct loadConD(vRegD dst, immD con) %{
7061 match(Set dst con);
7062
7063 ins_cost(INSN_COST * 5);
7064 format %{
7065 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7066 %}
7067
7068 ins_encode %{
7069 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7070 %}
7071
7072 ins_pipe(fp_load_constant_d);
7073 %}
7074
7075 // Load Half Float Constant
7076 instruct loadConH(vRegF dst, immH con) %{
7077 match(Set dst con);
7078 format %{ "mov rscratch1, $con\n\t"
7079 "fmov $dst, rscratch1"
7080 %}
7081 ins_encode %{
7082 __ movw(rscratch1, (uint32_t)$con$$constant);
7083 __ fmovs($dst$$FloatRegister, rscratch1);
7084 %}
7085 ins_pipe(pipe_class_default);
7086 %}
7087
7088 // Store Instructions
7089
7090 // Store Byte
7091 instruct storeB(iRegIorL2I src, memory1 mem)
7092 %{
7093 match(Set mem (StoreB mem src));
7094 predicate(!needs_releasing_store(n));
7095
7096 ins_cost(INSN_COST);
7097 format %{ "strb $src, $mem\t# byte" %}
7098
7099 ins_encode(aarch64_enc_strb(src, mem));
7100
7101 ins_pipe(istore_reg_mem);
7102 %}
7103
7104
7105 instruct storeimmB0(immI0 zero, memory1 mem)
7106 %{
7107 match(Set mem (StoreB mem zero));
7108 predicate(!needs_releasing_store(n));
7109
7110 ins_cost(INSN_COST);
7111 format %{ "strb rscractch2, $mem\t# byte" %}
7112
7113 ins_encode(aarch64_enc_strb0(mem));
7114
7115 ins_pipe(istore_mem);
7116 %}
7117
7118 // Store Char/Short
7119 instruct storeC(iRegIorL2I src, memory2 mem)
7120 %{
7121 match(Set mem (StoreC mem src));
7122 predicate(!needs_releasing_store(n));
7123
7124 ins_cost(INSN_COST);
7125 format %{ "strh $src, $mem\t# short" %}
7126
7127 ins_encode(aarch64_enc_strh(src, mem));
7128
7129 ins_pipe(istore_reg_mem);
7130 %}
7131
7132 instruct storeimmC0(immI0 zero, memory2 mem)
7133 %{
7134 match(Set mem (StoreC mem zero));
7135 predicate(!needs_releasing_store(n));
7136
7137 ins_cost(INSN_COST);
7138 format %{ "strh zr, $mem\t# short" %}
7139
7140 ins_encode(aarch64_enc_strh0(mem));
7141
7142 ins_pipe(istore_mem);
7143 %}
7144
7145 // Store Integer
7146
7147 instruct storeI(iRegIorL2I src, memory4 mem)
7148 %{
7149 match(Set mem(StoreI mem src));
7150 predicate(!needs_releasing_store(n));
7151
7152 ins_cost(INSN_COST);
7153 format %{ "strw $src, $mem\t# int" %}
7154
7155 ins_encode(aarch64_enc_strw(src, mem));
7156
7157 ins_pipe(istore_reg_mem);
7158 %}
7159
7160 instruct storeimmI0(immI0 zero, memory4 mem)
7161 %{
7162 match(Set mem(StoreI mem zero));
7163 predicate(!needs_releasing_store(n));
7164
7165 ins_cost(INSN_COST);
7166 format %{ "strw zr, $mem\t# int" %}
7167
7168 ins_encode(aarch64_enc_strw0(mem));
7169
7170 ins_pipe(istore_mem);
7171 %}
7172
7173 // Store Long (64 bit signed)
7174 instruct storeL(iRegL src, memory8 mem)
7175 %{
7176 match(Set mem (StoreL mem src));
7177 predicate(!needs_releasing_store(n));
7178
7179 ins_cost(INSN_COST);
7180 format %{ "str $src, $mem\t# int" %}
7181
7182 ins_encode(aarch64_enc_str(src, mem));
7183
7184 ins_pipe(istore_reg_mem);
7185 %}
7186
7187 // Store Long (64 bit signed)
7188 instruct storeimmL0(immL0 zero, memory8 mem)
7189 %{
7190 match(Set mem (StoreL mem zero));
7191 predicate(!needs_releasing_store(n));
7192
7193 ins_cost(INSN_COST);
7194 format %{ "str zr, $mem\t# int" %}
7195
7196 ins_encode(aarch64_enc_str0(mem));
7197
7198 ins_pipe(istore_mem);
7199 %}
7200
7201 // Store Pointer
7202 instruct storeP(iRegP src, memory8 mem)
7203 %{
7204 match(Set mem (StoreP mem src));
7205 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7206
7207 ins_cost(INSN_COST);
7208 format %{ "str $src, $mem\t# ptr" %}
7209
7210 ins_encode(aarch64_enc_str(src, mem));
7211
7212 ins_pipe(istore_reg_mem);
7213 %}
7214
7215 // Store Pointer
7216 instruct storeimmP0(immP0 zero, memory8 mem)
7217 %{
7218 match(Set mem (StoreP mem zero));
7219 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7220
7221 ins_cost(INSN_COST);
7222 format %{ "str zr, $mem\t# ptr" %}
7223
7224 ins_encode(aarch64_enc_str0(mem));
7225
7226 ins_pipe(istore_mem);
7227 %}
7228
7229 // Store Compressed Pointer
7230 instruct storeN(iRegN src, memory4 mem)
7231 %{
7232 match(Set mem (StoreN mem src));
7233 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7234
7235 ins_cost(INSN_COST);
7236 format %{ "strw $src, $mem\t# compressed ptr" %}
7237
7238 ins_encode(aarch64_enc_strw(src, mem));
7239
7240 ins_pipe(istore_reg_mem);
7241 %}
7242
7243 instruct storeImmN0(immN0 zero, memory4 mem)
7244 %{
7245 match(Set mem (StoreN mem zero));
7246 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7247
7248 ins_cost(INSN_COST);
7249 format %{ "strw zr, $mem\t# compressed ptr" %}
7250
7251 ins_encode(aarch64_enc_strw0(mem));
7252
7253 ins_pipe(istore_mem);
7254 %}
7255
7256 // Store Float
7257 instruct storeF(vRegF src, memory4 mem)
7258 %{
7259 match(Set mem (StoreF mem src));
7260 predicate(!needs_releasing_store(n));
7261
7262 ins_cost(INSN_COST);
7263 format %{ "strs $src, $mem\t# float" %}
7264
7265 ins_encode( aarch64_enc_strs(src, mem) );
7266
7267 ins_pipe(pipe_class_memory);
7268 %}
7269
7270 // TODO
7271 // implement storeImmF0 and storeFImmPacked
7272
7273 // Store Double
7274 instruct storeD(vRegD src, memory8 mem)
7275 %{
7276 match(Set mem (StoreD mem src));
7277 predicate(!needs_releasing_store(n));
7278
7279 ins_cost(INSN_COST);
7280 format %{ "strd $src, $mem\t# double" %}
7281
7282 ins_encode( aarch64_enc_strd(src, mem) );
7283
7284 ins_pipe(pipe_class_memory);
7285 %}
7286
7287 // Store Compressed Klass Pointer
7288 instruct storeNKlass(iRegN src, memory4 mem)
7289 %{
7290 predicate(!needs_releasing_store(n));
7291 match(Set mem (StoreNKlass mem src));
7292
7293 ins_cost(INSN_COST);
7294 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7295
7296 ins_encode(aarch64_enc_strw(src, mem));
7297
7298 ins_pipe(istore_reg_mem);
7299 %}
7300
7301 // TODO
7302 // implement storeImmD0 and storeDImmPacked
7303
7304 // prefetch instructions
7305 // Must be safe to execute with invalid address (cannot fault).
7306
7307 instruct prefetchalloc( memory8 mem ) %{
7308 match(PrefetchAllocation mem);
7309
7310 ins_cost(INSN_COST);
7311 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7312
7313 ins_encode( aarch64_enc_prefetchw(mem) );
7314
7315 ins_pipe(iload_prefetch);
7316 %}
7317
7318 // ---------------- volatile loads and stores ----------------
7319
7320 // Load Byte (8 bit signed)
7321 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7322 %{
7323 match(Set dst (LoadB mem));
7324
7325 ins_cost(VOLATILE_REF_COST);
7326 format %{ "ldarsb $dst, $mem\t# byte" %}
7327
7328 ins_encode(aarch64_enc_ldarsb(dst, mem));
7329
7330 ins_pipe(pipe_serial);
7331 %}
7332
7333 // Load Byte (8 bit signed) into long
7334 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7335 %{
7336 match(Set dst (ConvI2L (LoadB mem)));
7337
7338 ins_cost(VOLATILE_REF_COST);
7339 format %{ "ldarsb $dst, $mem\t# byte" %}
7340
7341 ins_encode(aarch64_enc_ldarsb(dst, mem));
7342
7343 ins_pipe(pipe_serial);
7344 %}
7345
7346 // Load Byte (8 bit unsigned)
7347 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7348 %{
7349 match(Set dst (LoadUB mem));
7350
7351 ins_cost(VOLATILE_REF_COST);
7352 format %{ "ldarb $dst, $mem\t# byte" %}
7353
7354 ins_encode(aarch64_enc_ldarb(dst, mem));
7355
7356 ins_pipe(pipe_serial);
7357 %}
7358
7359 // Load Byte (8 bit unsigned) into long
7360 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7361 %{
7362 match(Set dst (ConvI2L (LoadUB mem)));
7363
7364 ins_cost(VOLATILE_REF_COST);
7365 format %{ "ldarb $dst, $mem\t# byte" %}
7366
7367 ins_encode(aarch64_enc_ldarb(dst, mem));
7368
7369 ins_pipe(pipe_serial);
7370 %}
7371
7372 // Load Short (16 bit signed)
7373 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7374 %{
7375 match(Set dst (LoadS mem));
7376
7377 ins_cost(VOLATILE_REF_COST);
7378 format %{ "ldarshw $dst, $mem\t# short" %}
7379
7380 ins_encode(aarch64_enc_ldarshw(dst, mem));
7381
7382 ins_pipe(pipe_serial);
7383 %}
7384
7385 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7386 %{
7387 match(Set dst (LoadUS mem));
7388
7389 ins_cost(VOLATILE_REF_COST);
7390 format %{ "ldarhw $dst, $mem\t# short" %}
7391
7392 ins_encode(aarch64_enc_ldarhw(dst, mem));
7393
7394 ins_pipe(pipe_serial);
7395 %}
7396
7397 // Load Short/Char (16 bit unsigned) into long
7398 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7399 %{
7400 match(Set dst (ConvI2L (LoadUS mem)));
7401
7402 ins_cost(VOLATILE_REF_COST);
7403 format %{ "ldarh $dst, $mem\t# short" %}
7404
7405 ins_encode(aarch64_enc_ldarh(dst, mem));
7406
7407 ins_pipe(pipe_serial);
7408 %}
7409
7410 // Load Short/Char (16 bit signed) into long
7411 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7412 %{
7413 match(Set dst (ConvI2L (LoadS mem)));
7414
7415 ins_cost(VOLATILE_REF_COST);
7416 format %{ "ldarh $dst, $mem\t# short" %}
7417
7418 ins_encode(aarch64_enc_ldarsh(dst, mem));
7419
7420 ins_pipe(pipe_serial);
7421 %}
7422
7423 // Load Integer (32 bit signed)
7424 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7425 %{
7426 match(Set dst (LoadI mem));
7427
7428 ins_cost(VOLATILE_REF_COST);
7429 format %{ "ldarw $dst, $mem\t# int" %}
7430
7431 ins_encode(aarch64_enc_ldarw(dst, mem));
7432
7433 ins_pipe(pipe_serial);
7434 %}
7435
7436 // Load Integer (32 bit unsigned) into long
7437 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7438 %{
7439 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7440
7441 ins_cost(VOLATILE_REF_COST);
7442 format %{ "ldarw $dst, $mem\t# int" %}
7443
7444 ins_encode(aarch64_enc_ldarw(dst, mem));
7445
7446 ins_pipe(pipe_serial);
7447 %}
7448
7449 // Load Long (64 bit signed)
7450 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7451 %{
7452 match(Set dst (LoadL mem));
7453
7454 ins_cost(VOLATILE_REF_COST);
7455 format %{ "ldar $dst, $mem\t# int" %}
7456
7457 ins_encode(aarch64_enc_ldar(dst, mem));
7458
7459 ins_pipe(pipe_serial);
7460 %}
7461
7462 // Load Pointer
7463 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7464 %{
7465 match(Set dst (LoadP mem));
7466 predicate(n->as_Load()->barrier_data() == 0);
7467
7468 ins_cost(VOLATILE_REF_COST);
7469 format %{ "ldar $dst, $mem\t# ptr" %}
7470
7471 ins_encode(aarch64_enc_ldar(dst, mem));
7472
7473 ins_pipe(pipe_serial);
7474 %}
7475
7476 // Load Compressed Pointer
7477 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7478 %{
7479 match(Set dst (LoadN mem));
7480 predicate(n->as_Load()->barrier_data() == 0);
7481
7482 ins_cost(VOLATILE_REF_COST);
7483 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7484
7485 ins_encode(aarch64_enc_ldarw(dst, mem));
7486
7487 ins_pipe(pipe_serial);
7488 %}
7489
7490 // Load Float
7491 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7492 %{
7493 match(Set dst (LoadF mem));
7494
7495 ins_cost(VOLATILE_REF_COST);
7496 format %{ "ldars $dst, $mem\t# float" %}
7497
7498 ins_encode( aarch64_enc_fldars(dst, mem) );
7499
7500 ins_pipe(pipe_serial);
7501 %}
7502
7503 // Load Double
7504 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7505 %{
7506 match(Set dst (LoadD mem));
7507
7508 ins_cost(VOLATILE_REF_COST);
7509 format %{ "ldard $dst, $mem\t# double" %}
7510
7511 ins_encode( aarch64_enc_fldard(dst, mem) );
7512
7513 ins_pipe(pipe_serial);
7514 %}
7515
7516 // Store Byte
7517 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7518 %{
7519 match(Set mem (StoreB mem src));
7520
7521 ins_cost(VOLATILE_REF_COST);
7522 format %{ "stlrb $src, $mem\t# byte" %}
7523
7524 ins_encode(aarch64_enc_stlrb(src, mem));
7525
7526 ins_pipe(pipe_class_memory);
7527 %}
7528
7529 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7530 %{
7531 match(Set mem (StoreB mem zero));
7532
7533 ins_cost(VOLATILE_REF_COST);
7534 format %{ "stlrb zr, $mem\t# byte" %}
7535
7536 ins_encode(aarch64_enc_stlrb0(mem));
7537
7538 ins_pipe(pipe_class_memory);
7539 %}
7540
7541 // Store Char/Short
7542 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7543 %{
7544 match(Set mem (StoreC mem src));
7545
7546 ins_cost(VOLATILE_REF_COST);
7547 format %{ "stlrh $src, $mem\t# short" %}
7548
7549 ins_encode(aarch64_enc_stlrh(src, mem));
7550
7551 ins_pipe(pipe_class_memory);
7552 %}
7553
7554 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7555 %{
7556 match(Set mem (StoreC mem zero));
7557
7558 ins_cost(VOLATILE_REF_COST);
7559 format %{ "stlrh zr, $mem\t# short" %}
7560
7561 ins_encode(aarch64_enc_stlrh0(mem));
7562
7563 ins_pipe(pipe_class_memory);
7564 %}
7565
7566 // Store Integer
7567
7568 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7569 %{
7570 match(Set mem(StoreI mem src));
7571
7572 ins_cost(VOLATILE_REF_COST);
7573 format %{ "stlrw $src, $mem\t# int" %}
7574
7575 ins_encode(aarch64_enc_stlrw(src, mem));
7576
7577 ins_pipe(pipe_class_memory);
7578 %}
7579
7580 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7581 %{
7582 match(Set mem(StoreI mem zero));
7583
7584 ins_cost(VOLATILE_REF_COST);
7585 format %{ "stlrw zr, $mem\t# int" %}
7586
7587 ins_encode(aarch64_enc_stlrw0(mem));
7588
7589 ins_pipe(pipe_class_memory);
7590 %}
7591
7592 // Store Long (64 bit signed)
7593 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7594 %{
7595 match(Set mem (StoreL mem src));
7596
7597 ins_cost(VOLATILE_REF_COST);
7598 format %{ "stlr $src, $mem\t# int" %}
7599
7600 ins_encode(aarch64_enc_stlr(src, mem));
7601
7602 ins_pipe(pipe_class_memory);
7603 %}
7604
7605 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7606 %{
7607 match(Set mem (StoreL mem zero));
7608
7609 ins_cost(VOLATILE_REF_COST);
7610 format %{ "stlr zr, $mem\t# int" %}
7611
7612 ins_encode(aarch64_enc_stlr0(mem));
7613
7614 ins_pipe(pipe_class_memory);
7615 %}
7616
7617 // Store Pointer
7618 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7619 %{
7620 match(Set mem (StoreP mem src));
7621 predicate(n->as_Store()->barrier_data() == 0);
7622
7623 ins_cost(VOLATILE_REF_COST);
7624 format %{ "stlr $src, $mem\t# ptr" %}
7625
7626 ins_encode(aarch64_enc_stlr(src, mem));
7627
7628 ins_pipe(pipe_class_memory);
7629 %}
7630
7631 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7632 %{
7633 match(Set mem (StoreP mem zero));
7634 predicate(n->as_Store()->barrier_data() == 0);
7635
7636 ins_cost(VOLATILE_REF_COST);
7637 format %{ "stlr zr, $mem\t# ptr" %}
7638
7639 ins_encode(aarch64_enc_stlr0(mem));
7640
7641 ins_pipe(pipe_class_memory);
7642 %}
7643
7644 // Store Compressed Pointer
7645 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7646 %{
7647 match(Set mem (StoreN mem src));
7648 predicate(n->as_Store()->barrier_data() == 0);
7649
7650 ins_cost(VOLATILE_REF_COST);
7651 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7652
7653 ins_encode(aarch64_enc_stlrw(src, mem));
7654
7655 ins_pipe(pipe_class_memory);
7656 %}
7657
7658 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7659 %{
7660 match(Set mem (StoreN mem zero));
7661 predicate(n->as_Store()->barrier_data() == 0);
7662
7663 ins_cost(VOLATILE_REF_COST);
7664 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7665
7666 ins_encode(aarch64_enc_stlrw0(mem));
7667
7668 ins_pipe(pipe_class_memory);
7669 %}
7670
7671 // Store Float
7672 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7673 %{
7674 match(Set mem (StoreF mem src));
7675
7676 ins_cost(VOLATILE_REF_COST);
7677 format %{ "stlrs $src, $mem\t# float" %}
7678
7679 ins_encode( aarch64_enc_fstlrs(src, mem) );
7680
7681 ins_pipe(pipe_class_memory);
7682 %}
7683
7684 // TODO
7685 // implement storeImmF0 and storeFImmPacked
7686
7687 // Store Double
7688 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7689 %{
7690 match(Set mem (StoreD mem src));
7691
7692 ins_cost(VOLATILE_REF_COST);
7693 format %{ "stlrd $src, $mem\t# double" %}
7694
7695 ins_encode( aarch64_enc_fstlrd(src, mem) );
7696
7697 ins_pipe(pipe_class_memory);
7698 %}
7699
7700 // ---------------- end of volatile loads and stores ----------------
7701
7702 instruct cacheWB(indirect addr)
7703 %{
7704 predicate(VM_Version::supports_data_cache_line_flush());
7705 match(CacheWB addr);
7706
7707 ins_cost(100);
7708 format %{"cache wb $addr" %}
7709 ins_encode %{
7710 assert($addr->index_position() < 0, "should be");
7711 assert($addr$$disp == 0, "should be");
7712 __ cache_wb(Address($addr$$base$$Register, 0));
7713 %}
7714 ins_pipe(pipe_slow); // XXX
7715 %}
7716
7717 instruct cacheWBPreSync()
7718 %{
7719 predicate(VM_Version::supports_data_cache_line_flush());
7720 match(CacheWBPreSync);
7721
7722 ins_cost(100);
7723 format %{"cache wb presync" %}
7724 ins_encode %{
7725 __ cache_wbsync(true);
7726 %}
7727 ins_pipe(pipe_slow); // XXX
7728 %}
7729
7730 instruct cacheWBPostSync()
7731 %{
7732 predicate(VM_Version::supports_data_cache_line_flush());
7733 match(CacheWBPostSync);
7734
7735 ins_cost(100);
7736 format %{"cache wb postsync" %}
7737 ins_encode %{
7738 __ cache_wbsync(false);
7739 %}
7740 ins_pipe(pipe_slow); // XXX
7741 %}
7742
7743 // ============================================================================
7744 // BSWAP Instructions
7745
7746 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7747 match(Set dst (ReverseBytesI src));
7748
7749 ins_cost(INSN_COST);
7750 format %{ "revw $dst, $src" %}
7751
7752 ins_encode %{
7753 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7754 %}
7755
7756 ins_pipe(ialu_reg);
7757 %}
7758
7759 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7760 match(Set dst (ReverseBytesL src));
7761
7762 ins_cost(INSN_COST);
7763 format %{ "rev $dst, $src" %}
7764
7765 ins_encode %{
7766 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7767 %}
7768
7769 ins_pipe(ialu_reg);
7770 %}
7771
7772 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7773 match(Set dst (ReverseBytesUS src));
7774
7775 ins_cost(INSN_COST);
7776 format %{ "rev16w $dst, $src" %}
7777
7778 ins_encode %{
7779 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7780 %}
7781
7782 ins_pipe(ialu_reg);
7783 %}
7784
7785 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7786 match(Set dst (ReverseBytesS src));
7787
7788 ins_cost(INSN_COST);
7789 format %{ "rev16w $dst, $src\n\t"
7790 "sbfmw $dst, $dst, #0, #15" %}
7791
7792 ins_encode %{
7793 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7794 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7795 %}
7796
7797 ins_pipe(ialu_reg);
7798 %}
7799
7800 // ============================================================================
7801 // Zero Count Instructions
7802
7803 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7804 match(Set dst (CountLeadingZerosI src));
7805
7806 ins_cost(INSN_COST);
7807 format %{ "clzw $dst, $src" %}
7808 ins_encode %{
7809 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7810 %}
7811
7812 ins_pipe(ialu_reg);
7813 %}
7814
7815 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7816 match(Set dst (CountLeadingZerosL src));
7817
7818 ins_cost(INSN_COST);
7819 format %{ "clz $dst, $src" %}
7820 ins_encode %{
7821 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7822 %}
7823
7824 ins_pipe(ialu_reg);
7825 %}
7826
7827 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7828 match(Set dst (CountTrailingZerosI src));
7829
7830 ins_cost(INSN_COST * 2);
7831 format %{ "rbitw $dst, $src\n\t"
7832 "clzw $dst, $dst" %}
7833 ins_encode %{
7834 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7835 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7836 %}
7837
7838 ins_pipe(ialu_reg);
7839 %}
7840
7841 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7842 match(Set dst (CountTrailingZerosL src));
7843
7844 ins_cost(INSN_COST * 2);
7845 format %{ "rbit $dst, $src\n\t"
7846 "clz $dst, $dst" %}
7847 ins_encode %{
7848 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7849 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7850 %}
7851
7852 ins_pipe(ialu_reg);
7853 %}
7854
7855 //---------- Population Count Instructions -------------------------------------
7856 //
7857
7858 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7859 match(Set dst (PopCountI src));
7860 effect(TEMP tmp);
7861 ins_cost(INSN_COST * 13);
7862
7863 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7864 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7865 "addv $tmp, $tmp\t# vector (8B)\n\t"
7866 "mov $dst, $tmp\t# vector (1D)" %}
7867 ins_encode %{
7868 __ fmovs($tmp$$FloatRegister, $src$$Register);
7869 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7870 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7871 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7872 %}
7873
7874 ins_pipe(pipe_class_default);
7875 %}
7876
7877 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7878 match(Set dst (PopCountI (LoadI mem)));
7879 effect(TEMP tmp);
7880 ins_cost(INSN_COST * 13);
7881
7882 format %{ "ldrs $tmp, $mem\n\t"
7883 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7884 "addv $tmp, $tmp\t# vector (8B)\n\t"
7885 "mov $dst, $tmp\t# vector (1D)" %}
7886 ins_encode %{
7887 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7888 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7889 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7890 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7891 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7892 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7893 %}
7894
7895 ins_pipe(pipe_class_default);
7896 %}
7897
7898 // Note: Long.bitCount(long) returns an int.
7899 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7900 match(Set dst (PopCountL src));
7901 effect(TEMP tmp);
7902 ins_cost(INSN_COST * 13);
7903
7904 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7905 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7906 "addv $tmp, $tmp\t# vector (8B)\n\t"
7907 "mov $dst, $tmp\t# vector (1D)" %}
7908 ins_encode %{
7909 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7910 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7911 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7912 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7913 %}
7914
7915 ins_pipe(pipe_class_default);
7916 %}
7917
7918 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7919 match(Set dst (PopCountL (LoadL mem)));
7920 effect(TEMP tmp);
7921 ins_cost(INSN_COST * 13);
7922
7923 format %{ "ldrd $tmp, $mem\n\t"
7924 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7925 "addv $tmp, $tmp\t# vector (8B)\n\t"
7926 "mov $dst, $tmp\t# vector (1D)" %}
7927 ins_encode %{
7928 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7929 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7930 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7931 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7932 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7933 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7934 %}
7935
7936 ins_pipe(pipe_class_default);
7937 %}
7938
7939 // ============================================================================
7940 // VerifyVectorAlignment Instruction
7941
7942 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7943 match(Set addr (VerifyVectorAlignment addr mask));
7944 effect(KILL cr);
7945 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7946 ins_encode %{
7947 Label Lskip;
7948 // check if masked bits of addr are zero
7949 __ tst($addr$$Register, $mask$$constant);
7950 __ br(Assembler::EQ, Lskip);
7951 __ stop("verify_vector_alignment found a misaligned vector memory access");
7952 __ bind(Lskip);
7953 %}
7954 ins_pipe(pipe_slow);
7955 %}
7956
7957 // ============================================================================
7958 // MemBar Instruction
7959
7960 instruct load_fence() %{
7961 match(LoadFence);
7962 ins_cost(VOLATILE_REF_COST);
7963
7964 format %{ "load_fence" %}
7965
7966 ins_encode %{
7967 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7968 %}
7969 ins_pipe(pipe_serial);
7970 %}
7971
7972 instruct unnecessary_membar_acquire() %{
7973 predicate(unnecessary_acquire(n));
7974 match(MemBarAcquire);
7975 ins_cost(0);
7976
7977 format %{ "membar_acquire (elided)" %}
7978
7979 ins_encode %{
7980 __ block_comment("membar_acquire (elided)");
7981 %}
7982
7983 ins_pipe(pipe_class_empty);
7984 %}
7985
7986 instruct membar_acquire() %{
7987 match(MemBarAcquire);
7988 ins_cost(VOLATILE_REF_COST);
7989
7990 format %{ "membar_acquire\n\t"
7991 "dmb ishld" %}
7992
7993 ins_encode %{
7994 __ block_comment("membar_acquire");
7995 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7996 %}
7997
7998 ins_pipe(pipe_serial);
7999 %}
8000
8001
8002 instruct membar_acquire_lock() %{
8003 match(MemBarAcquireLock);
8004 ins_cost(VOLATILE_REF_COST);
8005
8006 format %{ "membar_acquire_lock (elided)" %}
8007
8008 ins_encode %{
8009 __ block_comment("membar_acquire_lock (elided)");
8010 %}
8011
8012 ins_pipe(pipe_serial);
8013 %}
8014
8015 instruct store_fence() %{
8016 match(StoreFence);
8017 ins_cost(VOLATILE_REF_COST);
8018
8019 format %{ "store_fence" %}
8020
8021 ins_encode %{
8022 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8023 %}
8024 ins_pipe(pipe_serial);
8025 %}
8026
8027 instruct unnecessary_membar_release() %{
8028 predicate(unnecessary_release(n));
8029 match(MemBarRelease);
8030 ins_cost(0);
8031
8032 format %{ "membar_release (elided)" %}
8033
8034 ins_encode %{
8035 __ block_comment("membar_release (elided)");
8036 %}
8037 ins_pipe(pipe_serial);
8038 %}
8039
8040 instruct membar_release() %{
8041 match(MemBarRelease);
8042 ins_cost(VOLATILE_REF_COST);
8043
8044 format %{ "membar_release\n\t"
8045 "dmb ishst\n\tdmb ishld" %}
8046
8047 ins_encode %{
8048 __ block_comment("membar_release");
8049 // These will be merged if AlwaysMergeDMB is enabled.
8050 __ membar(Assembler::StoreStore);
8051 __ membar(Assembler::LoadStore);
8052 %}
8053 ins_pipe(pipe_serial);
8054 %}
8055
8056 instruct membar_storestore() %{
8057 match(MemBarStoreStore);
8058 match(StoreStoreFence);
8059 ins_cost(VOLATILE_REF_COST);
8060
8061 format %{ "MEMBAR-store-store" %}
8062
8063 ins_encode %{
8064 __ membar(Assembler::StoreStore);
8065 %}
8066 ins_pipe(pipe_serial);
8067 %}
8068
8069 instruct membar_release_lock() %{
8070 match(MemBarReleaseLock);
8071 ins_cost(VOLATILE_REF_COST);
8072
8073 format %{ "membar_release_lock (elided)" %}
8074
8075 ins_encode %{
8076 __ block_comment("membar_release_lock (elided)");
8077 %}
8078
8079 ins_pipe(pipe_serial);
8080 %}
8081
8082 instruct unnecessary_membar_volatile() %{
8083 predicate(unnecessary_volatile(n));
8084 match(MemBarVolatile);
8085 ins_cost(0);
8086
8087 format %{ "membar_volatile (elided)" %}
8088
8089 ins_encode %{
8090 __ block_comment("membar_volatile (elided)");
8091 %}
8092
8093 ins_pipe(pipe_serial);
8094 %}
8095
8096 instruct membar_volatile() %{
8097 match(MemBarVolatile);
8098 ins_cost(VOLATILE_REF_COST*100);
8099
8100 format %{ "membar_volatile\n\t"
8101 "dmb ish"%}
8102
8103 ins_encode %{
8104 __ block_comment("membar_volatile");
8105 __ membar(Assembler::StoreLoad);
8106 %}
8107
8108 ins_pipe(pipe_serial);
8109 %}
8110
8111 // ============================================================================
8112 // Cast/Convert Instructions
8113
8114 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8115 match(Set dst (CastX2P src));
8116
8117 ins_cost(INSN_COST);
8118 format %{ "mov $dst, $src\t# long -> ptr" %}
8119
8120 ins_encode %{
8121 if ($dst$$reg != $src$$reg) {
8122 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8123 }
8124 %}
8125
8126 ins_pipe(ialu_reg);
8127 %}
8128
8129 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8130 match(Set dst (CastP2X src));
8131
8132 ins_cost(INSN_COST);
8133 format %{ "mov $dst, $src\t# ptr -> long" %}
8134
8135 ins_encode %{
8136 if ($dst$$reg != $src$$reg) {
8137 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8138 }
8139 %}
8140
8141 ins_pipe(ialu_reg);
8142 %}
8143
8144 // Convert oop into int for vectors alignment masking
8145 instruct convP2I(iRegINoSp dst, iRegP src) %{
8146 match(Set dst (ConvL2I (CastP2X src)));
8147
8148 ins_cost(INSN_COST);
8149 format %{ "movw $dst, $src\t# ptr -> int" %}
8150 ins_encode %{
8151 __ movw($dst$$Register, $src$$Register);
8152 %}
8153
8154 ins_pipe(ialu_reg);
8155 %}
8156
8157 // Convert compressed oop into int for vectors alignment masking
8158 // in case of 32bit oops (heap < 4Gb).
8159 instruct convN2I(iRegINoSp dst, iRegN src)
8160 %{
8161 predicate(CompressedOops::shift() == 0);
8162 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8163
8164 ins_cost(INSN_COST);
8165 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8166 ins_encode %{
8167 __ movw($dst$$Register, $src$$Register);
8168 %}
8169
8170 ins_pipe(ialu_reg);
8171 %}
8172
8173
8174 // Convert oop pointer into compressed form
8175 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8176 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8177 match(Set dst (EncodeP src));
8178 effect(KILL cr);
8179 ins_cost(INSN_COST * 3);
8180 format %{ "encode_heap_oop $dst, $src" %}
8181 ins_encode %{
8182 Register s = $src$$Register;
8183 Register d = $dst$$Register;
8184 __ encode_heap_oop(d, s);
8185 %}
8186 ins_pipe(ialu_reg);
8187 %}
8188
8189 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8190 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8191 match(Set dst (EncodeP src));
8192 ins_cost(INSN_COST * 3);
8193 format %{ "encode_heap_oop_not_null $dst, $src" %}
8194 ins_encode %{
8195 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8196 %}
8197 ins_pipe(ialu_reg);
8198 %}
8199
8200 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8201 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8202 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8203 match(Set dst (DecodeN src));
8204 ins_cost(INSN_COST * 3);
8205 format %{ "decode_heap_oop $dst, $src" %}
8206 ins_encode %{
8207 Register s = $src$$Register;
8208 Register d = $dst$$Register;
8209 __ decode_heap_oop(d, s);
8210 %}
8211 ins_pipe(ialu_reg);
8212 %}
8213
8214 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8215 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8216 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8217 match(Set dst (DecodeN src));
8218 ins_cost(INSN_COST * 3);
8219 format %{ "decode_heap_oop_not_null $dst, $src" %}
8220 ins_encode %{
8221 Register s = $src$$Register;
8222 Register d = $dst$$Register;
8223 __ decode_heap_oop_not_null(d, s);
8224 %}
8225 ins_pipe(ialu_reg);
8226 %}
8227
8228 // n.b. AArch64 implementations of encode_klass_not_null and
8229 // decode_klass_not_null do not modify the flags register so, unlike
8230 // Intel, we don't kill CR as a side effect here
8231
8232 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8233 match(Set dst (EncodePKlass src));
8234
8235 ins_cost(INSN_COST * 3);
8236 format %{ "encode_klass_not_null $dst,$src" %}
8237
8238 ins_encode %{
8239 Register src_reg = as_Register($src$$reg);
8240 Register dst_reg = as_Register($dst$$reg);
8241 __ encode_klass_not_null(dst_reg, src_reg);
8242 %}
8243
8244 ins_pipe(ialu_reg);
8245 %}
8246
8247 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8248 match(Set dst (DecodeNKlass src));
8249
8250 ins_cost(INSN_COST * 3);
8251 format %{ "decode_klass_not_null $dst,$src" %}
8252
8253 ins_encode %{
8254 Register src_reg = as_Register($src$$reg);
8255 Register dst_reg = as_Register($dst$$reg);
8256 if (dst_reg != src_reg) {
8257 __ decode_klass_not_null(dst_reg, src_reg);
8258 } else {
8259 __ decode_klass_not_null(dst_reg);
8260 }
8261 %}
8262
8263 ins_pipe(ialu_reg);
8264 %}
8265
8266 instruct checkCastPP(iRegPNoSp dst)
8267 %{
8268 match(Set dst (CheckCastPP dst));
8269
8270 size(0);
8271 format %{ "# checkcastPP of $dst" %}
8272 ins_encode(/* empty encoding */);
8273 ins_pipe(pipe_class_empty);
8274 %}
8275
8276 instruct castPP(iRegPNoSp dst)
8277 %{
8278 match(Set dst (CastPP dst));
8279
8280 size(0);
8281 format %{ "# castPP of $dst" %}
8282 ins_encode(/* empty encoding */);
8283 ins_pipe(pipe_class_empty);
8284 %}
8285
8286 instruct castII(iRegI dst)
8287 %{
8288 predicate(VerifyConstraintCasts == 0);
8289 match(Set dst (CastII dst));
8290
8291 size(0);
8292 format %{ "# castII of $dst" %}
8293 ins_encode(/* empty encoding */);
8294 ins_cost(0);
8295 ins_pipe(pipe_class_empty);
8296 %}
8297
8298 instruct castII_checked(iRegI dst, rFlagsReg cr)
8299 %{
8300 predicate(VerifyConstraintCasts > 0);
8301 match(Set dst (CastII dst));
8302 effect(KILL cr);
8303
8304 format %{ "# castII_checked of $dst" %}
8305 ins_encode %{
8306 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8307 %}
8308 ins_pipe(pipe_slow);
8309 %}
8310
8311 instruct castLL(iRegL dst)
8312 %{
8313 predicate(VerifyConstraintCasts == 0);
8314 match(Set dst (CastLL dst));
8315
8316 size(0);
8317 format %{ "# castLL of $dst" %}
8318 ins_encode(/* empty encoding */);
8319 ins_cost(0);
8320 ins_pipe(pipe_class_empty);
8321 %}
8322
8323 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8324 %{
8325 predicate(VerifyConstraintCasts > 0);
8326 match(Set dst (CastLL dst));
8327 effect(KILL cr);
8328
8329 format %{ "# castLL_checked of $dst" %}
8330 ins_encode %{
8331 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8332 %}
8333 ins_pipe(pipe_slow);
8334 %}
8335
8336 instruct castHH(vRegF dst)
8337 %{
8338 match(Set dst (CastHH dst));
8339 size(0);
8340 format %{ "# castHH of $dst" %}
8341 ins_encode(/* empty encoding */);
8342 ins_cost(0);
8343 ins_pipe(pipe_class_empty);
8344 %}
8345
8346 instruct castFF(vRegF dst)
8347 %{
8348 match(Set dst (CastFF dst));
8349
8350 size(0);
8351 format %{ "# castFF of $dst" %}
8352 ins_encode(/* empty encoding */);
8353 ins_cost(0);
8354 ins_pipe(pipe_class_empty);
8355 %}
8356
8357 instruct castDD(vRegD dst)
8358 %{
8359 match(Set dst (CastDD dst));
8360
8361 size(0);
8362 format %{ "# castDD of $dst" %}
8363 ins_encode(/* empty encoding */);
8364 ins_cost(0);
8365 ins_pipe(pipe_class_empty);
8366 %}
8367
8368 instruct castVV(vReg dst)
8369 %{
8370 match(Set dst (CastVV dst));
8371
8372 size(0);
8373 format %{ "# castVV of $dst" %}
8374 ins_encode(/* empty encoding */);
8375 ins_cost(0);
8376 ins_pipe(pipe_class_empty);
8377 %}
8378
8379 instruct castVVMask(pRegGov dst)
8380 %{
8381 match(Set dst (CastVV dst));
8382
8383 size(0);
8384 format %{ "# castVV of $dst" %}
8385 ins_encode(/* empty encoding */);
8386 ins_cost(0);
8387 ins_pipe(pipe_class_empty);
8388 %}
8389
8390 // ============================================================================
8391 // Atomic operation instructions
8392 //
8393
8394 // standard CompareAndSwapX when we are using barriers
8395 // these have higher priority than the rules selected by a predicate
8396
8397 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8398 // can't match them
8399
8400 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8401
8402 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8403 ins_cost(2 * VOLATILE_REF_COST);
8404
8405 effect(KILL cr);
8406
8407 format %{
8408 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8409 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8410 %}
8411
8412 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8413 aarch64_enc_cset_eq(res));
8414
8415 ins_pipe(pipe_slow);
8416 %}
8417
8418 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8419
8420 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8421 ins_cost(2 * VOLATILE_REF_COST);
8422
8423 effect(KILL cr);
8424
8425 format %{
8426 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8427 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8428 %}
8429
8430 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8431 aarch64_enc_cset_eq(res));
8432
8433 ins_pipe(pipe_slow);
8434 %}
8435
8436 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8437
8438 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8439 ins_cost(2 * VOLATILE_REF_COST);
8440
8441 effect(KILL cr);
8442
8443 format %{
8444 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8445 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8446 %}
8447
8448 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8449 aarch64_enc_cset_eq(res));
8450
8451 ins_pipe(pipe_slow);
8452 %}
8453
8454 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8455
8456 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8457 ins_cost(2 * VOLATILE_REF_COST);
8458
8459 effect(KILL cr);
8460
8461 format %{
8462 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8463 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8464 %}
8465
8466 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8467 aarch64_enc_cset_eq(res));
8468
8469 ins_pipe(pipe_slow);
8470 %}
8471
8472 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8473
8474 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8475 predicate(n->as_LoadStore()->barrier_data() == 0);
8476 ins_cost(2 * VOLATILE_REF_COST);
8477
8478 effect(KILL cr);
8479
8480 format %{
8481 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8482 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8483 %}
8484
8485 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8486 aarch64_enc_cset_eq(res));
8487
8488 ins_pipe(pipe_slow);
8489 %}
8490
8491 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8492
8493 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8494 predicate(n->as_LoadStore()->barrier_data() == 0);
8495 ins_cost(2 * VOLATILE_REF_COST);
8496
8497 effect(KILL cr);
8498
8499 format %{
8500 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8501 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8502 %}
8503
8504 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8505 aarch64_enc_cset_eq(res));
8506
8507 ins_pipe(pipe_slow);
8508 %}
8509
8510 // alternative CompareAndSwapX when we are eliding barriers
8511
8512 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8513
8514 predicate(needs_acquiring_load_exclusive(n));
8515 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8516 ins_cost(VOLATILE_REF_COST);
8517
8518 effect(KILL cr);
8519
8520 format %{
8521 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8522 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8523 %}
8524
8525 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8526 aarch64_enc_cset_eq(res));
8527
8528 ins_pipe(pipe_slow);
8529 %}
8530
8531 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8532
8533 predicate(needs_acquiring_load_exclusive(n));
8534 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8535 ins_cost(VOLATILE_REF_COST);
8536
8537 effect(KILL cr);
8538
8539 format %{
8540 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8541 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8542 %}
8543
8544 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8545 aarch64_enc_cset_eq(res));
8546
8547 ins_pipe(pipe_slow);
8548 %}
8549
8550 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8551
8552 predicate(needs_acquiring_load_exclusive(n));
8553 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8554 ins_cost(VOLATILE_REF_COST);
8555
8556 effect(KILL cr);
8557
8558 format %{
8559 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8560 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8561 %}
8562
8563 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8564 aarch64_enc_cset_eq(res));
8565
8566 ins_pipe(pipe_slow);
8567 %}
8568
8569 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8570
8571 predicate(needs_acquiring_load_exclusive(n));
8572 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8573 ins_cost(VOLATILE_REF_COST);
8574
8575 effect(KILL cr);
8576
8577 format %{
8578 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8579 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8580 %}
8581
8582 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8583 aarch64_enc_cset_eq(res));
8584
8585 ins_pipe(pipe_slow);
8586 %}
8587
8588 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8589
8590 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8591 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8592 ins_cost(VOLATILE_REF_COST);
8593
8594 effect(KILL cr);
8595
8596 format %{
8597 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8598 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8599 %}
8600
8601 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8602 aarch64_enc_cset_eq(res));
8603
8604 ins_pipe(pipe_slow);
8605 %}
8606
8607 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8608
8609 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8610 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8611 ins_cost(VOLATILE_REF_COST);
8612
8613 effect(KILL cr);
8614
8615 format %{
8616 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8617 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8618 %}
8619
8620 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8621 aarch64_enc_cset_eq(res));
8622
8623 ins_pipe(pipe_slow);
8624 %}
8625
8626
8627 // ---------------------------------------------------------------------
8628
8629 // BEGIN This section of the file is automatically generated. Do not edit --------------
8630
8631 // Sundry CAS operations. Note that release is always true,
8632 // regardless of the memory ordering of the CAS. This is because we
8633 // need the volatile case to be sequentially consistent but there is
8634 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8635 // can't check the type of memory ordering here, so we always emit a
8636 // STLXR.
8637
8638 // This section is generated from cas.m4
8639
8640
8641 // This pattern is generated automatically from cas.m4.
8642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8643 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8644 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8645 ins_cost(2 * VOLATILE_REF_COST);
8646 effect(TEMP_DEF res, KILL cr);
8647 format %{
8648 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8649 %}
8650 ins_encode %{
8651 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8652 Assembler::byte, /*acquire*/ false, /*release*/ true,
8653 /*weak*/ false, $res$$Register);
8654 __ sxtbw($res$$Register, $res$$Register);
8655 %}
8656 ins_pipe(pipe_slow);
8657 %}
8658
8659 // This pattern is generated automatically from cas.m4.
8660 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8661 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8662 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8663 ins_cost(2 * VOLATILE_REF_COST);
8664 effect(TEMP_DEF res, KILL cr);
8665 format %{
8666 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8667 %}
8668 ins_encode %{
8669 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8670 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8671 /*weak*/ false, $res$$Register);
8672 __ sxthw($res$$Register, $res$$Register);
8673 %}
8674 ins_pipe(pipe_slow);
8675 %}
8676
8677 // This pattern is generated automatically from cas.m4.
8678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8679 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8680 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8681 ins_cost(2 * VOLATILE_REF_COST);
8682 effect(TEMP_DEF res, KILL cr);
8683 format %{
8684 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8685 %}
8686 ins_encode %{
8687 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8688 Assembler::word, /*acquire*/ false, /*release*/ true,
8689 /*weak*/ false, $res$$Register);
8690 %}
8691 ins_pipe(pipe_slow);
8692 %}
8693
8694 // This pattern is generated automatically from cas.m4.
8695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8696 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8697 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8698 ins_cost(2 * VOLATILE_REF_COST);
8699 effect(TEMP_DEF res, KILL cr);
8700 format %{
8701 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8702 %}
8703 ins_encode %{
8704 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8705 Assembler::xword, /*acquire*/ false, /*release*/ true,
8706 /*weak*/ false, $res$$Register);
8707 %}
8708 ins_pipe(pipe_slow);
8709 %}
8710
8711 // This pattern is generated automatically from cas.m4.
8712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8713 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8714 predicate(n->as_LoadStore()->barrier_data() == 0);
8715 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8716 ins_cost(2 * VOLATILE_REF_COST);
8717 effect(TEMP_DEF res, KILL cr);
8718 format %{
8719 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8720 %}
8721 ins_encode %{
8722 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8723 Assembler::word, /*acquire*/ false, /*release*/ true,
8724 /*weak*/ false, $res$$Register);
8725 %}
8726 ins_pipe(pipe_slow);
8727 %}
8728
8729 // This pattern is generated automatically from cas.m4.
8730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8731 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8732 predicate(n->as_LoadStore()->barrier_data() == 0);
8733 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8734 ins_cost(2 * VOLATILE_REF_COST);
8735 effect(TEMP_DEF res, KILL cr);
8736 format %{
8737 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8738 %}
8739 ins_encode %{
8740 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8741 Assembler::xword, /*acquire*/ false, /*release*/ true,
8742 /*weak*/ false, $res$$Register);
8743 %}
8744 ins_pipe(pipe_slow);
8745 %}
8746
8747 // This pattern is generated automatically from cas.m4.
8748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8749 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8750 predicate(needs_acquiring_load_exclusive(n));
8751 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8752 ins_cost(VOLATILE_REF_COST);
8753 effect(TEMP_DEF res, KILL cr);
8754 format %{
8755 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8756 %}
8757 ins_encode %{
8758 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8759 Assembler::byte, /*acquire*/ true, /*release*/ true,
8760 /*weak*/ false, $res$$Register);
8761 __ sxtbw($res$$Register, $res$$Register);
8762 %}
8763 ins_pipe(pipe_slow);
8764 %}
8765
8766 // This pattern is generated automatically from cas.m4.
8767 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8768 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8769 predicate(needs_acquiring_load_exclusive(n));
8770 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8771 ins_cost(VOLATILE_REF_COST);
8772 effect(TEMP_DEF res, KILL cr);
8773 format %{
8774 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8775 %}
8776 ins_encode %{
8777 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8778 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8779 /*weak*/ false, $res$$Register);
8780 __ sxthw($res$$Register, $res$$Register);
8781 %}
8782 ins_pipe(pipe_slow);
8783 %}
8784
8785 // This pattern is generated automatically from cas.m4.
8786 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8787 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8788 predicate(needs_acquiring_load_exclusive(n));
8789 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8790 ins_cost(VOLATILE_REF_COST);
8791 effect(TEMP_DEF res, KILL cr);
8792 format %{
8793 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8794 %}
8795 ins_encode %{
8796 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8797 Assembler::word, /*acquire*/ true, /*release*/ true,
8798 /*weak*/ false, $res$$Register);
8799 %}
8800 ins_pipe(pipe_slow);
8801 %}
8802
8803 // This pattern is generated automatically from cas.m4.
8804 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8805 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8806 predicate(needs_acquiring_load_exclusive(n));
8807 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8808 ins_cost(VOLATILE_REF_COST);
8809 effect(TEMP_DEF res, KILL cr);
8810 format %{
8811 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8812 %}
8813 ins_encode %{
8814 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8815 Assembler::xword, /*acquire*/ true, /*release*/ true,
8816 /*weak*/ false, $res$$Register);
8817 %}
8818 ins_pipe(pipe_slow);
8819 %}
8820
8821 // This pattern is generated automatically from cas.m4.
8822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8823 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8824 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8825 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8826 ins_cost(VOLATILE_REF_COST);
8827 effect(TEMP_DEF res, KILL cr);
8828 format %{
8829 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8830 %}
8831 ins_encode %{
8832 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8833 Assembler::word, /*acquire*/ true, /*release*/ true,
8834 /*weak*/ false, $res$$Register);
8835 %}
8836 ins_pipe(pipe_slow);
8837 %}
8838
8839 // This pattern is generated automatically from cas.m4.
8840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8841 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8842 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8843 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8844 ins_cost(VOLATILE_REF_COST);
8845 effect(TEMP_DEF res, KILL cr);
8846 format %{
8847 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8848 %}
8849 ins_encode %{
8850 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8851 Assembler::xword, /*acquire*/ true, /*release*/ true,
8852 /*weak*/ false, $res$$Register);
8853 %}
8854 ins_pipe(pipe_slow);
8855 %}
8856
8857 // This pattern is generated automatically from cas.m4.
8858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8859 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8860 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8861 ins_cost(2 * VOLATILE_REF_COST);
8862 effect(KILL cr);
8863 format %{
8864 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8865 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8866 %}
8867 ins_encode %{
8868 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8869 Assembler::byte, /*acquire*/ false, /*release*/ true,
8870 /*weak*/ true, noreg);
8871 __ csetw($res$$Register, Assembler::EQ);
8872 %}
8873 ins_pipe(pipe_slow);
8874 %}
8875
8876 // This pattern is generated automatically from cas.m4.
8877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8878 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8879 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8880 ins_cost(2 * VOLATILE_REF_COST);
8881 effect(KILL cr);
8882 format %{
8883 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8884 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8885 %}
8886 ins_encode %{
8887 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8888 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8889 /*weak*/ true, noreg);
8890 __ csetw($res$$Register, Assembler::EQ);
8891 %}
8892 ins_pipe(pipe_slow);
8893 %}
8894
8895 // This pattern is generated automatically from cas.m4.
8896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8897 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8898 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8899 ins_cost(2 * VOLATILE_REF_COST);
8900 effect(KILL cr);
8901 format %{
8902 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8903 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8904 %}
8905 ins_encode %{
8906 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8907 Assembler::word, /*acquire*/ false, /*release*/ true,
8908 /*weak*/ true, noreg);
8909 __ csetw($res$$Register, Assembler::EQ);
8910 %}
8911 ins_pipe(pipe_slow);
8912 %}
8913
8914 // This pattern is generated automatically from cas.m4.
8915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8916 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8917 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8918 ins_cost(2 * VOLATILE_REF_COST);
8919 effect(KILL cr);
8920 format %{
8921 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8922 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8923 %}
8924 ins_encode %{
8925 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8926 Assembler::xword, /*acquire*/ false, /*release*/ true,
8927 /*weak*/ true, noreg);
8928 __ csetw($res$$Register, Assembler::EQ);
8929 %}
8930 ins_pipe(pipe_slow);
8931 %}
8932
8933 // This pattern is generated automatically from cas.m4.
8934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8935 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8936 predicate(n->as_LoadStore()->barrier_data() == 0);
8937 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8938 ins_cost(2 * VOLATILE_REF_COST);
8939 effect(KILL cr);
8940 format %{
8941 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8942 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8943 %}
8944 ins_encode %{
8945 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8946 Assembler::word, /*acquire*/ false, /*release*/ true,
8947 /*weak*/ true, noreg);
8948 __ csetw($res$$Register, Assembler::EQ);
8949 %}
8950 ins_pipe(pipe_slow);
8951 %}
8952
8953 // This pattern is generated automatically from cas.m4.
8954 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8955 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8956 predicate(n->as_LoadStore()->barrier_data() == 0);
8957 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8958 ins_cost(2 * VOLATILE_REF_COST);
8959 effect(KILL cr);
8960 format %{
8961 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8962 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8963 %}
8964 ins_encode %{
8965 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8966 Assembler::xword, /*acquire*/ false, /*release*/ true,
8967 /*weak*/ true, noreg);
8968 __ csetw($res$$Register, Assembler::EQ);
8969 %}
8970 ins_pipe(pipe_slow);
8971 %}
8972
8973 // This pattern is generated automatically from cas.m4.
8974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8975 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8976 predicate(needs_acquiring_load_exclusive(n));
8977 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8978 ins_cost(VOLATILE_REF_COST);
8979 effect(KILL cr);
8980 format %{
8981 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8982 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8983 %}
8984 ins_encode %{
8985 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8986 Assembler::byte, /*acquire*/ true, /*release*/ true,
8987 /*weak*/ true, noreg);
8988 __ csetw($res$$Register, Assembler::EQ);
8989 %}
8990 ins_pipe(pipe_slow);
8991 %}
8992
8993 // This pattern is generated automatically from cas.m4.
8994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8995 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8996 predicate(needs_acquiring_load_exclusive(n));
8997 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8998 ins_cost(VOLATILE_REF_COST);
8999 effect(KILL cr);
9000 format %{
9001 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9002 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9003 %}
9004 ins_encode %{
9005 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9006 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9007 /*weak*/ true, noreg);
9008 __ csetw($res$$Register, Assembler::EQ);
9009 %}
9010 ins_pipe(pipe_slow);
9011 %}
9012
9013 // This pattern is generated automatically from cas.m4.
9014 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9015 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9016 predicate(needs_acquiring_load_exclusive(n));
9017 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9018 ins_cost(VOLATILE_REF_COST);
9019 effect(KILL cr);
9020 format %{
9021 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9022 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9023 %}
9024 ins_encode %{
9025 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9026 Assembler::word, /*acquire*/ true, /*release*/ true,
9027 /*weak*/ true, noreg);
9028 __ csetw($res$$Register, Assembler::EQ);
9029 %}
9030 ins_pipe(pipe_slow);
9031 %}
9032
9033 // This pattern is generated automatically from cas.m4.
9034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9035 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9036 predicate(needs_acquiring_load_exclusive(n));
9037 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9038 ins_cost(VOLATILE_REF_COST);
9039 effect(KILL cr);
9040 format %{
9041 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9042 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9043 %}
9044 ins_encode %{
9045 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9046 Assembler::xword, /*acquire*/ true, /*release*/ true,
9047 /*weak*/ true, noreg);
9048 __ csetw($res$$Register, Assembler::EQ);
9049 %}
9050 ins_pipe(pipe_slow);
9051 %}
9052
9053 // This pattern is generated automatically from cas.m4.
9054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9055 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9056 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9057 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9058 ins_cost(VOLATILE_REF_COST);
9059 effect(KILL cr);
9060 format %{
9061 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9062 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9063 %}
9064 ins_encode %{
9065 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9066 Assembler::word, /*acquire*/ true, /*release*/ true,
9067 /*weak*/ true, noreg);
9068 __ csetw($res$$Register, Assembler::EQ);
9069 %}
9070 ins_pipe(pipe_slow);
9071 %}
9072
9073 // This pattern is generated automatically from cas.m4.
9074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9075 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9076 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9077 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9078 ins_cost(VOLATILE_REF_COST);
9079 effect(KILL cr);
9080 format %{
9081 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9082 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9083 %}
9084 ins_encode %{
9085 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9086 Assembler::xword, /*acquire*/ true, /*release*/ true,
9087 /*weak*/ true, noreg);
9088 __ csetw($res$$Register, Assembler::EQ);
9089 %}
9090 ins_pipe(pipe_slow);
9091 %}
9092
9093 // END This section of the file is automatically generated. Do not edit --------------
9094 // ---------------------------------------------------------------------
9095
9096 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9097 match(Set prev (GetAndSetI mem newv));
9098 ins_cost(2 * VOLATILE_REF_COST);
9099 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9100 ins_encode %{
9101 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9102 %}
9103 ins_pipe(pipe_serial);
9104 %}
9105
9106 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9107 match(Set prev (GetAndSetL mem newv));
9108 ins_cost(2 * VOLATILE_REF_COST);
9109 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9110 ins_encode %{
9111 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9112 %}
9113 ins_pipe(pipe_serial);
9114 %}
9115
9116 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9117 predicate(n->as_LoadStore()->barrier_data() == 0);
9118 match(Set prev (GetAndSetN mem newv));
9119 ins_cost(2 * VOLATILE_REF_COST);
9120 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9121 ins_encode %{
9122 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9123 %}
9124 ins_pipe(pipe_serial);
9125 %}
9126
9127 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9128 predicate(n->as_LoadStore()->barrier_data() == 0);
9129 match(Set prev (GetAndSetP mem newv));
9130 ins_cost(2 * VOLATILE_REF_COST);
9131 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9132 ins_encode %{
9133 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9134 %}
9135 ins_pipe(pipe_serial);
9136 %}
9137
9138 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9139 predicate(needs_acquiring_load_exclusive(n));
9140 match(Set prev (GetAndSetI mem newv));
9141 ins_cost(VOLATILE_REF_COST);
9142 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9143 ins_encode %{
9144 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9145 %}
9146 ins_pipe(pipe_serial);
9147 %}
9148
9149 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9150 predicate(needs_acquiring_load_exclusive(n));
9151 match(Set prev (GetAndSetL mem newv));
9152 ins_cost(VOLATILE_REF_COST);
9153 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9154 ins_encode %{
9155 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9156 %}
9157 ins_pipe(pipe_serial);
9158 %}
9159
9160 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9161 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9162 match(Set prev (GetAndSetN mem newv));
9163 ins_cost(VOLATILE_REF_COST);
9164 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9165 ins_encode %{
9166 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9167 %}
9168 ins_pipe(pipe_serial);
9169 %}
9170
9171 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9172 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9173 match(Set prev (GetAndSetP mem newv));
9174 ins_cost(VOLATILE_REF_COST);
9175 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9176 ins_encode %{
9177 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9178 %}
9179 ins_pipe(pipe_serial);
9180 %}
9181
9182
9183 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9184 match(Set newval (GetAndAddL mem incr));
9185 ins_cost(2 * VOLATILE_REF_COST + 1);
9186 format %{ "get_and_addL $newval, [$mem], $incr" %}
9187 ins_encode %{
9188 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9189 %}
9190 ins_pipe(pipe_serial);
9191 %}
9192
9193 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9194 predicate(n->as_LoadStore()->result_not_used());
9195 match(Set dummy (GetAndAddL mem incr));
9196 ins_cost(2 * VOLATILE_REF_COST);
9197 format %{ "get_and_addL [$mem], $incr" %}
9198 ins_encode %{
9199 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9200 %}
9201 ins_pipe(pipe_serial);
9202 %}
9203
9204 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9205 match(Set newval (GetAndAddL mem incr));
9206 ins_cost(2 * VOLATILE_REF_COST + 1);
9207 format %{ "get_and_addL $newval, [$mem], $incr" %}
9208 ins_encode %{
9209 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9210 %}
9211 ins_pipe(pipe_serial);
9212 %}
9213
9214 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9215 predicate(n->as_LoadStore()->result_not_used());
9216 match(Set dummy (GetAndAddL mem incr));
9217 ins_cost(2 * VOLATILE_REF_COST);
9218 format %{ "get_and_addL [$mem], $incr" %}
9219 ins_encode %{
9220 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9221 %}
9222 ins_pipe(pipe_serial);
9223 %}
9224
9225 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9226 match(Set newval (GetAndAddI mem incr));
9227 ins_cost(2 * VOLATILE_REF_COST + 1);
9228 format %{ "get_and_addI $newval, [$mem], $incr" %}
9229 ins_encode %{
9230 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9231 %}
9232 ins_pipe(pipe_serial);
9233 %}
9234
9235 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9236 predicate(n->as_LoadStore()->result_not_used());
9237 match(Set dummy (GetAndAddI mem incr));
9238 ins_cost(2 * VOLATILE_REF_COST);
9239 format %{ "get_and_addI [$mem], $incr" %}
9240 ins_encode %{
9241 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9242 %}
9243 ins_pipe(pipe_serial);
9244 %}
9245
9246 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9247 match(Set newval (GetAndAddI mem incr));
9248 ins_cost(2 * VOLATILE_REF_COST + 1);
9249 format %{ "get_and_addI $newval, [$mem], $incr" %}
9250 ins_encode %{
9251 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9252 %}
9253 ins_pipe(pipe_serial);
9254 %}
9255
9256 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9257 predicate(n->as_LoadStore()->result_not_used());
9258 match(Set dummy (GetAndAddI mem incr));
9259 ins_cost(2 * VOLATILE_REF_COST);
9260 format %{ "get_and_addI [$mem], $incr" %}
9261 ins_encode %{
9262 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9263 %}
9264 ins_pipe(pipe_serial);
9265 %}
9266
9267 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9268 predicate(needs_acquiring_load_exclusive(n));
9269 match(Set newval (GetAndAddL mem incr));
9270 ins_cost(VOLATILE_REF_COST + 1);
9271 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9272 ins_encode %{
9273 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9274 %}
9275 ins_pipe(pipe_serial);
9276 %}
9277
9278 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9279 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9280 match(Set dummy (GetAndAddL mem incr));
9281 ins_cost(VOLATILE_REF_COST);
9282 format %{ "get_and_addL_acq [$mem], $incr" %}
9283 ins_encode %{
9284 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9285 %}
9286 ins_pipe(pipe_serial);
9287 %}
9288
9289 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9290 predicate(needs_acquiring_load_exclusive(n));
9291 match(Set newval (GetAndAddL mem incr));
9292 ins_cost(VOLATILE_REF_COST + 1);
9293 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9294 ins_encode %{
9295 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9296 %}
9297 ins_pipe(pipe_serial);
9298 %}
9299
9300 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9301 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9302 match(Set dummy (GetAndAddL mem incr));
9303 ins_cost(VOLATILE_REF_COST);
9304 format %{ "get_and_addL_acq [$mem], $incr" %}
9305 ins_encode %{
9306 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9307 %}
9308 ins_pipe(pipe_serial);
9309 %}
9310
9311 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9312 predicate(needs_acquiring_load_exclusive(n));
9313 match(Set newval (GetAndAddI mem incr));
9314 ins_cost(VOLATILE_REF_COST + 1);
9315 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9316 ins_encode %{
9317 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9318 %}
9319 ins_pipe(pipe_serial);
9320 %}
9321
9322 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9323 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9324 match(Set dummy (GetAndAddI mem incr));
9325 ins_cost(VOLATILE_REF_COST);
9326 format %{ "get_and_addI_acq [$mem], $incr" %}
9327 ins_encode %{
9328 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9329 %}
9330 ins_pipe(pipe_serial);
9331 %}
9332
9333 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9334 predicate(needs_acquiring_load_exclusive(n));
9335 match(Set newval (GetAndAddI mem incr));
9336 ins_cost(VOLATILE_REF_COST + 1);
9337 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9338 ins_encode %{
9339 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9340 %}
9341 ins_pipe(pipe_serial);
9342 %}
9343
9344 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9345 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9346 match(Set dummy (GetAndAddI mem incr));
9347 ins_cost(VOLATILE_REF_COST);
9348 format %{ "get_and_addI_acq [$mem], $incr" %}
9349 ins_encode %{
9350 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9351 %}
9352 ins_pipe(pipe_serial);
9353 %}
9354
9355 // Manifest a CmpU result in an integer register.
9356 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9357 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9358 %{
9359 match(Set dst (CmpU3 src1 src2));
9360 effect(KILL flags);
9361
9362 ins_cost(INSN_COST * 3);
9363 format %{
9364 "cmpw $src1, $src2\n\t"
9365 "csetw $dst, ne\n\t"
9366 "cnegw $dst, lo\t# CmpU3(reg)"
9367 %}
9368 ins_encode %{
9369 __ cmpw($src1$$Register, $src2$$Register);
9370 __ csetw($dst$$Register, Assembler::NE);
9371 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9372 %}
9373
9374 ins_pipe(pipe_class_default);
9375 %}
9376
9377 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9378 %{
9379 match(Set dst (CmpU3 src1 src2));
9380 effect(KILL flags);
9381
9382 ins_cost(INSN_COST * 3);
9383 format %{
9384 "subsw zr, $src1, $src2\n\t"
9385 "csetw $dst, ne\n\t"
9386 "cnegw $dst, lo\t# CmpU3(imm)"
9387 %}
9388 ins_encode %{
9389 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9390 __ csetw($dst$$Register, Assembler::NE);
9391 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9392 %}
9393
9394 ins_pipe(pipe_class_default);
9395 %}
9396
9397 // Manifest a CmpUL result in an integer register.
9398 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9399 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9400 %{
9401 match(Set dst (CmpUL3 src1 src2));
9402 effect(KILL flags);
9403
9404 ins_cost(INSN_COST * 3);
9405 format %{
9406 "cmp $src1, $src2\n\t"
9407 "csetw $dst, ne\n\t"
9408 "cnegw $dst, lo\t# CmpUL3(reg)"
9409 %}
9410 ins_encode %{
9411 __ cmp($src1$$Register, $src2$$Register);
9412 __ csetw($dst$$Register, Assembler::NE);
9413 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9414 %}
9415
9416 ins_pipe(pipe_class_default);
9417 %}
9418
9419 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9420 %{
9421 match(Set dst (CmpUL3 src1 src2));
9422 effect(KILL flags);
9423
9424 ins_cost(INSN_COST * 3);
9425 format %{
9426 "subs zr, $src1, $src2\n\t"
9427 "csetw $dst, ne\n\t"
9428 "cnegw $dst, lo\t# CmpUL3(imm)"
9429 %}
9430 ins_encode %{
9431 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9432 __ csetw($dst$$Register, Assembler::NE);
9433 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9434 %}
9435
9436 ins_pipe(pipe_class_default);
9437 %}
9438
9439 // Manifest a CmpL result in an integer register.
9440 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9441 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9442 %{
9443 match(Set dst (CmpL3 src1 src2));
9444 effect(KILL flags);
9445
9446 ins_cost(INSN_COST * 3);
9447 format %{
9448 "cmp $src1, $src2\n\t"
9449 "csetw $dst, ne\n\t"
9450 "cnegw $dst, lt\t# CmpL3(reg)"
9451 %}
9452 ins_encode %{
9453 __ cmp($src1$$Register, $src2$$Register);
9454 __ csetw($dst$$Register, Assembler::NE);
9455 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9456 %}
9457
9458 ins_pipe(pipe_class_default);
9459 %}
9460
9461 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9462 %{
9463 match(Set dst (CmpL3 src1 src2));
9464 effect(KILL flags);
9465
9466 ins_cost(INSN_COST * 3);
9467 format %{
9468 "subs zr, $src1, $src2\n\t"
9469 "csetw $dst, ne\n\t"
9470 "cnegw $dst, lt\t# CmpL3(imm)"
9471 %}
9472 ins_encode %{
9473 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9474 __ csetw($dst$$Register, Assembler::NE);
9475 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9476 %}
9477
9478 ins_pipe(pipe_class_default);
9479 %}
9480
9481 // ============================================================================
9482 // Conditional Move Instructions
9483
9484 // n.b. we have identical rules for both a signed compare op (cmpOp)
9485 // and an unsigned compare op (cmpOpU). it would be nice if we could
9486 // define an op class which merged both inputs and use it to type the
9487 // argument to a single rule. unfortunatelyt his fails because the
9488 // opclass does not live up to the COND_INTER interface of its
9489 // component operands. When the generic code tries to negate the
9490 // operand it ends up running the generci Machoper::negate method
9491 // which throws a ShouldNotHappen. So, we have to provide two flavours
9492 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9493
9494 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9495 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9496
9497 ins_cost(INSN_COST * 2);
9498 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9499
9500 ins_encode %{
9501 __ cselw(as_Register($dst$$reg),
9502 as_Register($src2$$reg),
9503 as_Register($src1$$reg),
9504 (Assembler::Condition)$cmp$$cmpcode);
9505 %}
9506
9507 ins_pipe(icond_reg_reg);
9508 %}
9509
9510 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9511 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9512
9513 ins_cost(INSN_COST * 2);
9514 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9515
9516 ins_encode %{
9517 __ cselw(as_Register($dst$$reg),
9518 as_Register($src2$$reg),
9519 as_Register($src1$$reg),
9520 (Assembler::Condition)$cmp$$cmpcode);
9521 %}
9522
9523 ins_pipe(icond_reg_reg);
9524 %}
9525
9526 // special cases where one arg is zero
9527
9528 // n.b. this is selected in preference to the rule above because it
9529 // avoids loading constant 0 into a source register
9530
9531 // TODO
9532 // we ought only to be able to cull one of these variants as the ideal
9533 // transforms ought always to order the zero consistently (to left/right?)
9534
9535 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9536 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9537
9538 ins_cost(INSN_COST * 2);
9539 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9540
9541 ins_encode %{
9542 __ cselw(as_Register($dst$$reg),
9543 as_Register($src$$reg),
9544 zr,
9545 (Assembler::Condition)$cmp$$cmpcode);
9546 %}
9547
9548 ins_pipe(icond_reg);
9549 %}
9550
9551 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9552 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9553
9554 ins_cost(INSN_COST * 2);
9555 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9556
9557 ins_encode %{
9558 __ cselw(as_Register($dst$$reg),
9559 as_Register($src$$reg),
9560 zr,
9561 (Assembler::Condition)$cmp$$cmpcode);
9562 %}
9563
9564 ins_pipe(icond_reg);
9565 %}
9566
9567 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9568 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9569
9570 ins_cost(INSN_COST * 2);
9571 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9572
9573 ins_encode %{
9574 __ cselw(as_Register($dst$$reg),
9575 zr,
9576 as_Register($src$$reg),
9577 (Assembler::Condition)$cmp$$cmpcode);
9578 %}
9579
9580 ins_pipe(icond_reg);
9581 %}
9582
9583 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9584 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9585
9586 ins_cost(INSN_COST * 2);
9587 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9588
9589 ins_encode %{
9590 __ cselw(as_Register($dst$$reg),
9591 zr,
9592 as_Register($src$$reg),
9593 (Assembler::Condition)$cmp$$cmpcode);
9594 %}
9595
9596 ins_pipe(icond_reg);
9597 %}
9598
9599 // special case for creating a boolean 0 or 1
9600
9601 // n.b. this is selected in preference to the rule above because it
9602 // avoids loading constants 0 and 1 into a source register
9603
9604 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9605 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9606
9607 ins_cost(INSN_COST * 2);
9608 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9609
9610 ins_encode %{
9611 // equivalently
9612 // cset(as_Register($dst$$reg),
9613 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9614 __ csincw(as_Register($dst$$reg),
9615 zr,
9616 zr,
9617 (Assembler::Condition)$cmp$$cmpcode);
9618 %}
9619
9620 ins_pipe(icond_none);
9621 %}
9622
9623 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9624 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9625
9626 ins_cost(INSN_COST * 2);
9627 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9628
9629 ins_encode %{
9630 // equivalently
9631 // cset(as_Register($dst$$reg),
9632 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9633 __ csincw(as_Register($dst$$reg),
9634 zr,
9635 zr,
9636 (Assembler::Condition)$cmp$$cmpcode);
9637 %}
9638
9639 ins_pipe(icond_none);
9640 %}
9641
9642 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9643 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9644
9645 ins_cost(INSN_COST * 2);
9646 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9647
9648 ins_encode %{
9649 __ csel(as_Register($dst$$reg),
9650 as_Register($src2$$reg),
9651 as_Register($src1$$reg),
9652 (Assembler::Condition)$cmp$$cmpcode);
9653 %}
9654
9655 ins_pipe(icond_reg_reg);
9656 %}
9657
9658 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9659 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9660
9661 ins_cost(INSN_COST * 2);
9662 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9663
9664 ins_encode %{
9665 __ csel(as_Register($dst$$reg),
9666 as_Register($src2$$reg),
9667 as_Register($src1$$reg),
9668 (Assembler::Condition)$cmp$$cmpcode);
9669 %}
9670
9671 ins_pipe(icond_reg_reg);
9672 %}
9673
9674 // special cases where one arg is zero
9675
9676 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9677 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9678
9679 ins_cost(INSN_COST * 2);
9680 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9681
9682 ins_encode %{
9683 __ csel(as_Register($dst$$reg),
9684 zr,
9685 as_Register($src$$reg),
9686 (Assembler::Condition)$cmp$$cmpcode);
9687 %}
9688
9689 ins_pipe(icond_reg);
9690 %}
9691
9692 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9693 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9694
9695 ins_cost(INSN_COST * 2);
9696 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9697
9698 ins_encode %{
9699 __ csel(as_Register($dst$$reg),
9700 zr,
9701 as_Register($src$$reg),
9702 (Assembler::Condition)$cmp$$cmpcode);
9703 %}
9704
9705 ins_pipe(icond_reg);
9706 %}
9707
9708 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9709 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9710
9711 ins_cost(INSN_COST * 2);
9712 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9713
9714 ins_encode %{
9715 __ csel(as_Register($dst$$reg),
9716 as_Register($src$$reg),
9717 zr,
9718 (Assembler::Condition)$cmp$$cmpcode);
9719 %}
9720
9721 ins_pipe(icond_reg);
9722 %}
9723
9724 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9725 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9726
9727 ins_cost(INSN_COST * 2);
9728 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9729
9730 ins_encode %{
9731 __ csel(as_Register($dst$$reg),
9732 as_Register($src$$reg),
9733 zr,
9734 (Assembler::Condition)$cmp$$cmpcode);
9735 %}
9736
9737 ins_pipe(icond_reg);
9738 %}
9739
9740 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9741 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9742
9743 ins_cost(INSN_COST * 2);
9744 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9745
9746 ins_encode %{
9747 __ csel(as_Register($dst$$reg),
9748 as_Register($src2$$reg),
9749 as_Register($src1$$reg),
9750 (Assembler::Condition)$cmp$$cmpcode);
9751 %}
9752
9753 ins_pipe(icond_reg_reg);
9754 %}
9755
9756 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9757 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9758
9759 ins_cost(INSN_COST * 2);
9760 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9761
9762 ins_encode %{
9763 __ csel(as_Register($dst$$reg),
9764 as_Register($src2$$reg),
9765 as_Register($src1$$reg),
9766 (Assembler::Condition)$cmp$$cmpcode);
9767 %}
9768
9769 ins_pipe(icond_reg_reg);
9770 %}
9771
9772 // special cases where one arg is zero
9773
9774 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9775 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9776
9777 ins_cost(INSN_COST * 2);
9778 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9779
9780 ins_encode %{
9781 __ csel(as_Register($dst$$reg),
9782 zr,
9783 as_Register($src$$reg),
9784 (Assembler::Condition)$cmp$$cmpcode);
9785 %}
9786
9787 ins_pipe(icond_reg);
9788 %}
9789
9790 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9791 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9792
9793 ins_cost(INSN_COST * 2);
9794 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9795
9796 ins_encode %{
9797 __ csel(as_Register($dst$$reg),
9798 zr,
9799 as_Register($src$$reg),
9800 (Assembler::Condition)$cmp$$cmpcode);
9801 %}
9802
9803 ins_pipe(icond_reg);
9804 %}
9805
9806 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9807 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9808
9809 ins_cost(INSN_COST * 2);
9810 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9811
9812 ins_encode %{
9813 __ csel(as_Register($dst$$reg),
9814 as_Register($src$$reg),
9815 zr,
9816 (Assembler::Condition)$cmp$$cmpcode);
9817 %}
9818
9819 ins_pipe(icond_reg);
9820 %}
9821
9822 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9823 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9824
9825 ins_cost(INSN_COST * 2);
9826 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9827
9828 ins_encode %{
9829 __ csel(as_Register($dst$$reg),
9830 as_Register($src$$reg),
9831 zr,
9832 (Assembler::Condition)$cmp$$cmpcode);
9833 %}
9834
9835 ins_pipe(icond_reg);
9836 %}
9837
9838 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9839 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9840
9841 ins_cost(INSN_COST * 2);
9842 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9843
9844 ins_encode %{
9845 __ cselw(as_Register($dst$$reg),
9846 as_Register($src2$$reg),
9847 as_Register($src1$$reg),
9848 (Assembler::Condition)$cmp$$cmpcode);
9849 %}
9850
9851 ins_pipe(icond_reg_reg);
9852 %}
9853
9854 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9855 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9856
9857 ins_cost(INSN_COST * 2);
9858 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9859
9860 ins_encode %{
9861 __ cselw(as_Register($dst$$reg),
9862 as_Register($src2$$reg),
9863 as_Register($src1$$reg),
9864 (Assembler::Condition)$cmp$$cmpcode);
9865 %}
9866
9867 ins_pipe(icond_reg_reg);
9868 %}
9869
9870 // special cases where one arg is zero
9871
9872 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9873 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9874
9875 ins_cost(INSN_COST * 2);
9876 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9877
9878 ins_encode %{
9879 __ cselw(as_Register($dst$$reg),
9880 zr,
9881 as_Register($src$$reg),
9882 (Assembler::Condition)$cmp$$cmpcode);
9883 %}
9884
9885 ins_pipe(icond_reg);
9886 %}
9887
9888 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9889 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9890
9891 ins_cost(INSN_COST * 2);
9892 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9893
9894 ins_encode %{
9895 __ cselw(as_Register($dst$$reg),
9896 zr,
9897 as_Register($src$$reg),
9898 (Assembler::Condition)$cmp$$cmpcode);
9899 %}
9900
9901 ins_pipe(icond_reg);
9902 %}
9903
9904 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9905 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9906
9907 ins_cost(INSN_COST * 2);
9908 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9909
9910 ins_encode %{
9911 __ cselw(as_Register($dst$$reg),
9912 as_Register($src$$reg),
9913 zr,
9914 (Assembler::Condition)$cmp$$cmpcode);
9915 %}
9916
9917 ins_pipe(icond_reg);
9918 %}
9919
9920 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9921 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9922
9923 ins_cost(INSN_COST * 2);
9924 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9925
9926 ins_encode %{
9927 __ cselw(as_Register($dst$$reg),
9928 as_Register($src$$reg),
9929 zr,
9930 (Assembler::Condition)$cmp$$cmpcode);
9931 %}
9932
9933 ins_pipe(icond_reg);
9934 %}
9935
9936 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9937 %{
9938 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9939
9940 ins_cost(INSN_COST * 3);
9941
9942 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9943 ins_encode %{
9944 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9945 __ fcsels(as_FloatRegister($dst$$reg),
9946 as_FloatRegister($src2$$reg),
9947 as_FloatRegister($src1$$reg),
9948 cond);
9949 %}
9950
9951 ins_pipe(fp_cond_reg_reg_s);
9952 %}
9953
9954 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9955 %{
9956 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9957
9958 ins_cost(INSN_COST * 3);
9959
9960 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9961 ins_encode %{
9962 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9963 __ fcsels(as_FloatRegister($dst$$reg),
9964 as_FloatRegister($src2$$reg),
9965 as_FloatRegister($src1$$reg),
9966 cond);
9967 %}
9968
9969 ins_pipe(fp_cond_reg_reg_s);
9970 %}
9971
9972 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9973 %{
9974 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9975
9976 ins_cost(INSN_COST * 3);
9977
9978 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9979 ins_encode %{
9980 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9981 __ fcseld(as_FloatRegister($dst$$reg),
9982 as_FloatRegister($src2$$reg),
9983 as_FloatRegister($src1$$reg),
9984 cond);
9985 %}
9986
9987 ins_pipe(fp_cond_reg_reg_d);
9988 %}
9989
9990 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9991 %{
9992 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9993
9994 ins_cost(INSN_COST * 3);
9995
9996 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9997 ins_encode %{
9998 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9999 __ fcseld(as_FloatRegister($dst$$reg),
10000 as_FloatRegister($src2$$reg),
10001 as_FloatRegister($src1$$reg),
10002 cond);
10003 %}
10004
10005 ins_pipe(fp_cond_reg_reg_d);
10006 %}
10007
10008 // ============================================================================
10009 // Arithmetic Instructions
10010 //
10011
10012 // Integer Addition
10013
10014 // TODO
10015 // these currently employ operations which do not set CR and hence are
10016 // not flagged as killing CR but we would like to isolate the cases
10017 // where we want to set flags from those where we don't. need to work
10018 // out how to do that.
10019
10020 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10021 match(Set dst (AddI src1 src2));
10022
10023 ins_cost(INSN_COST);
10024 format %{ "addw $dst, $src1, $src2" %}
10025
10026 ins_encode %{
10027 __ addw(as_Register($dst$$reg),
10028 as_Register($src1$$reg),
10029 as_Register($src2$$reg));
10030 %}
10031
10032 ins_pipe(ialu_reg_reg);
10033 %}
10034
10035 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10036 match(Set dst (AddI src1 src2));
10037
10038 ins_cost(INSN_COST);
10039 format %{ "addw $dst, $src1, $src2" %}
10040
10041 // use opcode to indicate that this is an add not a sub
10042 opcode(0x0);
10043
10044 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10045
10046 ins_pipe(ialu_reg_imm);
10047 %}
10048
10049 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10050 match(Set dst (AddI (ConvL2I src1) src2));
10051
10052 ins_cost(INSN_COST);
10053 format %{ "addw $dst, $src1, $src2" %}
10054
10055 // use opcode to indicate that this is an add not a sub
10056 opcode(0x0);
10057
10058 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10059
10060 ins_pipe(ialu_reg_imm);
10061 %}
10062
10063 // Pointer Addition
10064 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10065 match(Set dst (AddP src1 src2));
10066
10067 ins_cost(INSN_COST);
10068 format %{ "add $dst, $src1, $src2\t# ptr" %}
10069
10070 ins_encode %{
10071 __ add(as_Register($dst$$reg),
10072 as_Register($src1$$reg),
10073 as_Register($src2$$reg));
10074 %}
10075
10076 ins_pipe(ialu_reg_reg);
10077 %}
10078
10079 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10080 match(Set dst (AddP src1 (ConvI2L src2)));
10081
10082 ins_cost(1.9 * INSN_COST);
10083 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10084
10085 ins_encode %{
10086 __ add(as_Register($dst$$reg),
10087 as_Register($src1$$reg),
10088 as_Register($src2$$reg), ext::sxtw);
10089 %}
10090
10091 ins_pipe(ialu_reg_reg);
10092 %}
10093
10094 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10095 match(Set dst (AddP src1 (LShiftL src2 scale)));
10096
10097 ins_cost(1.9 * INSN_COST);
10098 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10099
10100 ins_encode %{
10101 __ lea(as_Register($dst$$reg),
10102 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10103 Address::lsl($scale$$constant)));
10104 %}
10105
10106 ins_pipe(ialu_reg_reg_shift);
10107 %}
10108
10109 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10110 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10111
10112 ins_cost(1.9 * INSN_COST);
10113 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10114
10115 ins_encode %{
10116 __ lea(as_Register($dst$$reg),
10117 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10118 Address::sxtw($scale$$constant)));
10119 %}
10120
10121 ins_pipe(ialu_reg_reg_shift);
10122 %}
10123
10124 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10125 match(Set dst (LShiftL (ConvI2L src) scale));
10126
10127 ins_cost(INSN_COST);
10128 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10129
10130 ins_encode %{
10131 __ sbfiz(as_Register($dst$$reg),
10132 as_Register($src$$reg),
10133 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10134 %}
10135
10136 ins_pipe(ialu_reg_shift);
10137 %}
10138
10139 // Pointer Immediate Addition
10140 // n.b. this needs to be more expensive than using an indirect memory
10141 // operand
10142 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10143 match(Set dst (AddP src1 src2));
10144
10145 ins_cost(INSN_COST);
10146 format %{ "add $dst, $src1, $src2\t# ptr" %}
10147
10148 // use opcode to indicate that this is an add not a sub
10149 opcode(0x0);
10150
10151 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10152
10153 ins_pipe(ialu_reg_imm);
10154 %}
10155
10156 // Long Addition
10157 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10158
10159 match(Set dst (AddL src1 src2));
10160
10161 ins_cost(INSN_COST);
10162 format %{ "add $dst, $src1, $src2" %}
10163
10164 ins_encode %{
10165 __ add(as_Register($dst$$reg),
10166 as_Register($src1$$reg),
10167 as_Register($src2$$reg));
10168 %}
10169
10170 ins_pipe(ialu_reg_reg);
10171 %}
10172
10173 // No constant pool entries requiredLong Immediate Addition.
10174 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10175 match(Set dst (AddL src1 src2));
10176
10177 ins_cost(INSN_COST);
10178 format %{ "add $dst, $src1, $src2" %}
10179
10180 // use opcode to indicate that this is an add not a sub
10181 opcode(0x0);
10182
10183 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10184
10185 ins_pipe(ialu_reg_imm);
10186 %}
10187
10188 // Integer Subtraction
10189 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10190 match(Set dst (SubI src1 src2));
10191
10192 ins_cost(INSN_COST);
10193 format %{ "subw $dst, $src1, $src2" %}
10194
10195 ins_encode %{
10196 __ subw(as_Register($dst$$reg),
10197 as_Register($src1$$reg),
10198 as_Register($src2$$reg));
10199 %}
10200
10201 ins_pipe(ialu_reg_reg);
10202 %}
10203
10204 // Immediate Subtraction
10205 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10206 match(Set dst (SubI src1 src2));
10207
10208 ins_cost(INSN_COST);
10209 format %{ "subw $dst, $src1, $src2" %}
10210
10211 // use opcode to indicate that this is a sub not an add
10212 opcode(0x1);
10213
10214 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10215
10216 ins_pipe(ialu_reg_imm);
10217 %}
10218
10219 // Long Subtraction
10220 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10221
10222 match(Set dst (SubL src1 src2));
10223
10224 ins_cost(INSN_COST);
10225 format %{ "sub $dst, $src1, $src2" %}
10226
10227 ins_encode %{
10228 __ sub(as_Register($dst$$reg),
10229 as_Register($src1$$reg),
10230 as_Register($src2$$reg));
10231 %}
10232
10233 ins_pipe(ialu_reg_reg);
10234 %}
10235
10236 // No constant pool entries requiredLong Immediate Subtraction.
10237 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10238 match(Set dst (SubL src1 src2));
10239
10240 ins_cost(INSN_COST);
10241 format %{ "sub$dst, $src1, $src2" %}
10242
10243 // use opcode to indicate that this is a sub not an add
10244 opcode(0x1);
10245
10246 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10247
10248 ins_pipe(ialu_reg_imm);
10249 %}
10250
10251 // Integer Negation (special case for sub)
10252
10253 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10254 match(Set dst (SubI zero src));
10255
10256 ins_cost(INSN_COST);
10257 format %{ "negw $dst, $src\t# int" %}
10258
10259 ins_encode %{
10260 __ negw(as_Register($dst$$reg),
10261 as_Register($src$$reg));
10262 %}
10263
10264 ins_pipe(ialu_reg);
10265 %}
10266
10267 // Long Negation
10268
10269 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10270 match(Set dst (SubL zero src));
10271
10272 ins_cost(INSN_COST);
10273 format %{ "neg $dst, $src\t# long" %}
10274
10275 ins_encode %{
10276 __ neg(as_Register($dst$$reg),
10277 as_Register($src$$reg));
10278 %}
10279
10280 ins_pipe(ialu_reg);
10281 %}
10282
10283 // Integer Multiply
10284
10285 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10286 match(Set dst (MulI src1 src2));
10287
10288 ins_cost(INSN_COST * 3);
10289 format %{ "mulw $dst, $src1, $src2" %}
10290
10291 ins_encode %{
10292 __ mulw(as_Register($dst$$reg),
10293 as_Register($src1$$reg),
10294 as_Register($src2$$reg));
10295 %}
10296
10297 ins_pipe(imul_reg_reg);
10298 %}
10299
10300 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10301 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10302
10303 ins_cost(INSN_COST * 3);
10304 format %{ "smull $dst, $src1, $src2" %}
10305
10306 ins_encode %{
10307 __ smull(as_Register($dst$$reg),
10308 as_Register($src1$$reg),
10309 as_Register($src2$$reg));
10310 %}
10311
10312 ins_pipe(imul_reg_reg);
10313 %}
10314
10315 // Long Multiply
10316
10317 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10318 match(Set dst (MulL src1 src2));
10319
10320 ins_cost(INSN_COST * 5);
10321 format %{ "mul $dst, $src1, $src2" %}
10322
10323 ins_encode %{
10324 __ mul(as_Register($dst$$reg),
10325 as_Register($src1$$reg),
10326 as_Register($src2$$reg));
10327 %}
10328
10329 ins_pipe(lmul_reg_reg);
10330 %}
10331
10332 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10333 %{
10334 match(Set dst (MulHiL src1 src2));
10335
10336 ins_cost(INSN_COST * 7);
10337 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10338
10339 ins_encode %{
10340 __ smulh(as_Register($dst$$reg),
10341 as_Register($src1$$reg),
10342 as_Register($src2$$reg));
10343 %}
10344
10345 ins_pipe(lmul_reg_reg);
10346 %}
10347
10348 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10349 %{
10350 match(Set dst (UMulHiL src1 src2));
10351
10352 ins_cost(INSN_COST * 7);
10353 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10354
10355 ins_encode %{
10356 __ umulh(as_Register($dst$$reg),
10357 as_Register($src1$$reg),
10358 as_Register($src2$$reg));
10359 %}
10360
10361 ins_pipe(lmul_reg_reg);
10362 %}
10363
10364 // Combined Integer Multiply & Add/Sub
10365
10366 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10367 match(Set dst (AddI src3 (MulI src1 src2)));
10368
10369 ins_cost(INSN_COST * 3);
10370 format %{ "madd $dst, $src1, $src2, $src3" %}
10371
10372 ins_encode %{
10373 __ maddw(as_Register($dst$$reg),
10374 as_Register($src1$$reg),
10375 as_Register($src2$$reg),
10376 as_Register($src3$$reg));
10377 %}
10378
10379 ins_pipe(imac_reg_reg);
10380 %}
10381
10382 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10383 match(Set dst (SubI src3 (MulI src1 src2)));
10384
10385 ins_cost(INSN_COST * 3);
10386 format %{ "msub $dst, $src1, $src2, $src3" %}
10387
10388 ins_encode %{
10389 __ msubw(as_Register($dst$$reg),
10390 as_Register($src1$$reg),
10391 as_Register($src2$$reg),
10392 as_Register($src3$$reg));
10393 %}
10394
10395 ins_pipe(imac_reg_reg);
10396 %}
10397
10398 // Combined Integer Multiply & Neg
10399
10400 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10401 match(Set dst (MulI (SubI zero src1) src2));
10402
10403 ins_cost(INSN_COST * 3);
10404 format %{ "mneg $dst, $src1, $src2" %}
10405
10406 ins_encode %{
10407 __ mnegw(as_Register($dst$$reg),
10408 as_Register($src1$$reg),
10409 as_Register($src2$$reg));
10410 %}
10411
10412 ins_pipe(imac_reg_reg);
10413 %}
10414
10415 // Combined Long Multiply & Add/Sub
10416
10417 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10418 match(Set dst (AddL src3 (MulL src1 src2)));
10419
10420 ins_cost(INSN_COST * 5);
10421 format %{ "madd $dst, $src1, $src2, $src3" %}
10422
10423 ins_encode %{
10424 __ madd(as_Register($dst$$reg),
10425 as_Register($src1$$reg),
10426 as_Register($src2$$reg),
10427 as_Register($src3$$reg));
10428 %}
10429
10430 ins_pipe(lmac_reg_reg);
10431 %}
10432
10433 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10434 match(Set dst (SubL src3 (MulL src1 src2)));
10435
10436 ins_cost(INSN_COST * 5);
10437 format %{ "msub $dst, $src1, $src2, $src3" %}
10438
10439 ins_encode %{
10440 __ msub(as_Register($dst$$reg),
10441 as_Register($src1$$reg),
10442 as_Register($src2$$reg),
10443 as_Register($src3$$reg));
10444 %}
10445
10446 ins_pipe(lmac_reg_reg);
10447 %}
10448
10449 // Combined Long Multiply & Neg
10450
10451 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10452 match(Set dst (MulL (SubL zero src1) src2));
10453
10454 ins_cost(INSN_COST * 5);
10455 format %{ "mneg $dst, $src1, $src2" %}
10456
10457 ins_encode %{
10458 __ mneg(as_Register($dst$$reg),
10459 as_Register($src1$$reg),
10460 as_Register($src2$$reg));
10461 %}
10462
10463 ins_pipe(lmac_reg_reg);
10464 %}
10465
10466 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10467
10468 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10469 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10470
10471 ins_cost(INSN_COST * 3);
10472 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10473
10474 ins_encode %{
10475 __ smaddl(as_Register($dst$$reg),
10476 as_Register($src1$$reg),
10477 as_Register($src2$$reg),
10478 as_Register($src3$$reg));
10479 %}
10480
10481 ins_pipe(imac_reg_reg);
10482 %}
10483
10484 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10485 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10486
10487 ins_cost(INSN_COST * 3);
10488 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10489
10490 ins_encode %{
10491 __ smsubl(as_Register($dst$$reg),
10492 as_Register($src1$$reg),
10493 as_Register($src2$$reg),
10494 as_Register($src3$$reg));
10495 %}
10496
10497 ins_pipe(imac_reg_reg);
10498 %}
10499
10500 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10501 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10502
10503 ins_cost(INSN_COST * 3);
10504 format %{ "smnegl $dst, $src1, $src2" %}
10505
10506 ins_encode %{
10507 __ smnegl(as_Register($dst$$reg),
10508 as_Register($src1$$reg),
10509 as_Register($src2$$reg));
10510 %}
10511
10512 ins_pipe(imac_reg_reg);
10513 %}
10514
10515 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10516
10517 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10518 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10519
10520 ins_cost(INSN_COST * 5);
10521 format %{ "mulw rscratch1, $src1, $src2\n\t"
10522 "maddw $dst, $src3, $src4, rscratch1" %}
10523
10524 ins_encode %{
10525 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10526 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10527
10528 ins_pipe(imac_reg_reg);
10529 %}
10530
10531 // Integer Divide
10532
10533 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10534 match(Set dst (DivI src1 src2));
10535
10536 ins_cost(INSN_COST * 19);
10537 format %{ "sdivw $dst, $src1, $src2" %}
10538
10539 ins_encode(aarch64_enc_divw(dst, src1, src2));
10540 ins_pipe(idiv_reg_reg);
10541 %}
10542
10543 // Long Divide
10544
10545 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10546 match(Set dst (DivL src1 src2));
10547
10548 ins_cost(INSN_COST * 35);
10549 format %{ "sdiv $dst, $src1, $src2" %}
10550
10551 ins_encode(aarch64_enc_div(dst, src1, src2));
10552 ins_pipe(ldiv_reg_reg);
10553 %}
10554
10555 // Integer Remainder
10556
10557 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10558 match(Set dst (ModI src1 src2));
10559
10560 ins_cost(INSN_COST * 22);
10561 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10562 "msubw $dst, rscratch1, $src2, $src1" %}
10563
10564 ins_encode(aarch64_enc_modw(dst, src1, src2));
10565 ins_pipe(idiv_reg_reg);
10566 %}
10567
10568 // Long Remainder
10569
10570 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10571 match(Set dst (ModL src1 src2));
10572
10573 ins_cost(INSN_COST * 38);
10574 format %{ "sdiv rscratch1, $src1, $src2\n"
10575 "msub $dst, rscratch1, $src2, $src1" %}
10576
10577 ins_encode(aarch64_enc_mod(dst, src1, src2));
10578 ins_pipe(ldiv_reg_reg);
10579 %}
10580
10581 // Unsigned Integer Divide
10582
10583 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10584 match(Set dst (UDivI src1 src2));
10585
10586 ins_cost(INSN_COST * 19);
10587 format %{ "udivw $dst, $src1, $src2" %}
10588
10589 ins_encode %{
10590 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10591 %}
10592
10593 ins_pipe(idiv_reg_reg);
10594 %}
10595
10596 // Unsigned Long Divide
10597
10598 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10599 match(Set dst (UDivL src1 src2));
10600
10601 ins_cost(INSN_COST * 35);
10602 format %{ "udiv $dst, $src1, $src2" %}
10603
10604 ins_encode %{
10605 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10606 %}
10607
10608 ins_pipe(ldiv_reg_reg);
10609 %}
10610
10611 // Unsigned Integer Remainder
10612
10613 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10614 match(Set dst (UModI src1 src2));
10615
10616 ins_cost(INSN_COST * 22);
10617 format %{ "udivw rscratch1, $src1, $src2\n\t"
10618 "msubw $dst, rscratch1, $src2, $src1" %}
10619
10620 ins_encode %{
10621 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10622 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10623 %}
10624
10625 ins_pipe(idiv_reg_reg);
10626 %}
10627
10628 // Unsigned Long Remainder
10629
10630 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10631 match(Set dst (UModL src1 src2));
10632
10633 ins_cost(INSN_COST * 38);
10634 format %{ "udiv rscratch1, $src1, $src2\n"
10635 "msub $dst, rscratch1, $src2, $src1" %}
10636
10637 ins_encode %{
10638 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10639 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10640 %}
10641
10642 ins_pipe(ldiv_reg_reg);
10643 %}
10644
10645 // Integer Shifts
10646
10647 // Shift Left Register
10648 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10649 match(Set dst (LShiftI src1 src2));
10650
10651 ins_cost(INSN_COST * 2);
10652 format %{ "lslvw $dst, $src1, $src2" %}
10653
10654 ins_encode %{
10655 __ lslvw(as_Register($dst$$reg),
10656 as_Register($src1$$reg),
10657 as_Register($src2$$reg));
10658 %}
10659
10660 ins_pipe(ialu_reg_reg_vshift);
10661 %}
10662
10663 // Shift Left Immediate
10664 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10665 match(Set dst (LShiftI src1 src2));
10666
10667 ins_cost(INSN_COST);
10668 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10669
10670 ins_encode %{
10671 __ lslw(as_Register($dst$$reg),
10672 as_Register($src1$$reg),
10673 $src2$$constant & 0x1f);
10674 %}
10675
10676 ins_pipe(ialu_reg_shift);
10677 %}
10678
10679 // Shift Right Logical Register
10680 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10681 match(Set dst (URShiftI src1 src2));
10682
10683 ins_cost(INSN_COST * 2);
10684 format %{ "lsrvw $dst, $src1, $src2" %}
10685
10686 ins_encode %{
10687 __ lsrvw(as_Register($dst$$reg),
10688 as_Register($src1$$reg),
10689 as_Register($src2$$reg));
10690 %}
10691
10692 ins_pipe(ialu_reg_reg_vshift);
10693 %}
10694
10695 // Shift Right Logical Immediate
10696 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10697 match(Set dst (URShiftI src1 src2));
10698
10699 ins_cost(INSN_COST);
10700 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10701
10702 ins_encode %{
10703 __ lsrw(as_Register($dst$$reg),
10704 as_Register($src1$$reg),
10705 $src2$$constant & 0x1f);
10706 %}
10707
10708 ins_pipe(ialu_reg_shift);
10709 %}
10710
10711 // Shift Right Arithmetic Register
10712 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10713 match(Set dst (RShiftI src1 src2));
10714
10715 ins_cost(INSN_COST * 2);
10716 format %{ "asrvw $dst, $src1, $src2" %}
10717
10718 ins_encode %{
10719 __ asrvw(as_Register($dst$$reg),
10720 as_Register($src1$$reg),
10721 as_Register($src2$$reg));
10722 %}
10723
10724 ins_pipe(ialu_reg_reg_vshift);
10725 %}
10726
10727 // Shift Right Arithmetic Immediate
10728 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10729 match(Set dst (RShiftI src1 src2));
10730
10731 ins_cost(INSN_COST);
10732 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10733
10734 ins_encode %{
10735 __ asrw(as_Register($dst$$reg),
10736 as_Register($src1$$reg),
10737 $src2$$constant & 0x1f);
10738 %}
10739
10740 ins_pipe(ialu_reg_shift);
10741 %}
10742
10743 // Combined Int Mask and Right Shift (using UBFM)
10744 // TODO
10745
10746 // Long Shifts
10747
10748 // Shift Left Register
10749 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10750 match(Set dst (LShiftL src1 src2));
10751
10752 ins_cost(INSN_COST * 2);
10753 format %{ "lslv $dst, $src1, $src2" %}
10754
10755 ins_encode %{
10756 __ lslv(as_Register($dst$$reg),
10757 as_Register($src1$$reg),
10758 as_Register($src2$$reg));
10759 %}
10760
10761 ins_pipe(ialu_reg_reg_vshift);
10762 %}
10763
10764 // Shift Left Immediate
10765 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10766 match(Set dst (LShiftL src1 src2));
10767
10768 ins_cost(INSN_COST);
10769 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10770
10771 ins_encode %{
10772 __ lsl(as_Register($dst$$reg),
10773 as_Register($src1$$reg),
10774 $src2$$constant & 0x3f);
10775 %}
10776
10777 ins_pipe(ialu_reg_shift);
10778 %}
10779
10780 // Shift Right Logical Register
10781 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10782 match(Set dst (URShiftL src1 src2));
10783
10784 ins_cost(INSN_COST * 2);
10785 format %{ "lsrv $dst, $src1, $src2" %}
10786
10787 ins_encode %{
10788 __ lsrv(as_Register($dst$$reg),
10789 as_Register($src1$$reg),
10790 as_Register($src2$$reg));
10791 %}
10792
10793 ins_pipe(ialu_reg_reg_vshift);
10794 %}
10795
10796 // Shift Right Logical Immediate
10797 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10798 match(Set dst (URShiftL src1 src2));
10799
10800 ins_cost(INSN_COST);
10801 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10802
10803 ins_encode %{
10804 __ lsr(as_Register($dst$$reg),
10805 as_Register($src1$$reg),
10806 $src2$$constant & 0x3f);
10807 %}
10808
10809 ins_pipe(ialu_reg_shift);
10810 %}
10811
10812 // A special-case pattern for card table stores.
10813 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10814 match(Set dst (URShiftL (CastP2X src1) src2));
10815
10816 ins_cost(INSN_COST);
10817 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10818
10819 ins_encode %{
10820 __ lsr(as_Register($dst$$reg),
10821 as_Register($src1$$reg),
10822 $src2$$constant & 0x3f);
10823 %}
10824
10825 ins_pipe(ialu_reg_shift);
10826 %}
10827
10828 // Shift Right Arithmetic Register
10829 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10830 match(Set dst (RShiftL src1 src2));
10831
10832 ins_cost(INSN_COST * 2);
10833 format %{ "asrv $dst, $src1, $src2" %}
10834
10835 ins_encode %{
10836 __ asrv(as_Register($dst$$reg),
10837 as_Register($src1$$reg),
10838 as_Register($src2$$reg));
10839 %}
10840
10841 ins_pipe(ialu_reg_reg_vshift);
10842 %}
10843
10844 // Shift Right Arithmetic Immediate
10845 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10846 match(Set dst (RShiftL src1 src2));
10847
10848 ins_cost(INSN_COST);
10849 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10850
10851 ins_encode %{
10852 __ asr(as_Register($dst$$reg),
10853 as_Register($src1$$reg),
10854 $src2$$constant & 0x3f);
10855 %}
10856
10857 ins_pipe(ialu_reg_shift);
10858 %}
10859
10860 // BEGIN This section of the file is automatically generated. Do not edit --------------
10861 // This section is generated from aarch64_ad.m4
10862
10863 // This pattern is automatically generated from aarch64_ad.m4
10864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10865 instruct regL_not_reg(iRegLNoSp dst,
10866 iRegL src1, immL_M1 m1,
10867 rFlagsReg cr) %{
10868 match(Set dst (XorL src1 m1));
10869 ins_cost(INSN_COST);
10870 format %{ "eon $dst, $src1, zr" %}
10871
10872 ins_encode %{
10873 __ eon(as_Register($dst$$reg),
10874 as_Register($src1$$reg),
10875 zr,
10876 Assembler::LSL, 0);
10877 %}
10878
10879 ins_pipe(ialu_reg);
10880 %}
10881
10882 // This pattern is automatically generated from aarch64_ad.m4
10883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10884 instruct regI_not_reg(iRegINoSp dst,
10885 iRegIorL2I src1, immI_M1 m1,
10886 rFlagsReg cr) %{
10887 match(Set dst (XorI src1 m1));
10888 ins_cost(INSN_COST);
10889 format %{ "eonw $dst, $src1, zr" %}
10890
10891 ins_encode %{
10892 __ eonw(as_Register($dst$$reg),
10893 as_Register($src1$$reg),
10894 zr,
10895 Assembler::LSL, 0);
10896 %}
10897
10898 ins_pipe(ialu_reg);
10899 %}
10900
10901 // This pattern is automatically generated from aarch64_ad.m4
10902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10903 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10904 immI0 zero, iRegIorL2I src1, immI src2) %{
10905 match(Set dst (SubI zero (URShiftI src1 src2)));
10906
10907 ins_cost(1.9 * INSN_COST);
10908 format %{ "negw $dst, $src1, LSR $src2" %}
10909
10910 ins_encode %{
10911 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10912 Assembler::LSR, $src2$$constant & 0x1f);
10913 %}
10914
10915 ins_pipe(ialu_reg_shift);
10916 %}
10917
10918 // This pattern is automatically generated from aarch64_ad.m4
10919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10920 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10921 immI0 zero, iRegIorL2I src1, immI src2) %{
10922 match(Set dst (SubI zero (RShiftI src1 src2)));
10923
10924 ins_cost(1.9 * INSN_COST);
10925 format %{ "negw $dst, $src1, ASR $src2" %}
10926
10927 ins_encode %{
10928 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10929 Assembler::ASR, $src2$$constant & 0x1f);
10930 %}
10931
10932 ins_pipe(ialu_reg_shift);
10933 %}
10934
10935 // This pattern is automatically generated from aarch64_ad.m4
10936 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10937 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10938 immI0 zero, iRegIorL2I src1, immI src2) %{
10939 match(Set dst (SubI zero (LShiftI src1 src2)));
10940
10941 ins_cost(1.9 * INSN_COST);
10942 format %{ "negw $dst, $src1, LSL $src2" %}
10943
10944 ins_encode %{
10945 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10946 Assembler::LSL, $src2$$constant & 0x1f);
10947 %}
10948
10949 ins_pipe(ialu_reg_shift);
10950 %}
10951
10952 // This pattern is automatically generated from aarch64_ad.m4
10953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10954 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10955 immL0 zero, iRegL src1, immI src2) %{
10956 match(Set dst (SubL zero (URShiftL src1 src2)));
10957
10958 ins_cost(1.9 * INSN_COST);
10959 format %{ "neg $dst, $src1, LSR $src2" %}
10960
10961 ins_encode %{
10962 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10963 Assembler::LSR, $src2$$constant & 0x3f);
10964 %}
10965
10966 ins_pipe(ialu_reg_shift);
10967 %}
10968
10969 // This pattern is automatically generated from aarch64_ad.m4
10970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10971 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10972 immL0 zero, iRegL src1, immI src2) %{
10973 match(Set dst (SubL zero (RShiftL src1 src2)));
10974
10975 ins_cost(1.9 * INSN_COST);
10976 format %{ "neg $dst, $src1, ASR $src2" %}
10977
10978 ins_encode %{
10979 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10980 Assembler::ASR, $src2$$constant & 0x3f);
10981 %}
10982
10983 ins_pipe(ialu_reg_shift);
10984 %}
10985
10986 // This pattern is automatically generated from aarch64_ad.m4
10987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10988 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10989 immL0 zero, iRegL src1, immI src2) %{
10990 match(Set dst (SubL zero (LShiftL src1 src2)));
10991
10992 ins_cost(1.9 * INSN_COST);
10993 format %{ "neg $dst, $src1, LSL $src2" %}
10994
10995 ins_encode %{
10996 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10997 Assembler::LSL, $src2$$constant & 0x3f);
10998 %}
10999
11000 ins_pipe(ialu_reg_shift);
11001 %}
11002
11003 // This pattern is automatically generated from aarch64_ad.m4
11004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11005 instruct AndI_reg_not_reg(iRegINoSp dst,
11006 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11007 match(Set dst (AndI src1 (XorI src2 m1)));
11008 ins_cost(INSN_COST);
11009 format %{ "bicw $dst, $src1, $src2" %}
11010
11011 ins_encode %{
11012 __ bicw(as_Register($dst$$reg),
11013 as_Register($src1$$reg),
11014 as_Register($src2$$reg),
11015 Assembler::LSL, 0);
11016 %}
11017
11018 ins_pipe(ialu_reg_reg);
11019 %}
11020
11021 // This pattern is automatically generated from aarch64_ad.m4
11022 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11023 instruct AndL_reg_not_reg(iRegLNoSp dst,
11024 iRegL src1, iRegL src2, immL_M1 m1) %{
11025 match(Set dst (AndL src1 (XorL src2 m1)));
11026 ins_cost(INSN_COST);
11027 format %{ "bic $dst, $src1, $src2" %}
11028
11029 ins_encode %{
11030 __ bic(as_Register($dst$$reg),
11031 as_Register($src1$$reg),
11032 as_Register($src2$$reg),
11033 Assembler::LSL, 0);
11034 %}
11035
11036 ins_pipe(ialu_reg_reg);
11037 %}
11038
11039 // This pattern is automatically generated from aarch64_ad.m4
11040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11041 instruct OrI_reg_not_reg(iRegINoSp dst,
11042 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11043 match(Set dst (OrI src1 (XorI src2 m1)));
11044 ins_cost(INSN_COST);
11045 format %{ "ornw $dst, $src1, $src2" %}
11046
11047 ins_encode %{
11048 __ ornw(as_Register($dst$$reg),
11049 as_Register($src1$$reg),
11050 as_Register($src2$$reg),
11051 Assembler::LSL, 0);
11052 %}
11053
11054 ins_pipe(ialu_reg_reg);
11055 %}
11056
11057 // This pattern is automatically generated from aarch64_ad.m4
11058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11059 instruct OrL_reg_not_reg(iRegLNoSp dst,
11060 iRegL src1, iRegL src2, immL_M1 m1) %{
11061 match(Set dst (OrL src1 (XorL src2 m1)));
11062 ins_cost(INSN_COST);
11063 format %{ "orn $dst, $src1, $src2" %}
11064
11065 ins_encode %{
11066 __ orn(as_Register($dst$$reg),
11067 as_Register($src1$$reg),
11068 as_Register($src2$$reg),
11069 Assembler::LSL, 0);
11070 %}
11071
11072 ins_pipe(ialu_reg_reg);
11073 %}
11074
11075 // This pattern is automatically generated from aarch64_ad.m4
11076 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11077 instruct XorI_reg_not_reg(iRegINoSp dst,
11078 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11079 match(Set dst (XorI m1 (XorI src2 src1)));
11080 ins_cost(INSN_COST);
11081 format %{ "eonw $dst, $src1, $src2" %}
11082
11083 ins_encode %{
11084 __ eonw(as_Register($dst$$reg),
11085 as_Register($src1$$reg),
11086 as_Register($src2$$reg),
11087 Assembler::LSL, 0);
11088 %}
11089
11090 ins_pipe(ialu_reg_reg);
11091 %}
11092
11093 // This pattern is automatically generated from aarch64_ad.m4
11094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11095 instruct XorL_reg_not_reg(iRegLNoSp dst,
11096 iRegL src1, iRegL src2, immL_M1 m1) %{
11097 match(Set dst (XorL m1 (XorL src2 src1)));
11098 ins_cost(INSN_COST);
11099 format %{ "eon $dst, $src1, $src2" %}
11100
11101 ins_encode %{
11102 __ eon(as_Register($dst$$reg),
11103 as_Register($src1$$reg),
11104 as_Register($src2$$reg),
11105 Assembler::LSL, 0);
11106 %}
11107
11108 ins_pipe(ialu_reg_reg);
11109 %}
11110
11111 // This pattern is automatically generated from aarch64_ad.m4
11112 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11113 // val & (-1 ^ (val >>> shift)) ==> bicw
11114 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11115 iRegIorL2I src1, iRegIorL2I src2,
11116 immI src3, immI_M1 src4) %{
11117 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11118 ins_cost(1.9 * INSN_COST);
11119 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11120
11121 ins_encode %{
11122 __ bicw(as_Register($dst$$reg),
11123 as_Register($src1$$reg),
11124 as_Register($src2$$reg),
11125 Assembler::LSR,
11126 $src3$$constant & 0x1f);
11127 %}
11128
11129 ins_pipe(ialu_reg_reg_shift);
11130 %}
11131
11132 // This pattern is automatically generated from aarch64_ad.m4
11133 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11134 // val & (-1 ^ (val >>> shift)) ==> bic
11135 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11136 iRegL src1, iRegL src2,
11137 immI src3, immL_M1 src4) %{
11138 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11139 ins_cost(1.9 * INSN_COST);
11140 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11141
11142 ins_encode %{
11143 __ bic(as_Register($dst$$reg),
11144 as_Register($src1$$reg),
11145 as_Register($src2$$reg),
11146 Assembler::LSR,
11147 $src3$$constant & 0x3f);
11148 %}
11149
11150 ins_pipe(ialu_reg_reg_shift);
11151 %}
11152
11153 // This pattern is automatically generated from aarch64_ad.m4
11154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11155 // val & (-1 ^ (val >> shift)) ==> bicw
11156 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11157 iRegIorL2I src1, iRegIorL2I src2,
11158 immI src3, immI_M1 src4) %{
11159 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11160 ins_cost(1.9 * INSN_COST);
11161 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11162
11163 ins_encode %{
11164 __ bicw(as_Register($dst$$reg),
11165 as_Register($src1$$reg),
11166 as_Register($src2$$reg),
11167 Assembler::ASR,
11168 $src3$$constant & 0x1f);
11169 %}
11170
11171 ins_pipe(ialu_reg_reg_shift);
11172 %}
11173
11174 // This pattern is automatically generated from aarch64_ad.m4
11175 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11176 // val & (-1 ^ (val >> shift)) ==> bic
11177 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11178 iRegL src1, iRegL src2,
11179 immI src3, immL_M1 src4) %{
11180 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11181 ins_cost(1.9 * INSN_COST);
11182 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11183
11184 ins_encode %{
11185 __ bic(as_Register($dst$$reg),
11186 as_Register($src1$$reg),
11187 as_Register($src2$$reg),
11188 Assembler::ASR,
11189 $src3$$constant & 0x3f);
11190 %}
11191
11192 ins_pipe(ialu_reg_reg_shift);
11193 %}
11194
11195 // This pattern is automatically generated from aarch64_ad.m4
11196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11197 // val & (-1 ^ (val ror shift)) ==> bicw
11198 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11199 iRegIorL2I src1, iRegIorL2I src2,
11200 immI src3, immI_M1 src4) %{
11201 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11202 ins_cost(1.9 * INSN_COST);
11203 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11204
11205 ins_encode %{
11206 __ bicw(as_Register($dst$$reg),
11207 as_Register($src1$$reg),
11208 as_Register($src2$$reg),
11209 Assembler::ROR,
11210 $src3$$constant & 0x1f);
11211 %}
11212
11213 ins_pipe(ialu_reg_reg_shift);
11214 %}
11215
11216 // This pattern is automatically generated from aarch64_ad.m4
11217 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11218 // val & (-1 ^ (val ror shift)) ==> bic
11219 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11220 iRegL src1, iRegL src2,
11221 immI src3, immL_M1 src4) %{
11222 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11223 ins_cost(1.9 * INSN_COST);
11224 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11225
11226 ins_encode %{
11227 __ bic(as_Register($dst$$reg),
11228 as_Register($src1$$reg),
11229 as_Register($src2$$reg),
11230 Assembler::ROR,
11231 $src3$$constant & 0x3f);
11232 %}
11233
11234 ins_pipe(ialu_reg_reg_shift);
11235 %}
11236
11237 // This pattern is automatically generated from aarch64_ad.m4
11238 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11239 // val & (-1 ^ (val << shift)) ==> bicw
11240 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11241 iRegIorL2I src1, iRegIorL2I src2,
11242 immI src3, immI_M1 src4) %{
11243 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11244 ins_cost(1.9 * INSN_COST);
11245 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11246
11247 ins_encode %{
11248 __ bicw(as_Register($dst$$reg),
11249 as_Register($src1$$reg),
11250 as_Register($src2$$reg),
11251 Assembler::LSL,
11252 $src3$$constant & 0x1f);
11253 %}
11254
11255 ins_pipe(ialu_reg_reg_shift);
11256 %}
11257
11258 // This pattern is automatically generated from aarch64_ad.m4
11259 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11260 // val & (-1 ^ (val << shift)) ==> bic
11261 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11262 iRegL src1, iRegL src2,
11263 immI src3, immL_M1 src4) %{
11264 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11265 ins_cost(1.9 * INSN_COST);
11266 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11267
11268 ins_encode %{
11269 __ bic(as_Register($dst$$reg),
11270 as_Register($src1$$reg),
11271 as_Register($src2$$reg),
11272 Assembler::LSL,
11273 $src3$$constant & 0x3f);
11274 %}
11275
11276 ins_pipe(ialu_reg_reg_shift);
11277 %}
11278
11279 // This pattern is automatically generated from aarch64_ad.m4
11280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11281 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11282 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11283 iRegIorL2I src1, iRegIorL2I src2,
11284 immI src3, immI_M1 src4) %{
11285 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11286 ins_cost(1.9 * INSN_COST);
11287 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11288
11289 ins_encode %{
11290 __ eonw(as_Register($dst$$reg),
11291 as_Register($src1$$reg),
11292 as_Register($src2$$reg),
11293 Assembler::LSR,
11294 $src3$$constant & 0x1f);
11295 %}
11296
11297 ins_pipe(ialu_reg_reg_shift);
11298 %}
11299
11300 // This pattern is automatically generated from aarch64_ad.m4
11301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11302 // val ^ (-1 ^ (val >>> shift)) ==> eon
11303 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11304 iRegL src1, iRegL src2,
11305 immI src3, immL_M1 src4) %{
11306 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11307 ins_cost(1.9 * INSN_COST);
11308 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11309
11310 ins_encode %{
11311 __ eon(as_Register($dst$$reg),
11312 as_Register($src1$$reg),
11313 as_Register($src2$$reg),
11314 Assembler::LSR,
11315 $src3$$constant & 0x3f);
11316 %}
11317
11318 ins_pipe(ialu_reg_reg_shift);
11319 %}
11320
11321 // This pattern is automatically generated from aarch64_ad.m4
11322 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11323 // val ^ (-1 ^ (val >> shift)) ==> eonw
11324 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11325 iRegIorL2I src1, iRegIorL2I src2,
11326 immI src3, immI_M1 src4) %{
11327 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11328 ins_cost(1.9 * INSN_COST);
11329 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11330
11331 ins_encode %{
11332 __ eonw(as_Register($dst$$reg),
11333 as_Register($src1$$reg),
11334 as_Register($src2$$reg),
11335 Assembler::ASR,
11336 $src3$$constant & 0x1f);
11337 %}
11338
11339 ins_pipe(ialu_reg_reg_shift);
11340 %}
11341
11342 // This pattern is automatically generated from aarch64_ad.m4
11343 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11344 // val ^ (-1 ^ (val >> shift)) ==> eon
11345 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11346 iRegL src1, iRegL src2,
11347 immI src3, immL_M1 src4) %{
11348 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11349 ins_cost(1.9 * INSN_COST);
11350 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11351
11352 ins_encode %{
11353 __ eon(as_Register($dst$$reg),
11354 as_Register($src1$$reg),
11355 as_Register($src2$$reg),
11356 Assembler::ASR,
11357 $src3$$constant & 0x3f);
11358 %}
11359
11360 ins_pipe(ialu_reg_reg_shift);
11361 %}
11362
11363 // This pattern is automatically generated from aarch64_ad.m4
11364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11365 // val ^ (-1 ^ (val ror shift)) ==> eonw
11366 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11367 iRegIorL2I src1, iRegIorL2I src2,
11368 immI src3, immI_M1 src4) %{
11369 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11370 ins_cost(1.9 * INSN_COST);
11371 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11372
11373 ins_encode %{
11374 __ eonw(as_Register($dst$$reg),
11375 as_Register($src1$$reg),
11376 as_Register($src2$$reg),
11377 Assembler::ROR,
11378 $src3$$constant & 0x1f);
11379 %}
11380
11381 ins_pipe(ialu_reg_reg_shift);
11382 %}
11383
11384 // This pattern is automatically generated from aarch64_ad.m4
11385 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11386 // val ^ (-1 ^ (val ror shift)) ==> eon
11387 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11388 iRegL src1, iRegL src2,
11389 immI src3, immL_M1 src4) %{
11390 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11391 ins_cost(1.9 * INSN_COST);
11392 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11393
11394 ins_encode %{
11395 __ eon(as_Register($dst$$reg),
11396 as_Register($src1$$reg),
11397 as_Register($src2$$reg),
11398 Assembler::ROR,
11399 $src3$$constant & 0x3f);
11400 %}
11401
11402 ins_pipe(ialu_reg_reg_shift);
11403 %}
11404
11405 // This pattern is automatically generated from aarch64_ad.m4
11406 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11407 // val ^ (-1 ^ (val << shift)) ==> eonw
11408 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11409 iRegIorL2I src1, iRegIorL2I src2,
11410 immI src3, immI_M1 src4) %{
11411 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11412 ins_cost(1.9 * INSN_COST);
11413 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11414
11415 ins_encode %{
11416 __ eonw(as_Register($dst$$reg),
11417 as_Register($src1$$reg),
11418 as_Register($src2$$reg),
11419 Assembler::LSL,
11420 $src3$$constant & 0x1f);
11421 %}
11422
11423 ins_pipe(ialu_reg_reg_shift);
11424 %}
11425
11426 // This pattern is automatically generated from aarch64_ad.m4
11427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11428 // val ^ (-1 ^ (val << shift)) ==> eon
11429 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11430 iRegL src1, iRegL src2,
11431 immI src3, immL_M1 src4) %{
11432 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11433 ins_cost(1.9 * INSN_COST);
11434 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11435
11436 ins_encode %{
11437 __ eon(as_Register($dst$$reg),
11438 as_Register($src1$$reg),
11439 as_Register($src2$$reg),
11440 Assembler::LSL,
11441 $src3$$constant & 0x3f);
11442 %}
11443
11444 ins_pipe(ialu_reg_reg_shift);
11445 %}
11446
11447 // This pattern is automatically generated from aarch64_ad.m4
11448 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11449 // val | (-1 ^ (val >>> shift)) ==> ornw
11450 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11451 iRegIorL2I src1, iRegIorL2I src2,
11452 immI src3, immI_M1 src4) %{
11453 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11454 ins_cost(1.9 * INSN_COST);
11455 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11456
11457 ins_encode %{
11458 __ ornw(as_Register($dst$$reg),
11459 as_Register($src1$$reg),
11460 as_Register($src2$$reg),
11461 Assembler::LSR,
11462 $src3$$constant & 0x1f);
11463 %}
11464
11465 ins_pipe(ialu_reg_reg_shift);
11466 %}
11467
11468 // This pattern is automatically generated from aarch64_ad.m4
11469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11470 // val | (-1 ^ (val >>> shift)) ==> orn
11471 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11472 iRegL src1, iRegL src2,
11473 immI src3, immL_M1 src4) %{
11474 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11475 ins_cost(1.9 * INSN_COST);
11476 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11477
11478 ins_encode %{
11479 __ orn(as_Register($dst$$reg),
11480 as_Register($src1$$reg),
11481 as_Register($src2$$reg),
11482 Assembler::LSR,
11483 $src3$$constant & 0x3f);
11484 %}
11485
11486 ins_pipe(ialu_reg_reg_shift);
11487 %}
11488
11489 // This pattern is automatically generated from aarch64_ad.m4
11490 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11491 // val | (-1 ^ (val >> shift)) ==> ornw
11492 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11493 iRegIorL2I src1, iRegIorL2I src2,
11494 immI src3, immI_M1 src4) %{
11495 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11496 ins_cost(1.9 * INSN_COST);
11497 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11498
11499 ins_encode %{
11500 __ ornw(as_Register($dst$$reg),
11501 as_Register($src1$$reg),
11502 as_Register($src2$$reg),
11503 Assembler::ASR,
11504 $src3$$constant & 0x1f);
11505 %}
11506
11507 ins_pipe(ialu_reg_reg_shift);
11508 %}
11509
11510 // This pattern is automatically generated from aarch64_ad.m4
11511 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11512 // val | (-1 ^ (val >> shift)) ==> orn
11513 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11514 iRegL src1, iRegL src2,
11515 immI src3, immL_M1 src4) %{
11516 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11517 ins_cost(1.9 * INSN_COST);
11518 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11519
11520 ins_encode %{
11521 __ orn(as_Register($dst$$reg),
11522 as_Register($src1$$reg),
11523 as_Register($src2$$reg),
11524 Assembler::ASR,
11525 $src3$$constant & 0x3f);
11526 %}
11527
11528 ins_pipe(ialu_reg_reg_shift);
11529 %}
11530
11531 // This pattern is automatically generated from aarch64_ad.m4
11532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11533 // val | (-1 ^ (val ror shift)) ==> ornw
11534 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11535 iRegIorL2I src1, iRegIorL2I src2,
11536 immI src3, immI_M1 src4) %{
11537 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11538 ins_cost(1.9 * INSN_COST);
11539 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11540
11541 ins_encode %{
11542 __ ornw(as_Register($dst$$reg),
11543 as_Register($src1$$reg),
11544 as_Register($src2$$reg),
11545 Assembler::ROR,
11546 $src3$$constant & 0x1f);
11547 %}
11548
11549 ins_pipe(ialu_reg_reg_shift);
11550 %}
11551
11552 // This pattern is automatically generated from aarch64_ad.m4
11553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11554 // val | (-1 ^ (val ror shift)) ==> orn
11555 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11556 iRegL src1, iRegL src2,
11557 immI src3, immL_M1 src4) %{
11558 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11559 ins_cost(1.9 * INSN_COST);
11560 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11561
11562 ins_encode %{
11563 __ orn(as_Register($dst$$reg),
11564 as_Register($src1$$reg),
11565 as_Register($src2$$reg),
11566 Assembler::ROR,
11567 $src3$$constant & 0x3f);
11568 %}
11569
11570 ins_pipe(ialu_reg_reg_shift);
11571 %}
11572
11573 // This pattern is automatically generated from aarch64_ad.m4
11574 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11575 // val | (-1 ^ (val << shift)) ==> ornw
11576 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11577 iRegIorL2I src1, iRegIorL2I src2,
11578 immI src3, immI_M1 src4) %{
11579 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11580 ins_cost(1.9 * INSN_COST);
11581 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11582
11583 ins_encode %{
11584 __ ornw(as_Register($dst$$reg),
11585 as_Register($src1$$reg),
11586 as_Register($src2$$reg),
11587 Assembler::LSL,
11588 $src3$$constant & 0x1f);
11589 %}
11590
11591 ins_pipe(ialu_reg_reg_shift);
11592 %}
11593
11594 // This pattern is automatically generated from aarch64_ad.m4
11595 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11596 // val | (-1 ^ (val << shift)) ==> orn
11597 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11598 iRegL src1, iRegL src2,
11599 immI src3, immL_M1 src4) %{
11600 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11601 ins_cost(1.9 * INSN_COST);
11602 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11603
11604 ins_encode %{
11605 __ orn(as_Register($dst$$reg),
11606 as_Register($src1$$reg),
11607 as_Register($src2$$reg),
11608 Assembler::LSL,
11609 $src3$$constant & 0x3f);
11610 %}
11611
11612 ins_pipe(ialu_reg_reg_shift);
11613 %}
11614
11615 // This pattern is automatically generated from aarch64_ad.m4
11616 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11617 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11618 iRegIorL2I src1, iRegIorL2I src2,
11619 immI src3) %{
11620 match(Set dst (AndI src1 (URShiftI src2 src3)));
11621
11622 ins_cost(1.9 * INSN_COST);
11623 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11624
11625 ins_encode %{
11626 __ andw(as_Register($dst$$reg),
11627 as_Register($src1$$reg),
11628 as_Register($src2$$reg),
11629 Assembler::LSR,
11630 $src3$$constant & 0x1f);
11631 %}
11632
11633 ins_pipe(ialu_reg_reg_shift);
11634 %}
11635
11636 // This pattern is automatically generated from aarch64_ad.m4
11637 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11638 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11639 iRegL src1, iRegL src2,
11640 immI src3) %{
11641 match(Set dst (AndL src1 (URShiftL src2 src3)));
11642
11643 ins_cost(1.9 * INSN_COST);
11644 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11645
11646 ins_encode %{
11647 __ andr(as_Register($dst$$reg),
11648 as_Register($src1$$reg),
11649 as_Register($src2$$reg),
11650 Assembler::LSR,
11651 $src3$$constant & 0x3f);
11652 %}
11653
11654 ins_pipe(ialu_reg_reg_shift);
11655 %}
11656
11657 // This pattern is automatically generated from aarch64_ad.m4
11658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11659 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11660 iRegIorL2I src1, iRegIorL2I src2,
11661 immI src3) %{
11662 match(Set dst (AndI src1 (RShiftI src2 src3)));
11663
11664 ins_cost(1.9 * INSN_COST);
11665 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11666
11667 ins_encode %{
11668 __ andw(as_Register($dst$$reg),
11669 as_Register($src1$$reg),
11670 as_Register($src2$$reg),
11671 Assembler::ASR,
11672 $src3$$constant & 0x1f);
11673 %}
11674
11675 ins_pipe(ialu_reg_reg_shift);
11676 %}
11677
11678 // This pattern is automatically generated from aarch64_ad.m4
11679 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11680 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11681 iRegL src1, iRegL src2,
11682 immI src3) %{
11683 match(Set dst (AndL src1 (RShiftL src2 src3)));
11684
11685 ins_cost(1.9 * INSN_COST);
11686 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11687
11688 ins_encode %{
11689 __ andr(as_Register($dst$$reg),
11690 as_Register($src1$$reg),
11691 as_Register($src2$$reg),
11692 Assembler::ASR,
11693 $src3$$constant & 0x3f);
11694 %}
11695
11696 ins_pipe(ialu_reg_reg_shift);
11697 %}
11698
11699 // This pattern is automatically generated from aarch64_ad.m4
11700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11701 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11702 iRegIorL2I src1, iRegIorL2I src2,
11703 immI src3) %{
11704 match(Set dst (AndI src1 (LShiftI src2 src3)));
11705
11706 ins_cost(1.9 * INSN_COST);
11707 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11708
11709 ins_encode %{
11710 __ andw(as_Register($dst$$reg),
11711 as_Register($src1$$reg),
11712 as_Register($src2$$reg),
11713 Assembler::LSL,
11714 $src3$$constant & 0x1f);
11715 %}
11716
11717 ins_pipe(ialu_reg_reg_shift);
11718 %}
11719
11720 // This pattern is automatically generated from aarch64_ad.m4
11721 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11722 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11723 iRegL src1, iRegL src2,
11724 immI src3) %{
11725 match(Set dst (AndL src1 (LShiftL src2 src3)));
11726
11727 ins_cost(1.9 * INSN_COST);
11728 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11729
11730 ins_encode %{
11731 __ andr(as_Register($dst$$reg),
11732 as_Register($src1$$reg),
11733 as_Register($src2$$reg),
11734 Assembler::LSL,
11735 $src3$$constant & 0x3f);
11736 %}
11737
11738 ins_pipe(ialu_reg_reg_shift);
11739 %}
11740
11741 // This pattern is automatically generated from aarch64_ad.m4
11742 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11743 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11744 iRegIorL2I src1, iRegIorL2I src2,
11745 immI src3) %{
11746 match(Set dst (AndI src1 (RotateRight src2 src3)));
11747
11748 ins_cost(1.9 * INSN_COST);
11749 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11750
11751 ins_encode %{
11752 __ andw(as_Register($dst$$reg),
11753 as_Register($src1$$reg),
11754 as_Register($src2$$reg),
11755 Assembler::ROR,
11756 $src3$$constant & 0x1f);
11757 %}
11758
11759 ins_pipe(ialu_reg_reg_shift);
11760 %}
11761
11762 // This pattern is automatically generated from aarch64_ad.m4
11763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11764 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11765 iRegL src1, iRegL src2,
11766 immI src3) %{
11767 match(Set dst (AndL src1 (RotateRight src2 src3)));
11768
11769 ins_cost(1.9 * INSN_COST);
11770 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11771
11772 ins_encode %{
11773 __ andr(as_Register($dst$$reg),
11774 as_Register($src1$$reg),
11775 as_Register($src2$$reg),
11776 Assembler::ROR,
11777 $src3$$constant & 0x3f);
11778 %}
11779
11780 ins_pipe(ialu_reg_reg_shift);
11781 %}
11782
11783 // This pattern is automatically generated from aarch64_ad.m4
11784 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11785 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11786 iRegIorL2I src1, iRegIorL2I src2,
11787 immI src3) %{
11788 match(Set dst (XorI src1 (URShiftI src2 src3)));
11789
11790 ins_cost(1.9 * INSN_COST);
11791 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11792
11793 ins_encode %{
11794 __ eorw(as_Register($dst$$reg),
11795 as_Register($src1$$reg),
11796 as_Register($src2$$reg),
11797 Assembler::LSR,
11798 $src3$$constant & 0x1f);
11799 %}
11800
11801 ins_pipe(ialu_reg_reg_shift);
11802 %}
11803
11804 // This pattern is automatically generated from aarch64_ad.m4
11805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11806 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11807 iRegL src1, iRegL src2,
11808 immI src3) %{
11809 match(Set dst (XorL src1 (URShiftL src2 src3)));
11810
11811 ins_cost(1.9 * INSN_COST);
11812 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11813
11814 ins_encode %{
11815 __ eor(as_Register($dst$$reg),
11816 as_Register($src1$$reg),
11817 as_Register($src2$$reg),
11818 Assembler::LSR,
11819 $src3$$constant & 0x3f);
11820 %}
11821
11822 ins_pipe(ialu_reg_reg_shift);
11823 %}
11824
11825 // This pattern is automatically generated from aarch64_ad.m4
11826 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11827 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11828 iRegIorL2I src1, iRegIorL2I src2,
11829 immI src3) %{
11830 match(Set dst (XorI src1 (RShiftI src2 src3)));
11831
11832 ins_cost(1.9 * INSN_COST);
11833 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11834
11835 ins_encode %{
11836 __ eorw(as_Register($dst$$reg),
11837 as_Register($src1$$reg),
11838 as_Register($src2$$reg),
11839 Assembler::ASR,
11840 $src3$$constant & 0x1f);
11841 %}
11842
11843 ins_pipe(ialu_reg_reg_shift);
11844 %}
11845
11846 // This pattern is automatically generated from aarch64_ad.m4
11847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11848 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11849 iRegL src1, iRegL src2,
11850 immI src3) %{
11851 match(Set dst (XorL src1 (RShiftL src2 src3)));
11852
11853 ins_cost(1.9 * INSN_COST);
11854 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11855
11856 ins_encode %{
11857 __ eor(as_Register($dst$$reg),
11858 as_Register($src1$$reg),
11859 as_Register($src2$$reg),
11860 Assembler::ASR,
11861 $src3$$constant & 0x3f);
11862 %}
11863
11864 ins_pipe(ialu_reg_reg_shift);
11865 %}
11866
11867 // This pattern is automatically generated from aarch64_ad.m4
11868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11869 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11870 iRegIorL2I src1, iRegIorL2I src2,
11871 immI src3) %{
11872 match(Set dst (XorI src1 (LShiftI src2 src3)));
11873
11874 ins_cost(1.9 * INSN_COST);
11875 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11876
11877 ins_encode %{
11878 __ eorw(as_Register($dst$$reg),
11879 as_Register($src1$$reg),
11880 as_Register($src2$$reg),
11881 Assembler::LSL,
11882 $src3$$constant & 0x1f);
11883 %}
11884
11885 ins_pipe(ialu_reg_reg_shift);
11886 %}
11887
11888 // This pattern is automatically generated from aarch64_ad.m4
11889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11890 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11891 iRegL src1, iRegL src2,
11892 immI src3) %{
11893 match(Set dst (XorL src1 (LShiftL src2 src3)));
11894
11895 ins_cost(1.9 * INSN_COST);
11896 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11897
11898 ins_encode %{
11899 __ eor(as_Register($dst$$reg),
11900 as_Register($src1$$reg),
11901 as_Register($src2$$reg),
11902 Assembler::LSL,
11903 $src3$$constant & 0x3f);
11904 %}
11905
11906 ins_pipe(ialu_reg_reg_shift);
11907 %}
11908
11909 // This pattern is automatically generated from aarch64_ad.m4
11910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11911 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11912 iRegIorL2I src1, iRegIorL2I src2,
11913 immI src3) %{
11914 match(Set dst (XorI src1 (RotateRight src2 src3)));
11915
11916 ins_cost(1.9 * INSN_COST);
11917 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11918
11919 ins_encode %{
11920 __ eorw(as_Register($dst$$reg),
11921 as_Register($src1$$reg),
11922 as_Register($src2$$reg),
11923 Assembler::ROR,
11924 $src3$$constant & 0x1f);
11925 %}
11926
11927 ins_pipe(ialu_reg_reg_shift);
11928 %}
11929
11930 // This pattern is automatically generated from aarch64_ad.m4
11931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11932 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11933 iRegL src1, iRegL src2,
11934 immI src3) %{
11935 match(Set dst (XorL src1 (RotateRight src2 src3)));
11936
11937 ins_cost(1.9 * INSN_COST);
11938 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11939
11940 ins_encode %{
11941 __ eor(as_Register($dst$$reg),
11942 as_Register($src1$$reg),
11943 as_Register($src2$$reg),
11944 Assembler::ROR,
11945 $src3$$constant & 0x3f);
11946 %}
11947
11948 ins_pipe(ialu_reg_reg_shift);
11949 %}
11950
11951 // This pattern is automatically generated from aarch64_ad.m4
11952 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11953 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11954 iRegIorL2I src1, iRegIorL2I src2,
11955 immI src3) %{
11956 match(Set dst (OrI src1 (URShiftI src2 src3)));
11957
11958 ins_cost(1.9 * INSN_COST);
11959 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11960
11961 ins_encode %{
11962 __ orrw(as_Register($dst$$reg),
11963 as_Register($src1$$reg),
11964 as_Register($src2$$reg),
11965 Assembler::LSR,
11966 $src3$$constant & 0x1f);
11967 %}
11968
11969 ins_pipe(ialu_reg_reg_shift);
11970 %}
11971
11972 // This pattern is automatically generated from aarch64_ad.m4
11973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11974 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11975 iRegL src1, iRegL src2,
11976 immI src3) %{
11977 match(Set dst (OrL src1 (URShiftL src2 src3)));
11978
11979 ins_cost(1.9 * INSN_COST);
11980 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11981
11982 ins_encode %{
11983 __ orr(as_Register($dst$$reg),
11984 as_Register($src1$$reg),
11985 as_Register($src2$$reg),
11986 Assembler::LSR,
11987 $src3$$constant & 0x3f);
11988 %}
11989
11990 ins_pipe(ialu_reg_reg_shift);
11991 %}
11992
11993 // This pattern is automatically generated from aarch64_ad.m4
11994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11995 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11996 iRegIorL2I src1, iRegIorL2I src2,
11997 immI src3) %{
11998 match(Set dst (OrI src1 (RShiftI src2 src3)));
11999
12000 ins_cost(1.9 * INSN_COST);
12001 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12002
12003 ins_encode %{
12004 __ orrw(as_Register($dst$$reg),
12005 as_Register($src1$$reg),
12006 as_Register($src2$$reg),
12007 Assembler::ASR,
12008 $src3$$constant & 0x1f);
12009 %}
12010
12011 ins_pipe(ialu_reg_reg_shift);
12012 %}
12013
12014 // This pattern is automatically generated from aarch64_ad.m4
12015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12016 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12017 iRegL src1, iRegL src2,
12018 immI src3) %{
12019 match(Set dst (OrL src1 (RShiftL src2 src3)));
12020
12021 ins_cost(1.9 * INSN_COST);
12022 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12023
12024 ins_encode %{
12025 __ orr(as_Register($dst$$reg),
12026 as_Register($src1$$reg),
12027 as_Register($src2$$reg),
12028 Assembler::ASR,
12029 $src3$$constant & 0x3f);
12030 %}
12031
12032 ins_pipe(ialu_reg_reg_shift);
12033 %}
12034
12035 // This pattern is automatically generated from aarch64_ad.m4
12036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12037 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12038 iRegIorL2I src1, iRegIorL2I src2,
12039 immI src3) %{
12040 match(Set dst (OrI src1 (LShiftI src2 src3)));
12041
12042 ins_cost(1.9 * INSN_COST);
12043 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12044
12045 ins_encode %{
12046 __ orrw(as_Register($dst$$reg),
12047 as_Register($src1$$reg),
12048 as_Register($src2$$reg),
12049 Assembler::LSL,
12050 $src3$$constant & 0x1f);
12051 %}
12052
12053 ins_pipe(ialu_reg_reg_shift);
12054 %}
12055
12056 // This pattern is automatically generated from aarch64_ad.m4
12057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12058 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12059 iRegL src1, iRegL src2,
12060 immI src3) %{
12061 match(Set dst (OrL src1 (LShiftL src2 src3)));
12062
12063 ins_cost(1.9 * INSN_COST);
12064 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12065
12066 ins_encode %{
12067 __ orr(as_Register($dst$$reg),
12068 as_Register($src1$$reg),
12069 as_Register($src2$$reg),
12070 Assembler::LSL,
12071 $src3$$constant & 0x3f);
12072 %}
12073
12074 ins_pipe(ialu_reg_reg_shift);
12075 %}
12076
12077 // This pattern is automatically generated from aarch64_ad.m4
12078 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12079 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12080 iRegIorL2I src1, iRegIorL2I src2,
12081 immI src3) %{
12082 match(Set dst (OrI src1 (RotateRight src2 src3)));
12083
12084 ins_cost(1.9 * INSN_COST);
12085 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12086
12087 ins_encode %{
12088 __ orrw(as_Register($dst$$reg),
12089 as_Register($src1$$reg),
12090 as_Register($src2$$reg),
12091 Assembler::ROR,
12092 $src3$$constant & 0x1f);
12093 %}
12094
12095 ins_pipe(ialu_reg_reg_shift);
12096 %}
12097
12098 // This pattern is automatically generated from aarch64_ad.m4
12099 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12100 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12101 iRegL src1, iRegL src2,
12102 immI src3) %{
12103 match(Set dst (OrL src1 (RotateRight src2 src3)));
12104
12105 ins_cost(1.9 * INSN_COST);
12106 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12107
12108 ins_encode %{
12109 __ orr(as_Register($dst$$reg),
12110 as_Register($src1$$reg),
12111 as_Register($src2$$reg),
12112 Assembler::ROR,
12113 $src3$$constant & 0x3f);
12114 %}
12115
12116 ins_pipe(ialu_reg_reg_shift);
12117 %}
12118
12119 // This pattern is automatically generated from aarch64_ad.m4
12120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12121 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12122 iRegIorL2I src1, iRegIorL2I src2,
12123 immI src3) %{
12124 match(Set dst (AddI src1 (URShiftI src2 src3)));
12125
12126 ins_cost(1.9 * INSN_COST);
12127 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12128
12129 ins_encode %{
12130 __ addw(as_Register($dst$$reg),
12131 as_Register($src1$$reg),
12132 as_Register($src2$$reg),
12133 Assembler::LSR,
12134 $src3$$constant & 0x1f);
12135 %}
12136
12137 ins_pipe(ialu_reg_reg_shift);
12138 %}
12139
12140 // This pattern is automatically generated from aarch64_ad.m4
12141 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12142 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12143 iRegL src1, iRegL src2,
12144 immI src3) %{
12145 match(Set dst (AddL src1 (URShiftL src2 src3)));
12146
12147 ins_cost(1.9 * INSN_COST);
12148 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12149
12150 ins_encode %{
12151 __ add(as_Register($dst$$reg),
12152 as_Register($src1$$reg),
12153 as_Register($src2$$reg),
12154 Assembler::LSR,
12155 $src3$$constant & 0x3f);
12156 %}
12157
12158 ins_pipe(ialu_reg_reg_shift);
12159 %}
12160
12161 // This pattern is automatically generated from aarch64_ad.m4
12162 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12163 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12164 iRegIorL2I src1, iRegIorL2I src2,
12165 immI src3) %{
12166 match(Set dst (AddI src1 (RShiftI src2 src3)));
12167
12168 ins_cost(1.9 * INSN_COST);
12169 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12170
12171 ins_encode %{
12172 __ addw(as_Register($dst$$reg),
12173 as_Register($src1$$reg),
12174 as_Register($src2$$reg),
12175 Assembler::ASR,
12176 $src3$$constant & 0x1f);
12177 %}
12178
12179 ins_pipe(ialu_reg_reg_shift);
12180 %}
12181
12182 // This pattern is automatically generated from aarch64_ad.m4
12183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12184 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12185 iRegL src1, iRegL src2,
12186 immI src3) %{
12187 match(Set dst (AddL src1 (RShiftL src2 src3)));
12188
12189 ins_cost(1.9 * INSN_COST);
12190 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12191
12192 ins_encode %{
12193 __ add(as_Register($dst$$reg),
12194 as_Register($src1$$reg),
12195 as_Register($src2$$reg),
12196 Assembler::ASR,
12197 $src3$$constant & 0x3f);
12198 %}
12199
12200 ins_pipe(ialu_reg_reg_shift);
12201 %}
12202
12203 // This pattern is automatically generated from aarch64_ad.m4
12204 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12205 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12206 iRegIorL2I src1, iRegIorL2I src2,
12207 immI src3) %{
12208 match(Set dst (AddI src1 (LShiftI src2 src3)));
12209
12210 ins_cost(1.9 * INSN_COST);
12211 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12212
12213 ins_encode %{
12214 __ addw(as_Register($dst$$reg),
12215 as_Register($src1$$reg),
12216 as_Register($src2$$reg),
12217 Assembler::LSL,
12218 $src3$$constant & 0x1f);
12219 %}
12220
12221 ins_pipe(ialu_reg_reg_shift);
12222 %}
12223
12224 // This pattern is automatically generated from aarch64_ad.m4
12225 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12226 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12227 iRegL src1, iRegL src2,
12228 immI src3) %{
12229 match(Set dst (AddL src1 (LShiftL src2 src3)));
12230
12231 ins_cost(1.9 * INSN_COST);
12232 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12233
12234 ins_encode %{
12235 __ add(as_Register($dst$$reg),
12236 as_Register($src1$$reg),
12237 as_Register($src2$$reg),
12238 Assembler::LSL,
12239 $src3$$constant & 0x3f);
12240 %}
12241
12242 ins_pipe(ialu_reg_reg_shift);
12243 %}
12244
12245 // This pattern is automatically generated from aarch64_ad.m4
12246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12247 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12248 iRegIorL2I src1, iRegIorL2I src2,
12249 immI src3) %{
12250 match(Set dst (SubI src1 (URShiftI src2 src3)));
12251
12252 ins_cost(1.9 * INSN_COST);
12253 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12254
12255 ins_encode %{
12256 __ subw(as_Register($dst$$reg),
12257 as_Register($src1$$reg),
12258 as_Register($src2$$reg),
12259 Assembler::LSR,
12260 $src3$$constant & 0x1f);
12261 %}
12262
12263 ins_pipe(ialu_reg_reg_shift);
12264 %}
12265
12266 // This pattern is automatically generated from aarch64_ad.m4
12267 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12268 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12269 iRegL src1, iRegL src2,
12270 immI src3) %{
12271 match(Set dst (SubL src1 (URShiftL src2 src3)));
12272
12273 ins_cost(1.9 * INSN_COST);
12274 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12275
12276 ins_encode %{
12277 __ sub(as_Register($dst$$reg),
12278 as_Register($src1$$reg),
12279 as_Register($src2$$reg),
12280 Assembler::LSR,
12281 $src3$$constant & 0x3f);
12282 %}
12283
12284 ins_pipe(ialu_reg_reg_shift);
12285 %}
12286
12287 // This pattern is automatically generated from aarch64_ad.m4
12288 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12289 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12290 iRegIorL2I src1, iRegIorL2I src2,
12291 immI src3) %{
12292 match(Set dst (SubI src1 (RShiftI src2 src3)));
12293
12294 ins_cost(1.9 * INSN_COST);
12295 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12296
12297 ins_encode %{
12298 __ subw(as_Register($dst$$reg),
12299 as_Register($src1$$reg),
12300 as_Register($src2$$reg),
12301 Assembler::ASR,
12302 $src3$$constant & 0x1f);
12303 %}
12304
12305 ins_pipe(ialu_reg_reg_shift);
12306 %}
12307
12308 // This pattern is automatically generated from aarch64_ad.m4
12309 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12310 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12311 iRegL src1, iRegL src2,
12312 immI src3) %{
12313 match(Set dst (SubL src1 (RShiftL src2 src3)));
12314
12315 ins_cost(1.9 * INSN_COST);
12316 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12317
12318 ins_encode %{
12319 __ sub(as_Register($dst$$reg),
12320 as_Register($src1$$reg),
12321 as_Register($src2$$reg),
12322 Assembler::ASR,
12323 $src3$$constant & 0x3f);
12324 %}
12325
12326 ins_pipe(ialu_reg_reg_shift);
12327 %}
12328
12329 // This pattern is automatically generated from aarch64_ad.m4
12330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12331 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12332 iRegIorL2I src1, iRegIorL2I src2,
12333 immI src3) %{
12334 match(Set dst (SubI src1 (LShiftI src2 src3)));
12335
12336 ins_cost(1.9 * INSN_COST);
12337 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12338
12339 ins_encode %{
12340 __ subw(as_Register($dst$$reg),
12341 as_Register($src1$$reg),
12342 as_Register($src2$$reg),
12343 Assembler::LSL,
12344 $src3$$constant & 0x1f);
12345 %}
12346
12347 ins_pipe(ialu_reg_reg_shift);
12348 %}
12349
12350 // This pattern is automatically generated from aarch64_ad.m4
12351 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12352 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12353 iRegL src1, iRegL src2,
12354 immI src3) %{
12355 match(Set dst (SubL src1 (LShiftL src2 src3)));
12356
12357 ins_cost(1.9 * INSN_COST);
12358 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12359
12360 ins_encode %{
12361 __ sub(as_Register($dst$$reg),
12362 as_Register($src1$$reg),
12363 as_Register($src2$$reg),
12364 Assembler::LSL,
12365 $src3$$constant & 0x3f);
12366 %}
12367
12368 ins_pipe(ialu_reg_reg_shift);
12369 %}
12370
12371 // This pattern is automatically generated from aarch64_ad.m4
12372 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12373
12374 // Shift Left followed by Shift Right.
12375 // This idiom is used by the compiler for the i2b bytecode etc.
12376 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12377 %{
12378 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12379 ins_cost(INSN_COST * 2);
12380 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12381 ins_encode %{
12382 int lshift = $lshift_count$$constant & 63;
12383 int rshift = $rshift_count$$constant & 63;
12384 int s = 63 - lshift;
12385 int r = (rshift - lshift) & 63;
12386 __ sbfm(as_Register($dst$$reg),
12387 as_Register($src$$reg),
12388 r, s);
12389 %}
12390
12391 ins_pipe(ialu_reg_shift);
12392 %}
12393
12394 // This pattern is automatically generated from aarch64_ad.m4
12395 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12396
12397 // Shift Left followed by Shift Right.
12398 // This idiom is used by the compiler for the i2b bytecode etc.
12399 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12400 %{
12401 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12402 ins_cost(INSN_COST * 2);
12403 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12404 ins_encode %{
12405 int lshift = $lshift_count$$constant & 31;
12406 int rshift = $rshift_count$$constant & 31;
12407 int s = 31 - lshift;
12408 int r = (rshift - lshift) & 31;
12409 __ sbfmw(as_Register($dst$$reg),
12410 as_Register($src$$reg),
12411 r, s);
12412 %}
12413
12414 ins_pipe(ialu_reg_shift);
12415 %}
12416
12417 // This pattern is automatically generated from aarch64_ad.m4
12418 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12419
12420 // Shift Left followed by Shift Right.
12421 // This idiom is used by the compiler for the i2b bytecode etc.
12422 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12423 %{
12424 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12425 ins_cost(INSN_COST * 2);
12426 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12427 ins_encode %{
12428 int lshift = $lshift_count$$constant & 63;
12429 int rshift = $rshift_count$$constant & 63;
12430 int s = 63 - lshift;
12431 int r = (rshift - lshift) & 63;
12432 __ ubfm(as_Register($dst$$reg),
12433 as_Register($src$$reg),
12434 r, s);
12435 %}
12436
12437 ins_pipe(ialu_reg_shift);
12438 %}
12439
12440 // This pattern is automatically generated from aarch64_ad.m4
12441 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12442
12443 // Shift Left followed by Shift Right.
12444 // This idiom is used by the compiler for the i2b bytecode etc.
12445 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12446 %{
12447 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12448 ins_cost(INSN_COST * 2);
12449 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12450 ins_encode %{
12451 int lshift = $lshift_count$$constant & 31;
12452 int rshift = $rshift_count$$constant & 31;
12453 int s = 31 - lshift;
12454 int r = (rshift - lshift) & 31;
12455 __ ubfmw(as_Register($dst$$reg),
12456 as_Register($src$$reg),
12457 r, s);
12458 %}
12459
12460 ins_pipe(ialu_reg_shift);
12461 %}
12462
12463 // Bitfield extract with shift & mask
12464
12465 // This pattern is automatically generated from aarch64_ad.m4
12466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12467 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12468 %{
12469 match(Set dst (AndI (URShiftI src rshift) mask));
12470 // Make sure we are not going to exceed what ubfxw can do.
12471 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12472
12473 ins_cost(INSN_COST);
12474 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12475 ins_encode %{
12476 int rshift = $rshift$$constant & 31;
12477 intptr_t mask = $mask$$constant;
12478 int width = exact_log2(mask+1);
12479 __ ubfxw(as_Register($dst$$reg),
12480 as_Register($src$$reg), rshift, width);
12481 %}
12482 ins_pipe(ialu_reg_shift);
12483 %}
12484
12485 // This pattern is automatically generated from aarch64_ad.m4
12486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12487 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12488 %{
12489 match(Set dst (AndL (URShiftL src rshift) mask));
12490 // Make sure we are not going to exceed what ubfx can do.
12491 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12492
12493 ins_cost(INSN_COST);
12494 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12495 ins_encode %{
12496 int rshift = $rshift$$constant & 63;
12497 intptr_t mask = $mask$$constant;
12498 int width = exact_log2_long(mask+1);
12499 __ ubfx(as_Register($dst$$reg),
12500 as_Register($src$$reg), rshift, width);
12501 %}
12502 ins_pipe(ialu_reg_shift);
12503 %}
12504
12505
12506 // This pattern is automatically generated from aarch64_ad.m4
12507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12508
12509 // We can use ubfx when extending an And with a mask when we know mask
12510 // is positive. We know that because immI_bitmask guarantees it.
12511 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12512 %{
12513 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12514 // Make sure we are not going to exceed what ubfxw can do.
12515 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12516
12517 ins_cost(INSN_COST * 2);
12518 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12519 ins_encode %{
12520 int rshift = $rshift$$constant & 31;
12521 intptr_t mask = $mask$$constant;
12522 int width = exact_log2(mask+1);
12523 __ ubfx(as_Register($dst$$reg),
12524 as_Register($src$$reg), rshift, width);
12525 %}
12526 ins_pipe(ialu_reg_shift);
12527 %}
12528
12529
12530 // This pattern is automatically generated from aarch64_ad.m4
12531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12532
12533 // We can use ubfiz when masking by a positive number and then left shifting the result.
12534 // We know that the mask is positive because immI_bitmask guarantees it.
12535 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12536 %{
12537 match(Set dst (LShiftI (AndI src mask) lshift));
12538 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12539
12540 ins_cost(INSN_COST);
12541 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12542 ins_encode %{
12543 int lshift = $lshift$$constant & 31;
12544 intptr_t mask = $mask$$constant;
12545 int width = exact_log2(mask+1);
12546 __ ubfizw(as_Register($dst$$reg),
12547 as_Register($src$$reg), lshift, width);
12548 %}
12549 ins_pipe(ialu_reg_shift);
12550 %}
12551
12552 // This pattern is automatically generated from aarch64_ad.m4
12553 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12554
12555 // We can use ubfiz when masking by a positive number and then left shifting the result.
12556 // We know that the mask is positive because immL_bitmask guarantees it.
12557 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12558 %{
12559 match(Set dst (LShiftL (AndL src mask) lshift));
12560 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12561
12562 ins_cost(INSN_COST);
12563 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12564 ins_encode %{
12565 int lshift = $lshift$$constant & 63;
12566 intptr_t mask = $mask$$constant;
12567 int width = exact_log2_long(mask+1);
12568 __ ubfiz(as_Register($dst$$reg),
12569 as_Register($src$$reg), lshift, width);
12570 %}
12571 ins_pipe(ialu_reg_shift);
12572 %}
12573
12574 // This pattern is automatically generated from aarch64_ad.m4
12575 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12576
12577 // We can use ubfiz when masking by a positive number and then left shifting the result.
12578 // We know that the mask is positive because immI_bitmask guarantees it.
12579 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12580 %{
12581 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12582 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12583
12584 ins_cost(INSN_COST);
12585 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12586 ins_encode %{
12587 int lshift = $lshift$$constant & 31;
12588 intptr_t mask = $mask$$constant;
12589 int width = exact_log2(mask+1);
12590 __ ubfizw(as_Register($dst$$reg),
12591 as_Register($src$$reg), lshift, width);
12592 %}
12593 ins_pipe(ialu_reg_shift);
12594 %}
12595
12596 // This pattern is automatically generated from aarch64_ad.m4
12597 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12598
12599 // We can use ubfiz when masking by a positive number and then left shifting the result.
12600 // We know that the mask is positive because immL_bitmask guarantees it.
12601 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12602 %{
12603 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12604 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12605
12606 ins_cost(INSN_COST);
12607 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12608 ins_encode %{
12609 int lshift = $lshift$$constant & 63;
12610 intptr_t mask = $mask$$constant;
12611 int width = exact_log2_long(mask+1);
12612 __ ubfiz(as_Register($dst$$reg),
12613 as_Register($src$$reg), lshift, width);
12614 %}
12615 ins_pipe(ialu_reg_shift);
12616 %}
12617
12618
12619 // This pattern is automatically generated from aarch64_ad.m4
12620 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12621
12622 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12623 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12624 %{
12625 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12626 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12627
12628 ins_cost(INSN_COST);
12629 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12630 ins_encode %{
12631 int lshift = $lshift$$constant & 63;
12632 intptr_t mask = $mask$$constant;
12633 int width = exact_log2(mask+1);
12634 __ ubfiz(as_Register($dst$$reg),
12635 as_Register($src$$reg), lshift, width);
12636 %}
12637 ins_pipe(ialu_reg_shift);
12638 %}
12639
12640 // This pattern is automatically generated from aarch64_ad.m4
12641 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12642
12643 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12644 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12645 %{
12646 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12647 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12648
12649 ins_cost(INSN_COST);
12650 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12651 ins_encode %{
12652 int lshift = $lshift$$constant & 31;
12653 intptr_t mask = $mask$$constant;
12654 int width = exact_log2(mask+1);
12655 __ ubfiz(as_Register($dst$$reg),
12656 as_Register($src$$reg), lshift, width);
12657 %}
12658 ins_pipe(ialu_reg_shift);
12659 %}
12660
12661 // This pattern is automatically generated from aarch64_ad.m4
12662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12663
12664 // Can skip int2long conversions after AND with small bitmask
12665 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12666 %{
12667 match(Set dst (ConvI2L (AndI src msk)));
12668 ins_cost(INSN_COST);
12669 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12670 ins_encode %{
12671 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12672 %}
12673 ins_pipe(ialu_reg_shift);
12674 %}
12675
12676
12677 // Rotations
12678
12679 // This pattern is automatically generated from aarch64_ad.m4
12680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12681 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12682 %{
12683 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12684 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12685
12686 ins_cost(INSN_COST);
12687 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12688
12689 ins_encode %{
12690 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12691 $rshift$$constant & 63);
12692 %}
12693 ins_pipe(ialu_reg_reg_extr);
12694 %}
12695
12696
12697 // This pattern is automatically generated from aarch64_ad.m4
12698 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12699 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12700 %{
12701 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12702 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12703
12704 ins_cost(INSN_COST);
12705 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12706
12707 ins_encode %{
12708 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12709 $rshift$$constant & 31);
12710 %}
12711 ins_pipe(ialu_reg_reg_extr);
12712 %}
12713
12714
12715 // This pattern is automatically generated from aarch64_ad.m4
12716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12717 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12718 %{
12719 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12720 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12721
12722 ins_cost(INSN_COST);
12723 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12724
12725 ins_encode %{
12726 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12727 $rshift$$constant & 63);
12728 %}
12729 ins_pipe(ialu_reg_reg_extr);
12730 %}
12731
12732
12733 // This pattern is automatically generated from aarch64_ad.m4
12734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12735 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12736 %{
12737 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12738 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12739
12740 ins_cost(INSN_COST);
12741 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12742
12743 ins_encode %{
12744 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12745 $rshift$$constant & 31);
12746 %}
12747 ins_pipe(ialu_reg_reg_extr);
12748 %}
12749
12750 // This pattern is automatically generated from aarch64_ad.m4
12751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12752 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12753 %{
12754 match(Set dst (RotateRight src shift));
12755
12756 ins_cost(INSN_COST);
12757 format %{ "ror $dst, $src, $shift" %}
12758
12759 ins_encode %{
12760 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12761 $shift$$constant & 0x1f);
12762 %}
12763 ins_pipe(ialu_reg_reg_vshift);
12764 %}
12765
12766 // This pattern is automatically generated from aarch64_ad.m4
12767 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12768 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12769 %{
12770 match(Set dst (RotateRight src shift));
12771
12772 ins_cost(INSN_COST);
12773 format %{ "ror $dst, $src, $shift" %}
12774
12775 ins_encode %{
12776 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12777 $shift$$constant & 0x3f);
12778 %}
12779 ins_pipe(ialu_reg_reg_vshift);
12780 %}
12781
12782 // This pattern is automatically generated from aarch64_ad.m4
12783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12784 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12785 %{
12786 match(Set dst (RotateRight src shift));
12787
12788 ins_cost(INSN_COST);
12789 format %{ "ror $dst, $src, $shift" %}
12790
12791 ins_encode %{
12792 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12793 %}
12794 ins_pipe(ialu_reg_reg_vshift);
12795 %}
12796
12797 // This pattern is automatically generated from aarch64_ad.m4
12798 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12799 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12800 %{
12801 match(Set dst (RotateRight src shift));
12802
12803 ins_cost(INSN_COST);
12804 format %{ "ror $dst, $src, $shift" %}
12805
12806 ins_encode %{
12807 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12808 %}
12809 ins_pipe(ialu_reg_reg_vshift);
12810 %}
12811
12812 // This pattern is automatically generated from aarch64_ad.m4
12813 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12814 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12815 %{
12816 match(Set dst (RotateLeft src shift));
12817
12818 ins_cost(INSN_COST);
12819 format %{ "rol $dst, $src, $shift" %}
12820
12821 ins_encode %{
12822 __ subw(rscratch1, zr, as_Register($shift$$reg));
12823 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12824 %}
12825 ins_pipe(ialu_reg_reg_vshift);
12826 %}
12827
12828 // This pattern is automatically generated from aarch64_ad.m4
12829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12830 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12831 %{
12832 match(Set dst (RotateLeft src shift));
12833
12834 ins_cost(INSN_COST);
12835 format %{ "rol $dst, $src, $shift" %}
12836
12837 ins_encode %{
12838 __ subw(rscratch1, zr, as_Register($shift$$reg));
12839 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12840 %}
12841 ins_pipe(ialu_reg_reg_vshift);
12842 %}
12843
12844
12845 // Add/subtract (extended)
12846
12847 // This pattern is automatically generated from aarch64_ad.m4
12848 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12849 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12850 %{
12851 match(Set dst (AddL src1 (ConvI2L src2)));
12852 ins_cost(INSN_COST);
12853 format %{ "add $dst, $src1, $src2, sxtw" %}
12854
12855 ins_encode %{
12856 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12857 as_Register($src2$$reg), ext::sxtw);
12858 %}
12859 ins_pipe(ialu_reg_reg);
12860 %}
12861
12862 // This pattern is automatically generated from aarch64_ad.m4
12863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12864 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12865 %{
12866 match(Set dst (SubL src1 (ConvI2L src2)));
12867 ins_cost(INSN_COST);
12868 format %{ "sub $dst, $src1, $src2, sxtw" %}
12869
12870 ins_encode %{
12871 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12872 as_Register($src2$$reg), ext::sxtw);
12873 %}
12874 ins_pipe(ialu_reg_reg);
12875 %}
12876
12877 // This pattern is automatically generated from aarch64_ad.m4
12878 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12879 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12880 %{
12881 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12882 ins_cost(INSN_COST);
12883 format %{ "add $dst, $src1, $src2, sxth" %}
12884
12885 ins_encode %{
12886 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12887 as_Register($src2$$reg), ext::sxth);
12888 %}
12889 ins_pipe(ialu_reg_reg);
12890 %}
12891
12892 // This pattern is automatically generated from aarch64_ad.m4
12893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12894 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12895 %{
12896 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12897 ins_cost(INSN_COST);
12898 format %{ "add $dst, $src1, $src2, sxtb" %}
12899
12900 ins_encode %{
12901 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12902 as_Register($src2$$reg), ext::sxtb);
12903 %}
12904 ins_pipe(ialu_reg_reg);
12905 %}
12906
12907 // This pattern is automatically generated from aarch64_ad.m4
12908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12909 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12910 %{
12911 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12912 ins_cost(INSN_COST);
12913 format %{ "add $dst, $src1, $src2, uxtb" %}
12914
12915 ins_encode %{
12916 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12917 as_Register($src2$$reg), ext::uxtb);
12918 %}
12919 ins_pipe(ialu_reg_reg);
12920 %}
12921
12922 // This pattern is automatically generated from aarch64_ad.m4
12923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12924 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12925 %{
12926 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12927 ins_cost(INSN_COST);
12928 format %{ "add $dst, $src1, $src2, sxth" %}
12929
12930 ins_encode %{
12931 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12932 as_Register($src2$$reg), ext::sxth);
12933 %}
12934 ins_pipe(ialu_reg_reg);
12935 %}
12936
12937 // This pattern is automatically generated from aarch64_ad.m4
12938 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12939 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12940 %{
12941 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12942 ins_cost(INSN_COST);
12943 format %{ "add $dst, $src1, $src2, sxtw" %}
12944
12945 ins_encode %{
12946 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12947 as_Register($src2$$reg), ext::sxtw);
12948 %}
12949 ins_pipe(ialu_reg_reg);
12950 %}
12951
12952 // This pattern is automatically generated from aarch64_ad.m4
12953 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12954 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12955 %{
12956 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12957 ins_cost(INSN_COST);
12958 format %{ "add $dst, $src1, $src2, sxtb" %}
12959
12960 ins_encode %{
12961 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12962 as_Register($src2$$reg), ext::sxtb);
12963 %}
12964 ins_pipe(ialu_reg_reg);
12965 %}
12966
12967 // This pattern is automatically generated from aarch64_ad.m4
12968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12969 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12970 %{
12971 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12972 ins_cost(INSN_COST);
12973 format %{ "add $dst, $src1, $src2, uxtb" %}
12974
12975 ins_encode %{
12976 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12977 as_Register($src2$$reg), ext::uxtb);
12978 %}
12979 ins_pipe(ialu_reg_reg);
12980 %}
12981
12982 // This pattern is automatically generated from aarch64_ad.m4
12983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12984 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12985 %{
12986 match(Set dst (AddI src1 (AndI src2 mask)));
12987 ins_cost(INSN_COST);
12988 format %{ "addw $dst, $src1, $src2, uxtb" %}
12989
12990 ins_encode %{
12991 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12992 as_Register($src2$$reg), ext::uxtb);
12993 %}
12994 ins_pipe(ialu_reg_reg);
12995 %}
12996
12997 // This pattern is automatically generated from aarch64_ad.m4
12998 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12999 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13000 %{
13001 match(Set dst (AddI src1 (AndI src2 mask)));
13002 ins_cost(INSN_COST);
13003 format %{ "addw $dst, $src1, $src2, uxth" %}
13004
13005 ins_encode %{
13006 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13007 as_Register($src2$$reg), ext::uxth);
13008 %}
13009 ins_pipe(ialu_reg_reg);
13010 %}
13011
13012 // This pattern is automatically generated from aarch64_ad.m4
13013 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13014 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13015 %{
13016 match(Set dst (AddL src1 (AndL src2 mask)));
13017 ins_cost(INSN_COST);
13018 format %{ "add $dst, $src1, $src2, uxtb" %}
13019
13020 ins_encode %{
13021 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13022 as_Register($src2$$reg), ext::uxtb);
13023 %}
13024 ins_pipe(ialu_reg_reg);
13025 %}
13026
13027 // This pattern is automatically generated from aarch64_ad.m4
13028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13029 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13030 %{
13031 match(Set dst (AddL src1 (AndL src2 mask)));
13032 ins_cost(INSN_COST);
13033 format %{ "add $dst, $src1, $src2, uxth" %}
13034
13035 ins_encode %{
13036 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13037 as_Register($src2$$reg), ext::uxth);
13038 %}
13039 ins_pipe(ialu_reg_reg);
13040 %}
13041
13042 // This pattern is automatically generated from aarch64_ad.m4
13043 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13044 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13045 %{
13046 match(Set dst (AddL src1 (AndL src2 mask)));
13047 ins_cost(INSN_COST);
13048 format %{ "add $dst, $src1, $src2, uxtw" %}
13049
13050 ins_encode %{
13051 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13052 as_Register($src2$$reg), ext::uxtw);
13053 %}
13054 ins_pipe(ialu_reg_reg);
13055 %}
13056
13057 // This pattern is automatically generated from aarch64_ad.m4
13058 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13059 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13060 %{
13061 match(Set dst (SubI src1 (AndI src2 mask)));
13062 ins_cost(INSN_COST);
13063 format %{ "subw $dst, $src1, $src2, uxtb" %}
13064
13065 ins_encode %{
13066 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13067 as_Register($src2$$reg), ext::uxtb);
13068 %}
13069 ins_pipe(ialu_reg_reg);
13070 %}
13071
13072 // This pattern is automatically generated from aarch64_ad.m4
13073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13074 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13075 %{
13076 match(Set dst (SubI src1 (AndI src2 mask)));
13077 ins_cost(INSN_COST);
13078 format %{ "subw $dst, $src1, $src2, uxth" %}
13079
13080 ins_encode %{
13081 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13082 as_Register($src2$$reg), ext::uxth);
13083 %}
13084 ins_pipe(ialu_reg_reg);
13085 %}
13086
13087 // This pattern is automatically generated from aarch64_ad.m4
13088 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13089 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13090 %{
13091 match(Set dst (SubL src1 (AndL src2 mask)));
13092 ins_cost(INSN_COST);
13093 format %{ "sub $dst, $src1, $src2, uxtb" %}
13094
13095 ins_encode %{
13096 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13097 as_Register($src2$$reg), ext::uxtb);
13098 %}
13099 ins_pipe(ialu_reg_reg);
13100 %}
13101
13102 // This pattern is automatically generated from aarch64_ad.m4
13103 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13104 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13105 %{
13106 match(Set dst (SubL src1 (AndL src2 mask)));
13107 ins_cost(INSN_COST);
13108 format %{ "sub $dst, $src1, $src2, uxth" %}
13109
13110 ins_encode %{
13111 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13112 as_Register($src2$$reg), ext::uxth);
13113 %}
13114 ins_pipe(ialu_reg_reg);
13115 %}
13116
13117 // This pattern is automatically generated from aarch64_ad.m4
13118 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13119 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13120 %{
13121 match(Set dst (SubL src1 (AndL src2 mask)));
13122 ins_cost(INSN_COST);
13123 format %{ "sub $dst, $src1, $src2, uxtw" %}
13124
13125 ins_encode %{
13126 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13127 as_Register($src2$$reg), ext::uxtw);
13128 %}
13129 ins_pipe(ialu_reg_reg);
13130 %}
13131
13132
13133 // This pattern is automatically generated from aarch64_ad.m4
13134 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13135 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13136 %{
13137 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13138 ins_cost(1.9 * INSN_COST);
13139 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13140
13141 ins_encode %{
13142 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13143 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13144 %}
13145 ins_pipe(ialu_reg_reg_shift);
13146 %}
13147
13148 // This pattern is automatically generated from aarch64_ad.m4
13149 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13150 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13151 %{
13152 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13153 ins_cost(1.9 * INSN_COST);
13154 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13155
13156 ins_encode %{
13157 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13158 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13159 %}
13160 ins_pipe(ialu_reg_reg_shift);
13161 %}
13162
13163 // This pattern is automatically generated from aarch64_ad.m4
13164 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13165 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13166 %{
13167 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13168 ins_cost(1.9 * INSN_COST);
13169 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13170
13171 ins_encode %{
13172 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13173 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13174 %}
13175 ins_pipe(ialu_reg_reg_shift);
13176 %}
13177
13178 // This pattern is automatically generated from aarch64_ad.m4
13179 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13180 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13181 %{
13182 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13183 ins_cost(1.9 * INSN_COST);
13184 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13185
13186 ins_encode %{
13187 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13188 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13189 %}
13190 ins_pipe(ialu_reg_reg_shift);
13191 %}
13192
13193 // This pattern is automatically generated from aarch64_ad.m4
13194 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13195 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13196 %{
13197 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13198 ins_cost(1.9 * INSN_COST);
13199 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13200
13201 ins_encode %{
13202 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13203 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13204 %}
13205 ins_pipe(ialu_reg_reg_shift);
13206 %}
13207
13208 // This pattern is automatically generated from aarch64_ad.m4
13209 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13210 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13211 %{
13212 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13213 ins_cost(1.9 * INSN_COST);
13214 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13215
13216 ins_encode %{
13217 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13218 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13219 %}
13220 ins_pipe(ialu_reg_reg_shift);
13221 %}
13222
13223 // This pattern is automatically generated from aarch64_ad.m4
13224 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13225 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13226 %{
13227 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13228 ins_cost(1.9 * INSN_COST);
13229 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13230
13231 ins_encode %{
13232 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13233 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13234 %}
13235 ins_pipe(ialu_reg_reg_shift);
13236 %}
13237
13238 // This pattern is automatically generated from aarch64_ad.m4
13239 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13240 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13241 %{
13242 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13243 ins_cost(1.9 * INSN_COST);
13244 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13245
13246 ins_encode %{
13247 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13248 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13249 %}
13250 ins_pipe(ialu_reg_reg_shift);
13251 %}
13252
13253 // This pattern is automatically generated from aarch64_ad.m4
13254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13255 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13256 %{
13257 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13258 ins_cost(1.9 * INSN_COST);
13259 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13260
13261 ins_encode %{
13262 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13263 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13264 %}
13265 ins_pipe(ialu_reg_reg_shift);
13266 %}
13267
13268 // This pattern is automatically generated from aarch64_ad.m4
13269 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13270 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13271 %{
13272 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13273 ins_cost(1.9 * INSN_COST);
13274 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13275
13276 ins_encode %{
13277 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13278 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13279 %}
13280 ins_pipe(ialu_reg_reg_shift);
13281 %}
13282
13283 // This pattern is automatically generated from aarch64_ad.m4
13284 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13285 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13286 %{
13287 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13288 ins_cost(1.9 * INSN_COST);
13289 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13290
13291 ins_encode %{
13292 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13293 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13294 %}
13295 ins_pipe(ialu_reg_reg_shift);
13296 %}
13297
13298 // This pattern is automatically generated from aarch64_ad.m4
13299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13300 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13301 %{
13302 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13303 ins_cost(1.9 * INSN_COST);
13304 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13305
13306 ins_encode %{
13307 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13308 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13309 %}
13310 ins_pipe(ialu_reg_reg_shift);
13311 %}
13312
13313 // This pattern is automatically generated from aarch64_ad.m4
13314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13315 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13316 %{
13317 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13318 ins_cost(1.9 * INSN_COST);
13319 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13320
13321 ins_encode %{
13322 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13323 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13324 %}
13325 ins_pipe(ialu_reg_reg_shift);
13326 %}
13327
13328 // This pattern is automatically generated from aarch64_ad.m4
13329 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13330 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13331 %{
13332 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13333 ins_cost(1.9 * INSN_COST);
13334 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13335
13336 ins_encode %{
13337 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13338 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13339 %}
13340 ins_pipe(ialu_reg_reg_shift);
13341 %}
13342
13343 // This pattern is automatically generated from aarch64_ad.m4
13344 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13345 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13346 %{
13347 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13348 ins_cost(1.9 * INSN_COST);
13349 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13350
13351 ins_encode %{
13352 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13353 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13354 %}
13355 ins_pipe(ialu_reg_reg_shift);
13356 %}
13357
13358 // This pattern is automatically generated from aarch64_ad.m4
13359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13360 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13361 %{
13362 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13363 ins_cost(1.9 * INSN_COST);
13364 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13365
13366 ins_encode %{
13367 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13368 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13369 %}
13370 ins_pipe(ialu_reg_reg_shift);
13371 %}
13372
13373 // This pattern is automatically generated from aarch64_ad.m4
13374 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13375 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13376 %{
13377 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13378 ins_cost(1.9 * INSN_COST);
13379 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13380
13381 ins_encode %{
13382 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13383 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13384 %}
13385 ins_pipe(ialu_reg_reg_shift);
13386 %}
13387
13388 // This pattern is automatically generated from aarch64_ad.m4
13389 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13390 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13391 %{
13392 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13393 ins_cost(1.9 * INSN_COST);
13394 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13395
13396 ins_encode %{
13397 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13398 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13399 %}
13400 ins_pipe(ialu_reg_reg_shift);
13401 %}
13402
13403 // This pattern is automatically generated from aarch64_ad.m4
13404 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13405 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13406 %{
13407 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13408 ins_cost(1.9 * INSN_COST);
13409 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13410
13411 ins_encode %{
13412 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13413 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13414 %}
13415 ins_pipe(ialu_reg_reg_shift);
13416 %}
13417
13418 // This pattern is automatically generated from aarch64_ad.m4
13419 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13420 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13421 %{
13422 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13423 ins_cost(1.9 * INSN_COST);
13424 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13425
13426 ins_encode %{
13427 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13428 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13429 %}
13430 ins_pipe(ialu_reg_reg_shift);
13431 %}
13432
13433 // This pattern is automatically generated from aarch64_ad.m4
13434 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13435 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13436 %{
13437 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13438 ins_cost(1.9 * INSN_COST);
13439 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13440
13441 ins_encode %{
13442 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13443 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13444 %}
13445 ins_pipe(ialu_reg_reg_shift);
13446 %}
13447
13448 // This pattern is automatically generated from aarch64_ad.m4
13449 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13450 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13451 %{
13452 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13453 ins_cost(1.9 * INSN_COST);
13454 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13455
13456 ins_encode %{
13457 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13458 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13459 %}
13460 ins_pipe(ialu_reg_reg_shift);
13461 %}
13462
13463 // This pattern is automatically generated from aarch64_ad.m4
13464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13465 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13466 %{
13467 effect(DEF dst, USE src1, USE src2, USE cr);
13468 ins_cost(INSN_COST * 2);
13469 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13470
13471 ins_encode %{
13472 __ cselw($dst$$Register,
13473 $src1$$Register,
13474 $src2$$Register,
13475 Assembler::LT);
13476 %}
13477 ins_pipe(icond_reg_reg);
13478 %}
13479
13480 // This pattern is automatically generated from aarch64_ad.m4
13481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13482 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13483 %{
13484 effect(DEF dst, USE src1, USE src2, USE cr);
13485 ins_cost(INSN_COST * 2);
13486 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13487
13488 ins_encode %{
13489 __ cselw($dst$$Register,
13490 $src1$$Register,
13491 $src2$$Register,
13492 Assembler::GT);
13493 %}
13494 ins_pipe(icond_reg_reg);
13495 %}
13496
13497 // This pattern is automatically generated from aarch64_ad.m4
13498 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13499 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13500 %{
13501 effect(DEF dst, USE src1, USE cr);
13502 ins_cost(INSN_COST * 2);
13503 format %{ "cselw $dst, $src1, zr lt\t" %}
13504
13505 ins_encode %{
13506 __ cselw($dst$$Register,
13507 $src1$$Register,
13508 zr,
13509 Assembler::LT);
13510 %}
13511 ins_pipe(icond_reg);
13512 %}
13513
13514 // This pattern is automatically generated from aarch64_ad.m4
13515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13516 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13517 %{
13518 effect(DEF dst, USE src1, USE cr);
13519 ins_cost(INSN_COST * 2);
13520 format %{ "cselw $dst, $src1, zr gt\t" %}
13521
13522 ins_encode %{
13523 __ cselw($dst$$Register,
13524 $src1$$Register,
13525 zr,
13526 Assembler::GT);
13527 %}
13528 ins_pipe(icond_reg);
13529 %}
13530
13531 // This pattern is automatically generated from aarch64_ad.m4
13532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13533 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13534 %{
13535 effect(DEF dst, USE src1, USE cr);
13536 ins_cost(INSN_COST * 2);
13537 format %{ "csincw $dst, $src1, zr le\t" %}
13538
13539 ins_encode %{
13540 __ csincw($dst$$Register,
13541 $src1$$Register,
13542 zr,
13543 Assembler::LE);
13544 %}
13545 ins_pipe(icond_reg);
13546 %}
13547
13548 // This pattern is automatically generated from aarch64_ad.m4
13549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13550 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13551 %{
13552 effect(DEF dst, USE src1, USE cr);
13553 ins_cost(INSN_COST * 2);
13554 format %{ "csincw $dst, $src1, zr gt\t" %}
13555
13556 ins_encode %{
13557 __ csincw($dst$$Register,
13558 $src1$$Register,
13559 zr,
13560 Assembler::GT);
13561 %}
13562 ins_pipe(icond_reg);
13563 %}
13564
13565 // This pattern is automatically generated from aarch64_ad.m4
13566 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13567 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13568 %{
13569 effect(DEF dst, USE src1, USE cr);
13570 ins_cost(INSN_COST * 2);
13571 format %{ "csinvw $dst, $src1, zr lt\t" %}
13572
13573 ins_encode %{
13574 __ csinvw($dst$$Register,
13575 $src1$$Register,
13576 zr,
13577 Assembler::LT);
13578 %}
13579 ins_pipe(icond_reg);
13580 %}
13581
13582 // This pattern is automatically generated from aarch64_ad.m4
13583 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13584 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13585 %{
13586 effect(DEF dst, USE src1, USE cr);
13587 ins_cost(INSN_COST * 2);
13588 format %{ "csinvw $dst, $src1, zr ge\t" %}
13589
13590 ins_encode %{
13591 __ csinvw($dst$$Register,
13592 $src1$$Register,
13593 zr,
13594 Assembler::GE);
13595 %}
13596 ins_pipe(icond_reg);
13597 %}
13598
13599 // This pattern is automatically generated from aarch64_ad.m4
13600 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13601 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13602 %{
13603 match(Set dst (MinI src imm));
13604 ins_cost(INSN_COST * 3);
13605 expand %{
13606 rFlagsReg cr;
13607 compI_reg_imm0(cr, src);
13608 cmovI_reg_imm0_lt(dst, src, cr);
13609 %}
13610 %}
13611
13612 // This pattern is automatically generated from aarch64_ad.m4
13613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13614 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13615 %{
13616 match(Set dst (MinI imm src));
13617 ins_cost(INSN_COST * 3);
13618 expand %{
13619 rFlagsReg cr;
13620 compI_reg_imm0(cr, src);
13621 cmovI_reg_imm0_lt(dst, src, cr);
13622 %}
13623 %}
13624
13625 // This pattern is automatically generated from aarch64_ad.m4
13626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13627 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13628 %{
13629 match(Set dst (MinI src imm));
13630 ins_cost(INSN_COST * 3);
13631 expand %{
13632 rFlagsReg cr;
13633 compI_reg_imm0(cr, src);
13634 cmovI_reg_imm1_le(dst, src, cr);
13635 %}
13636 %}
13637
13638 // This pattern is automatically generated from aarch64_ad.m4
13639 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13640 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13641 %{
13642 match(Set dst (MinI imm src));
13643 ins_cost(INSN_COST * 3);
13644 expand %{
13645 rFlagsReg cr;
13646 compI_reg_imm0(cr, src);
13647 cmovI_reg_imm1_le(dst, src, cr);
13648 %}
13649 %}
13650
13651 // This pattern is automatically generated from aarch64_ad.m4
13652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13653 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13654 %{
13655 match(Set dst (MinI src imm));
13656 ins_cost(INSN_COST * 3);
13657 expand %{
13658 rFlagsReg cr;
13659 compI_reg_imm0(cr, src);
13660 cmovI_reg_immM1_lt(dst, src, cr);
13661 %}
13662 %}
13663
13664 // This pattern is automatically generated from aarch64_ad.m4
13665 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13666 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13667 %{
13668 match(Set dst (MinI imm src));
13669 ins_cost(INSN_COST * 3);
13670 expand %{
13671 rFlagsReg cr;
13672 compI_reg_imm0(cr, src);
13673 cmovI_reg_immM1_lt(dst, src, cr);
13674 %}
13675 %}
13676
13677 // This pattern is automatically generated from aarch64_ad.m4
13678 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13679 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13680 %{
13681 match(Set dst (MaxI src imm));
13682 ins_cost(INSN_COST * 3);
13683 expand %{
13684 rFlagsReg cr;
13685 compI_reg_imm0(cr, src);
13686 cmovI_reg_imm0_gt(dst, src, cr);
13687 %}
13688 %}
13689
13690 // This pattern is automatically generated from aarch64_ad.m4
13691 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13692 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13693 %{
13694 match(Set dst (MaxI imm src));
13695 ins_cost(INSN_COST * 3);
13696 expand %{
13697 rFlagsReg cr;
13698 compI_reg_imm0(cr, src);
13699 cmovI_reg_imm0_gt(dst, src, cr);
13700 %}
13701 %}
13702
13703 // This pattern is automatically generated from aarch64_ad.m4
13704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13705 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13706 %{
13707 match(Set dst (MaxI src imm));
13708 ins_cost(INSN_COST * 3);
13709 expand %{
13710 rFlagsReg cr;
13711 compI_reg_imm0(cr, src);
13712 cmovI_reg_imm1_gt(dst, src, cr);
13713 %}
13714 %}
13715
13716 // This pattern is automatically generated from aarch64_ad.m4
13717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13718 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13719 %{
13720 match(Set dst (MaxI imm src));
13721 ins_cost(INSN_COST * 3);
13722 expand %{
13723 rFlagsReg cr;
13724 compI_reg_imm0(cr, src);
13725 cmovI_reg_imm1_gt(dst, src, cr);
13726 %}
13727 %}
13728
13729 // This pattern is automatically generated from aarch64_ad.m4
13730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13731 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13732 %{
13733 match(Set dst (MaxI src imm));
13734 ins_cost(INSN_COST * 3);
13735 expand %{
13736 rFlagsReg cr;
13737 compI_reg_imm0(cr, src);
13738 cmovI_reg_immM1_ge(dst, src, cr);
13739 %}
13740 %}
13741
13742 // This pattern is automatically generated from aarch64_ad.m4
13743 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13744 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13745 %{
13746 match(Set dst (MaxI imm src));
13747 ins_cost(INSN_COST * 3);
13748 expand %{
13749 rFlagsReg cr;
13750 compI_reg_imm0(cr, src);
13751 cmovI_reg_immM1_ge(dst, src, cr);
13752 %}
13753 %}
13754
13755 // This pattern is automatically generated from aarch64_ad.m4
13756 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13757 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13758 %{
13759 match(Set dst (ReverseI src));
13760 ins_cost(INSN_COST);
13761 format %{ "rbitw $dst, $src" %}
13762 ins_encode %{
13763 __ rbitw($dst$$Register, $src$$Register);
13764 %}
13765 ins_pipe(ialu_reg);
13766 %}
13767
13768 // This pattern is automatically generated from aarch64_ad.m4
13769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13770 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13771 %{
13772 match(Set dst (ReverseL src));
13773 ins_cost(INSN_COST);
13774 format %{ "rbit $dst, $src" %}
13775 ins_encode %{
13776 __ rbit($dst$$Register, $src$$Register);
13777 %}
13778 ins_pipe(ialu_reg);
13779 %}
13780
13781
13782 // END This section of the file is automatically generated. Do not edit --------------
13783
13784
13785 // ============================================================================
13786 // Floating Point Arithmetic Instructions
13787
13788 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13789 match(Set dst (AddHF src1 src2));
13790 format %{ "faddh $dst, $src1, $src2" %}
13791 ins_encode %{
13792 __ faddh($dst$$FloatRegister,
13793 $src1$$FloatRegister,
13794 $src2$$FloatRegister);
13795 %}
13796 ins_pipe(fp_dop_reg_reg_s);
13797 %}
13798
13799 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13800 match(Set dst (AddF src1 src2));
13801
13802 ins_cost(INSN_COST * 5);
13803 format %{ "fadds $dst, $src1, $src2" %}
13804
13805 ins_encode %{
13806 __ fadds(as_FloatRegister($dst$$reg),
13807 as_FloatRegister($src1$$reg),
13808 as_FloatRegister($src2$$reg));
13809 %}
13810
13811 ins_pipe(fp_dop_reg_reg_s);
13812 %}
13813
13814 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13815 match(Set dst (AddD src1 src2));
13816
13817 ins_cost(INSN_COST * 5);
13818 format %{ "faddd $dst, $src1, $src2" %}
13819
13820 ins_encode %{
13821 __ faddd(as_FloatRegister($dst$$reg),
13822 as_FloatRegister($src1$$reg),
13823 as_FloatRegister($src2$$reg));
13824 %}
13825
13826 ins_pipe(fp_dop_reg_reg_d);
13827 %}
13828
13829 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13830 match(Set dst (SubHF src1 src2));
13831 format %{ "fsubh $dst, $src1, $src2" %}
13832 ins_encode %{
13833 __ fsubh($dst$$FloatRegister,
13834 $src1$$FloatRegister,
13835 $src2$$FloatRegister);
13836 %}
13837 ins_pipe(fp_dop_reg_reg_s);
13838 %}
13839
13840 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13841 match(Set dst (SubF src1 src2));
13842
13843 ins_cost(INSN_COST * 5);
13844 format %{ "fsubs $dst, $src1, $src2" %}
13845
13846 ins_encode %{
13847 __ fsubs(as_FloatRegister($dst$$reg),
13848 as_FloatRegister($src1$$reg),
13849 as_FloatRegister($src2$$reg));
13850 %}
13851
13852 ins_pipe(fp_dop_reg_reg_s);
13853 %}
13854
13855 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13856 match(Set dst (SubD src1 src2));
13857
13858 ins_cost(INSN_COST * 5);
13859 format %{ "fsubd $dst, $src1, $src2" %}
13860
13861 ins_encode %{
13862 __ fsubd(as_FloatRegister($dst$$reg),
13863 as_FloatRegister($src1$$reg),
13864 as_FloatRegister($src2$$reg));
13865 %}
13866
13867 ins_pipe(fp_dop_reg_reg_d);
13868 %}
13869
13870 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13871 match(Set dst (MulHF src1 src2));
13872 format %{ "fmulh $dst, $src1, $src2" %}
13873 ins_encode %{
13874 __ fmulh($dst$$FloatRegister,
13875 $src1$$FloatRegister,
13876 $src2$$FloatRegister);
13877 %}
13878 ins_pipe(fp_dop_reg_reg_s);
13879 %}
13880
13881 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13882 match(Set dst (MulF src1 src2));
13883
13884 ins_cost(INSN_COST * 6);
13885 format %{ "fmuls $dst, $src1, $src2" %}
13886
13887 ins_encode %{
13888 __ fmuls(as_FloatRegister($dst$$reg),
13889 as_FloatRegister($src1$$reg),
13890 as_FloatRegister($src2$$reg));
13891 %}
13892
13893 ins_pipe(fp_dop_reg_reg_s);
13894 %}
13895
13896 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13897 match(Set dst (MulD src1 src2));
13898
13899 ins_cost(INSN_COST * 6);
13900 format %{ "fmuld $dst, $src1, $src2" %}
13901
13902 ins_encode %{
13903 __ fmuld(as_FloatRegister($dst$$reg),
13904 as_FloatRegister($src1$$reg),
13905 as_FloatRegister($src2$$reg));
13906 %}
13907
13908 ins_pipe(fp_dop_reg_reg_d);
13909 %}
13910
13911 // src1 * src2 + src3 (half-precision float)
13912 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13913 match(Set dst (FmaHF src3 (Binary src1 src2)));
13914 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13915 ins_encode %{
13916 assert(UseFMA, "Needs FMA instructions support.");
13917 __ fmaddh($dst$$FloatRegister,
13918 $src1$$FloatRegister,
13919 $src2$$FloatRegister,
13920 $src3$$FloatRegister);
13921 %}
13922 ins_pipe(pipe_class_default);
13923 %}
13924
13925 // src1 * src2 + src3
13926 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13927 match(Set dst (FmaF src3 (Binary src1 src2)));
13928
13929 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13930
13931 ins_encode %{
13932 assert(UseFMA, "Needs FMA instructions support.");
13933 __ fmadds(as_FloatRegister($dst$$reg),
13934 as_FloatRegister($src1$$reg),
13935 as_FloatRegister($src2$$reg),
13936 as_FloatRegister($src3$$reg));
13937 %}
13938
13939 ins_pipe(pipe_class_default);
13940 %}
13941
13942 // src1 * src2 + src3
13943 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13944 match(Set dst (FmaD src3 (Binary src1 src2)));
13945
13946 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13947
13948 ins_encode %{
13949 assert(UseFMA, "Needs FMA instructions support.");
13950 __ fmaddd(as_FloatRegister($dst$$reg),
13951 as_FloatRegister($src1$$reg),
13952 as_FloatRegister($src2$$reg),
13953 as_FloatRegister($src3$$reg));
13954 %}
13955
13956 ins_pipe(pipe_class_default);
13957 %}
13958
13959 // src1 * (-src2) + src3
13960 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13961 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13962 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13963
13964 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13965
13966 ins_encode %{
13967 assert(UseFMA, "Needs FMA instructions support.");
13968 __ fmsubs(as_FloatRegister($dst$$reg),
13969 as_FloatRegister($src1$$reg),
13970 as_FloatRegister($src2$$reg),
13971 as_FloatRegister($src3$$reg));
13972 %}
13973
13974 ins_pipe(pipe_class_default);
13975 %}
13976
13977 // src1 * (-src2) + src3
13978 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13979 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13980 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13981
13982 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13983
13984 ins_encode %{
13985 assert(UseFMA, "Needs FMA instructions support.");
13986 __ fmsubd(as_FloatRegister($dst$$reg),
13987 as_FloatRegister($src1$$reg),
13988 as_FloatRegister($src2$$reg),
13989 as_FloatRegister($src3$$reg));
13990 %}
13991
13992 ins_pipe(pipe_class_default);
13993 %}
13994
13995 // src1 * (-src2) - src3
13996 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13997 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13998 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13999
14000 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14001
14002 ins_encode %{
14003 assert(UseFMA, "Needs FMA instructions support.");
14004 __ fnmadds(as_FloatRegister($dst$$reg),
14005 as_FloatRegister($src1$$reg),
14006 as_FloatRegister($src2$$reg),
14007 as_FloatRegister($src3$$reg));
14008 %}
14009
14010 ins_pipe(pipe_class_default);
14011 %}
14012
14013 // src1 * (-src2) - src3
14014 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14015 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14016 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14017
14018 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14019
14020 ins_encode %{
14021 assert(UseFMA, "Needs FMA instructions support.");
14022 __ fnmaddd(as_FloatRegister($dst$$reg),
14023 as_FloatRegister($src1$$reg),
14024 as_FloatRegister($src2$$reg),
14025 as_FloatRegister($src3$$reg));
14026 %}
14027
14028 ins_pipe(pipe_class_default);
14029 %}
14030
14031 // src1 * src2 - src3
14032 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14033 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14034
14035 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14036
14037 ins_encode %{
14038 assert(UseFMA, "Needs FMA instructions support.");
14039 __ fnmsubs(as_FloatRegister($dst$$reg),
14040 as_FloatRegister($src1$$reg),
14041 as_FloatRegister($src2$$reg),
14042 as_FloatRegister($src3$$reg));
14043 %}
14044
14045 ins_pipe(pipe_class_default);
14046 %}
14047
14048 // src1 * src2 - src3
14049 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14050 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14051
14052 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14053
14054 ins_encode %{
14055 assert(UseFMA, "Needs FMA instructions support.");
14056 // n.b. insn name should be fnmsubd
14057 __ fnmsub(as_FloatRegister($dst$$reg),
14058 as_FloatRegister($src1$$reg),
14059 as_FloatRegister($src2$$reg),
14060 as_FloatRegister($src3$$reg));
14061 %}
14062
14063 ins_pipe(pipe_class_default);
14064 %}
14065
14066 // Math.max(HH)H (half-precision float)
14067 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14068 match(Set dst (MaxHF src1 src2));
14069 format %{ "fmaxh $dst, $src1, $src2" %}
14070 ins_encode %{
14071 __ fmaxh($dst$$FloatRegister,
14072 $src1$$FloatRegister,
14073 $src2$$FloatRegister);
14074 %}
14075 ins_pipe(fp_dop_reg_reg_s);
14076 %}
14077
14078 // Math.min(HH)H (half-precision float)
14079 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14080 match(Set dst (MinHF src1 src2));
14081 format %{ "fminh $dst, $src1, $src2" %}
14082 ins_encode %{
14083 __ fminh($dst$$FloatRegister,
14084 $src1$$FloatRegister,
14085 $src2$$FloatRegister);
14086 %}
14087 ins_pipe(fp_dop_reg_reg_s);
14088 %}
14089
14090 // Math.max(FF)F
14091 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14092 match(Set dst (MaxF src1 src2));
14093
14094 format %{ "fmaxs $dst, $src1, $src2" %}
14095 ins_encode %{
14096 __ fmaxs(as_FloatRegister($dst$$reg),
14097 as_FloatRegister($src1$$reg),
14098 as_FloatRegister($src2$$reg));
14099 %}
14100
14101 ins_pipe(fp_dop_reg_reg_s);
14102 %}
14103
14104 // Math.min(FF)F
14105 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14106 match(Set dst (MinF src1 src2));
14107
14108 format %{ "fmins $dst, $src1, $src2" %}
14109 ins_encode %{
14110 __ fmins(as_FloatRegister($dst$$reg),
14111 as_FloatRegister($src1$$reg),
14112 as_FloatRegister($src2$$reg));
14113 %}
14114
14115 ins_pipe(fp_dop_reg_reg_s);
14116 %}
14117
14118 // Math.max(DD)D
14119 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14120 match(Set dst (MaxD src1 src2));
14121
14122 format %{ "fmaxd $dst, $src1, $src2" %}
14123 ins_encode %{
14124 __ fmaxd(as_FloatRegister($dst$$reg),
14125 as_FloatRegister($src1$$reg),
14126 as_FloatRegister($src2$$reg));
14127 %}
14128
14129 ins_pipe(fp_dop_reg_reg_d);
14130 %}
14131
14132 // Math.min(DD)D
14133 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14134 match(Set dst (MinD src1 src2));
14135
14136 format %{ "fmind $dst, $src1, $src2" %}
14137 ins_encode %{
14138 __ fmind(as_FloatRegister($dst$$reg),
14139 as_FloatRegister($src1$$reg),
14140 as_FloatRegister($src2$$reg));
14141 %}
14142
14143 ins_pipe(fp_dop_reg_reg_d);
14144 %}
14145
14146 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14147 match(Set dst (DivHF src1 src2));
14148 format %{ "fdivh $dst, $src1, $src2" %}
14149 ins_encode %{
14150 __ fdivh($dst$$FloatRegister,
14151 $src1$$FloatRegister,
14152 $src2$$FloatRegister);
14153 %}
14154 ins_pipe(fp_div_s);
14155 %}
14156
14157 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14158 match(Set dst (DivF src1 src2));
14159
14160 ins_cost(INSN_COST * 18);
14161 format %{ "fdivs $dst, $src1, $src2" %}
14162
14163 ins_encode %{
14164 __ fdivs(as_FloatRegister($dst$$reg),
14165 as_FloatRegister($src1$$reg),
14166 as_FloatRegister($src2$$reg));
14167 %}
14168
14169 ins_pipe(fp_div_s);
14170 %}
14171
14172 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14173 match(Set dst (DivD src1 src2));
14174
14175 ins_cost(INSN_COST * 32);
14176 format %{ "fdivd $dst, $src1, $src2" %}
14177
14178 ins_encode %{
14179 __ fdivd(as_FloatRegister($dst$$reg),
14180 as_FloatRegister($src1$$reg),
14181 as_FloatRegister($src2$$reg));
14182 %}
14183
14184 ins_pipe(fp_div_d);
14185 %}
14186
14187 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14188 match(Set dst (NegF src));
14189
14190 ins_cost(INSN_COST * 3);
14191 format %{ "fneg $dst, $src" %}
14192
14193 ins_encode %{
14194 __ fnegs(as_FloatRegister($dst$$reg),
14195 as_FloatRegister($src$$reg));
14196 %}
14197
14198 ins_pipe(fp_uop_s);
14199 %}
14200
14201 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14202 match(Set dst (NegD src));
14203
14204 ins_cost(INSN_COST * 3);
14205 format %{ "fnegd $dst, $src" %}
14206
14207 ins_encode %{
14208 __ fnegd(as_FloatRegister($dst$$reg),
14209 as_FloatRegister($src$$reg));
14210 %}
14211
14212 ins_pipe(fp_uop_d);
14213 %}
14214
14215 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14216 %{
14217 match(Set dst (AbsI src));
14218
14219 effect(KILL cr);
14220 ins_cost(INSN_COST * 2);
14221 format %{ "cmpw $src, zr\n\t"
14222 "cnegw $dst, $src, Assembler::LT\t# int abs"
14223 %}
14224
14225 ins_encode %{
14226 __ cmpw(as_Register($src$$reg), zr);
14227 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14228 %}
14229 ins_pipe(pipe_class_default);
14230 %}
14231
14232 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14233 %{
14234 match(Set dst (AbsL src));
14235
14236 effect(KILL cr);
14237 ins_cost(INSN_COST * 2);
14238 format %{ "cmp $src, zr\n\t"
14239 "cneg $dst, $src, Assembler::LT\t# long abs"
14240 %}
14241
14242 ins_encode %{
14243 __ cmp(as_Register($src$$reg), zr);
14244 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14245 %}
14246 ins_pipe(pipe_class_default);
14247 %}
14248
14249 instruct absF_reg(vRegF dst, vRegF src) %{
14250 match(Set dst (AbsF src));
14251
14252 ins_cost(INSN_COST * 3);
14253 format %{ "fabss $dst, $src" %}
14254 ins_encode %{
14255 __ fabss(as_FloatRegister($dst$$reg),
14256 as_FloatRegister($src$$reg));
14257 %}
14258
14259 ins_pipe(fp_uop_s);
14260 %}
14261
14262 instruct absD_reg(vRegD dst, vRegD src) %{
14263 match(Set dst (AbsD src));
14264
14265 ins_cost(INSN_COST * 3);
14266 format %{ "fabsd $dst, $src" %}
14267 ins_encode %{
14268 __ fabsd(as_FloatRegister($dst$$reg),
14269 as_FloatRegister($src$$reg));
14270 %}
14271
14272 ins_pipe(fp_uop_d);
14273 %}
14274
14275 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14276 match(Set dst (AbsF (SubF src1 src2)));
14277
14278 ins_cost(INSN_COST * 3);
14279 format %{ "fabds $dst, $src1, $src2" %}
14280 ins_encode %{
14281 __ fabds(as_FloatRegister($dst$$reg),
14282 as_FloatRegister($src1$$reg),
14283 as_FloatRegister($src2$$reg));
14284 %}
14285
14286 ins_pipe(fp_uop_s);
14287 %}
14288
14289 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14290 match(Set dst (AbsD (SubD src1 src2)));
14291
14292 ins_cost(INSN_COST * 3);
14293 format %{ "fabdd $dst, $src1, $src2" %}
14294 ins_encode %{
14295 __ fabdd(as_FloatRegister($dst$$reg),
14296 as_FloatRegister($src1$$reg),
14297 as_FloatRegister($src2$$reg));
14298 %}
14299
14300 ins_pipe(fp_uop_d);
14301 %}
14302
14303 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14304 match(Set dst (SqrtD src));
14305
14306 ins_cost(INSN_COST * 50);
14307 format %{ "fsqrtd $dst, $src" %}
14308 ins_encode %{
14309 __ fsqrtd(as_FloatRegister($dst$$reg),
14310 as_FloatRegister($src$$reg));
14311 %}
14312
14313 ins_pipe(fp_div_s);
14314 %}
14315
14316 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14317 match(Set dst (SqrtF src));
14318
14319 ins_cost(INSN_COST * 50);
14320 format %{ "fsqrts $dst, $src" %}
14321 ins_encode %{
14322 __ fsqrts(as_FloatRegister($dst$$reg),
14323 as_FloatRegister($src$$reg));
14324 %}
14325
14326 ins_pipe(fp_div_d);
14327 %}
14328
14329 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14330 match(Set dst (SqrtHF src));
14331 format %{ "fsqrth $dst, $src" %}
14332 ins_encode %{
14333 __ fsqrth($dst$$FloatRegister,
14334 $src$$FloatRegister);
14335 %}
14336 ins_pipe(fp_div_s);
14337 %}
14338
14339 // Math.rint, floor, ceil
14340 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14341 match(Set dst (RoundDoubleMode src rmode));
14342 format %{ "frint $dst, $src, $rmode" %}
14343 ins_encode %{
14344 switch ($rmode$$constant) {
14345 case RoundDoubleModeNode::rmode_rint:
14346 __ frintnd(as_FloatRegister($dst$$reg),
14347 as_FloatRegister($src$$reg));
14348 break;
14349 case RoundDoubleModeNode::rmode_floor:
14350 __ frintmd(as_FloatRegister($dst$$reg),
14351 as_FloatRegister($src$$reg));
14352 break;
14353 case RoundDoubleModeNode::rmode_ceil:
14354 __ frintpd(as_FloatRegister($dst$$reg),
14355 as_FloatRegister($src$$reg));
14356 break;
14357 }
14358 %}
14359 ins_pipe(fp_uop_d);
14360 %}
14361
14362 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14363 match(Set dst (CopySignD src1 (Binary src2 zero)));
14364 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14365 format %{ "CopySignD $dst $src1 $src2" %}
14366 ins_encode %{
14367 FloatRegister dst = as_FloatRegister($dst$$reg),
14368 src1 = as_FloatRegister($src1$$reg),
14369 src2 = as_FloatRegister($src2$$reg),
14370 zero = as_FloatRegister($zero$$reg);
14371 __ fnegd(dst, zero);
14372 __ bsl(dst, __ T8B, src2, src1);
14373 %}
14374 ins_pipe(fp_uop_d);
14375 %}
14376
14377 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14378 match(Set dst (CopySignF src1 src2));
14379 effect(TEMP_DEF dst, USE src1, USE src2);
14380 format %{ "CopySignF $dst $src1 $src2" %}
14381 ins_encode %{
14382 FloatRegister dst = as_FloatRegister($dst$$reg),
14383 src1 = as_FloatRegister($src1$$reg),
14384 src2 = as_FloatRegister($src2$$reg);
14385 __ movi(dst, __ T2S, 0x80, 24);
14386 __ bsl(dst, __ T8B, src2, src1);
14387 %}
14388 ins_pipe(fp_uop_d);
14389 %}
14390
14391 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14392 match(Set dst (SignumD src (Binary zero one)));
14393 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14394 format %{ "signumD $dst, $src" %}
14395 ins_encode %{
14396 FloatRegister src = as_FloatRegister($src$$reg),
14397 dst = as_FloatRegister($dst$$reg),
14398 zero = as_FloatRegister($zero$$reg),
14399 one = as_FloatRegister($one$$reg);
14400 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14401 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14402 // Bit selection instruction gets bit from "one" for each enabled bit in
14403 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14404 // NaN the whole "src" will be copied because "dst" is zero. For all other
14405 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14406 // from "src", and all other bits are copied from 1.0.
14407 __ bsl(dst, __ T8B, one, src);
14408 %}
14409 ins_pipe(fp_uop_d);
14410 %}
14411
14412 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14413 match(Set dst (SignumF src (Binary zero one)));
14414 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14415 format %{ "signumF $dst, $src" %}
14416 ins_encode %{
14417 FloatRegister src = as_FloatRegister($src$$reg),
14418 dst = as_FloatRegister($dst$$reg),
14419 zero = as_FloatRegister($zero$$reg),
14420 one = as_FloatRegister($one$$reg);
14421 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14422 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14423 // Bit selection instruction gets bit from "one" for each enabled bit in
14424 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14425 // NaN the whole "src" will be copied because "dst" is zero. For all other
14426 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14427 // from "src", and all other bits are copied from 1.0.
14428 __ bsl(dst, __ T8B, one, src);
14429 %}
14430 ins_pipe(fp_uop_d);
14431 %}
14432
14433 instruct onspinwait() %{
14434 match(OnSpinWait);
14435 ins_cost(INSN_COST);
14436
14437 format %{ "onspinwait" %}
14438
14439 ins_encode %{
14440 __ spin_wait();
14441 %}
14442 ins_pipe(pipe_class_empty);
14443 %}
14444
14445 // ============================================================================
14446 // Logical Instructions
14447
14448 // Integer Logical Instructions
14449
14450 // And Instructions
14451
14452
14453 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14454 match(Set dst (AndI src1 src2));
14455
14456 format %{ "andw $dst, $src1, $src2\t# int" %}
14457
14458 ins_cost(INSN_COST);
14459 ins_encode %{
14460 __ andw(as_Register($dst$$reg),
14461 as_Register($src1$$reg),
14462 as_Register($src2$$reg));
14463 %}
14464
14465 ins_pipe(ialu_reg_reg);
14466 %}
14467
14468 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14469 match(Set dst (AndI src1 src2));
14470
14471 format %{ "andsw $dst, $src1, $src2\t# int" %}
14472
14473 ins_cost(INSN_COST);
14474 ins_encode %{
14475 __ andw(as_Register($dst$$reg),
14476 as_Register($src1$$reg),
14477 (uint64_t)($src2$$constant));
14478 %}
14479
14480 ins_pipe(ialu_reg_imm);
14481 %}
14482
14483 // Or Instructions
14484
14485 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14486 match(Set dst (OrI src1 src2));
14487
14488 format %{ "orrw $dst, $src1, $src2\t# int" %}
14489
14490 ins_cost(INSN_COST);
14491 ins_encode %{
14492 __ orrw(as_Register($dst$$reg),
14493 as_Register($src1$$reg),
14494 as_Register($src2$$reg));
14495 %}
14496
14497 ins_pipe(ialu_reg_reg);
14498 %}
14499
14500 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14501 match(Set dst (OrI src1 src2));
14502
14503 format %{ "orrw $dst, $src1, $src2\t# int" %}
14504
14505 ins_cost(INSN_COST);
14506 ins_encode %{
14507 __ orrw(as_Register($dst$$reg),
14508 as_Register($src1$$reg),
14509 (uint64_t)($src2$$constant));
14510 %}
14511
14512 ins_pipe(ialu_reg_imm);
14513 %}
14514
14515 // Xor Instructions
14516
14517 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14518 match(Set dst (XorI src1 src2));
14519
14520 format %{ "eorw $dst, $src1, $src2\t# int" %}
14521
14522 ins_cost(INSN_COST);
14523 ins_encode %{
14524 __ eorw(as_Register($dst$$reg),
14525 as_Register($src1$$reg),
14526 as_Register($src2$$reg));
14527 %}
14528
14529 ins_pipe(ialu_reg_reg);
14530 %}
14531
14532 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14533 match(Set dst (XorI src1 src2));
14534
14535 format %{ "eorw $dst, $src1, $src2\t# int" %}
14536
14537 ins_cost(INSN_COST);
14538 ins_encode %{
14539 __ eorw(as_Register($dst$$reg),
14540 as_Register($src1$$reg),
14541 (uint64_t)($src2$$constant));
14542 %}
14543
14544 ins_pipe(ialu_reg_imm);
14545 %}
14546
14547 // Long Logical Instructions
14548 // TODO
14549
14550 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14551 match(Set dst (AndL src1 src2));
14552
14553 format %{ "and $dst, $src1, $src2\t# int" %}
14554
14555 ins_cost(INSN_COST);
14556 ins_encode %{
14557 __ andr(as_Register($dst$$reg),
14558 as_Register($src1$$reg),
14559 as_Register($src2$$reg));
14560 %}
14561
14562 ins_pipe(ialu_reg_reg);
14563 %}
14564
14565 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14566 match(Set dst (AndL src1 src2));
14567
14568 format %{ "and $dst, $src1, $src2\t# int" %}
14569
14570 ins_cost(INSN_COST);
14571 ins_encode %{
14572 __ andr(as_Register($dst$$reg),
14573 as_Register($src1$$reg),
14574 (uint64_t)($src2$$constant));
14575 %}
14576
14577 ins_pipe(ialu_reg_imm);
14578 %}
14579
14580 // Or Instructions
14581
14582 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14583 match(Set dst (OrL src1 src2));
14584
14585 format %{ "orr $dst, $src1, $src2\t# int" %}
14586
14587 ins_cost(INSN_COST);
14588 ins_encode %{
14589 __ orr(as_Register($dst$$reg),
14590 as_Register($src1$$reg),
14591 as_Register($src2$$reg));
14592 %}
14593
14594 ins_pipe(ialu_reg_reg);
14595 %}
14596
14597 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14598 match(Set dst (OrL src1 src2));
14599
14600 format %{ "orr $dst, $src1, $src2\t# int" %}
14601
14602 ins_cost(INSN_COST);
14603 ins_encode %{
14604 __ orr(as_Register($dst$$reg),
14605 as_Register($src1$$reg),
14606 (uint64_t)($src2$$constant));
14607 %}
14608
14609 ins_pipe(ialu_reg_imm);
14610 %}
14611
14612 // Xor Instructions
14613
14614 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14615 match(Set dst (XorL src1 src2));
14616
14617 format %{ "eor $dst, $src1, $src2\t# int" %}
14618
14619 ins_cost(INSN_COST);
14620 ins_encode %{
14621 __ eor(as_Register($dst$$reg),
14622 as_Register($src1$$reg),
14623 as_Register($src2$$reg));
14624 %}
14625
14626 ins_pipe(ialu_reg_reg);
14627 %}
14628
14629 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14630 match(Set dst (XorL src1 src2));
14631
14632 ins_cost(INSN_COST);
14633 format %{ "eor $dst, $src1, $src2\t# int" %}
14634
14635 ins_encode %{
14636 __ eor(as_Register($dst$$reg),
14637 as_Register($src1$$reg),
14638 (uint64_t)($src2$$constant));
14639 %}
14640
14641 ins_pipe(ialu_reg_imm);
14642 %}
14643
14644 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14645 %{
14646 match(Set dst (ConvI2L src));
14647
14648 ins_cost(INSN_COST);
14649 format %{ "sxtw $dst, $src\t# i2l" %}
14650 ins_encode %{
14651 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14652 %}
14653 ins_pipe(ialu_reg_shift);
14654 %}
14655
14656 // this pattern occurs in bigmath arithmetic
14657 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14658 %{
14659 match(Set dst (AndL (ConvI2L src) mask));
14660
14661 ins_cost(INSN_COST);
14662 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14663 ins_encode %{
14664 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14665 %}
14666
14667 ins_pipe(ialu_reg_shift);
14668 %}
14669
14670 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14671 match(Set dst (ConvL2I src));
14672
14673 ins_cost(INSN_COST);
14674 format %{ "movw $dst, $src \t// l2i" %}
14675
14676 ins_encode %{
14677 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14678 %}
14679
14680 ins_pipe(ialu_reg);
14681 %}
14682
14683 instruct convD2F_reg(vRegF dst, vRegD src) %{
14684 match(Set dst (ConvD2F src));
14685
14686 ins_cost(INSN_COST * 5);
14687 format %{ "fcvtd $dst, $src \t// d2f" %}
14688
14689 ins_encode %{
14690 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14691 %}
14692
14693 ins_pipe(fp_d2f);
14694 %}
14695
14696 instruct convF2D_reg(vRegD dst, vRegF src) %{
14697 match(Set dst (ConvF2D src));
14698
14699 ins_cost(INSN_COST * 5);
14700 format %{ "fcvts $dst, $src \t// f2d" %}
14701
14702 ins_encode %{
14703 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14704 %}
14705
14706 ins_pipe(fp_f2d);
14707 %}
14708
14709 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14710 match(Set dst (ConvF2I src));
14711
14712 ins_cost(INSN_COST * 5);
14713 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14714
14715 ins_encode %{
14716 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14717 %}
14718
14719 ins_pipe(fp_f2i);
14720 %}
14721
14722 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14723 match(Set dst (ConvF2L src));
14724
14725 ins_cost(INSN_COST * 5);
14726 format %{ "fcvtzs $dst, $src \t// f2l" %}
14727
14728 ins_encode %{
14729 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14730 %}
14731
14732 ins_pipe(fp_f2l);
14733 %}
14734
14735 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14736 match(Set dst (ConvF2HF src));
14737 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14738 "smov $dst, $tmp\t# move result from $tmp to $dst"
14739 %}
14740 effect(TEMP tmp);
14741 ins_encode %{
14742 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14743 %}
14744 ins_pipe(pipe_slow);
14745 %}
14746
14747 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14748 match(Set dst (ConvHF2F src));
14749 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14750 "fcvt $dst, $tmp\t# convert half to single precision"
14751 %}
14752 effect(TEMP tmp);
14753 ins_encode %{
14754 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14755 %}
14756 ins_pipe(pipe_slow);
14757 %}
14758
14759 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14760 match(Set dst (ConvI2F src));
14761
14762 ins_cost(INSN_COST * 5);
14763 format %{ "scvtfws $dst, $src \t// i2f" %}
14764
14765 ins_encode %{
14766 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14767 %}
14768
14769 ins_pipe(fp_i2f);
14770 %}
14771
14772 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14773 match(Set dst (ConvL2F src));
14774
14775 ins_cost(INSN_COST * 5);
14776 format %{ "scvtfs $dst, $src \t// l2f" %}
14777
14778 ins_encode %{
14779 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14780 %}
14781
14782 ins_pipe(fp_l2f);
14783 %}
14784
14785 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14786 match(Set dst (ConvD2I src));
14787
14788 ins_cost(INSN_COST * 5);
14789 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14790
14791 ins_encode %{
14792 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14793 %}
14794
14795 ins_pipe(fp_d2i);
14796 %}
14797
14798 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14799 match(Set dst (ConvD2L src));
14800
14801 ins_cost(INSN_COST * 5);
14802 format %{ "fcvtzd $dst, $src \t// d2l" %}
14803
14804 ins_encode %{
14805 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14806 %}
14807
14808 ins_pipe(fp_d2l);
14809 %}
14810
14811 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14812 match(Set dst (ConvI2D src));
14813
14814 ins_cost(INSN_COST * 5);
14815 format %{ "scvtfwd $dst, $src \t// i2d" %}
14816
14817 ins_encode %{
14818 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14819 %}
14820
14821 ins_pipe(fp_i2d);
14822 %}
14823
14824 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14825 match(Set dst (ConvL2D src));
14826
14827 ins_cost(INSN_COST * 5);
14828 format %{ "scvtfd $dst, $src \t// l2d" %}
14829
14830 ins_encode %{
14831 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14832 %}
14833
14834 ins_pipe(fp_l2d);
14835 %}
14836
14837 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14838 %{
14839 match(Set dst (RoundD src));
14840 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14841 format %{ "java_round_double $dst,$src"%}
14842 ins_encode %{
14843 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14844 as_FloatRegister($ftmp$$reg));
14845 %}
14846 ins_pipe(pipe_slow);
14847 %}
14848
14849 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14850 %{
14851 match(Set dst (RoundF src));
14852 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14853 format %{ "java_round_float $dst,$src"%}
14854 ins_encode %{
14855 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14856 as_FloatRegister($ftmp$$reg));
14857 %}
14858 ins_pipe(pipe_slow);
14859 %}
14860
14861 // stack <-> reg and reg <-> reg shuffles with no conversion
14862
14863 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14864
14865 match(Set dst (MoveF2I src));
14866
14867 effect(DEF dst, USE src);
14868
14869 ins_cost(4 * INSN_COST);
14870
14871 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14872
14873 ins_encode %{
14874 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14875 %}
14876
14877 ins_pipe(iload_reg_reg);
14878
14879 %}
14880
14881 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14882
14883 match(Set dst (MoveI2F src));
14884
14885 effect(DEF dst, USE src);
14886
14887 ins_cost(4 * INSN_COST);
14888
14889 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14890
14891 ins_encode %{
14892 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14893 %}
14894
14895 ins_pipe(pipe_class_memory);
14896
14897 %}
14898
14899 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14900
14901 match(Set dst (MoveD2L src));
14902
14903 effect(DEF dst, USE src);
14904
14905 ins_cost(4 * INSN_COST);
14906
14907 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14908
14909 ins_encode %{
14910 __ ldr($dst$$Register, Address(sp, $src$$disp));
14911 %}
14912
14913 ins_pipe(iload_reg_reg);
14914
14915 %}
14916
14917 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14918
14919 match(Set dst (MoveL2D src));
14920
14921 effect(DEF dst, USE src);
14922
14923 ins_cost(4 * INSN_COST);
14924
14925 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14926
14927 ins_encode %{
14928 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14929 %}
14930
14931 ins_pipe(pipe_class_memory);
14932
14933 %}
14934
14935 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14936
14937 match(Set dst (MoveF2I src));
14938
14939 effect(DEF dst, USE src);
14940
14941 ins_cost(INSN_COST);
14942
14943 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14944
14945 ins_encode %{
14946 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14947 %}
14948
14949 ins_pipe(pipe_class_memory);
14950
14951 %}
14952
14953 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14954
14955 match(Set dst (MoveI2F src));
14956
14957 effect(DEF dst, USE src);
14958
14959 ins_cost(INSN_COST);
14960
14961 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14962
14963 ins_encode %{
14964 __ strw($src$$Register, Address(sp, $dst$$disp));
14965 %}
14966
14967 ins_pipe(istore_reg_reg);
14968
14969 %}
14970
14971 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14972
14973 match(Set dst (MoveD2L src));
14974
14975 effect(DEF dst, USE src);
14976
14977 ins_cost(INSN_COST);
14978
14979 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14980
14981 ins_encode %{
14982 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14983 %}
14984
14985 ins_pipe(pipe_class_memory);
14986
14987 %}
14988
14989 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14990
14991 match(Set dst (MoveL2D src));
14992
14993 effect(DEF dst, USE src);
14994
14995 ins_cost(INSN_COST);
14996
14997 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14998
14999 ins_encode %{
15000 __ str($src$$Register, Address(sp, $dst$$disp));
15001 %}
15002
15003 ins_pipe(istore_reg_reg);
15004
15005 %}
15006
15007 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15008
15009 match(Set dst (MoveF2I src));
15010
15011 effect(DEF dst, USE src);
15012
15013 ins_cost(INSN_COST);
15014
15015 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15016
15017 ins_encode %{
15018 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15019 %}
15020
15021 ins_pipe(fp_f2i);
15022
15023 %}
15024
15025 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15026
15027 match(Set dst (MoveI2F src));
15028
15029 effect(DEF dst, USE src);
15030
15031 ins_cost(INSN_COST);
15032
15033 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15034
15035 ins_encode %{
15036 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15037 %}
15038
15039 ins_pipe(fp_i2f);
15040
15041 %}
15042
15043 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15044
15045 match(Set dst (MoveD2L src));
15046
15047 effect(DEF dst, USE src);
15048
15049 ins_cost(INSN_COST);
15050
15051 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15052
15053 ins_encode %{
15054 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15055 %}
15056
15057 ins_pipe(fp_d2l);
15058
15059 %}
15060
15061 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15062
15063 match(Set dst (MoveL2D src));
15064
15065 effect(DEF dst, USE src);
15066
15067 ins_cost(INSN_COST);
15068
15069 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15070
15071 ins_encode %{
15072 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15073 %}
15074
15075 ins_pipe(fp_l2d);
15076
15077 %}
15078
15079 // ============================================================================
15080 // clearing of an array
15081
15082 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15083 %{
15084 match(Set dummy (ClearArray cnt base));
15085 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15086
15087 ins_cost(4 * INSN_COST);
15088 format %{ "ClearArray $cnt, $base" %}
15089
15090 ins_encode %{
15091 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15092 if (tpc == nullptr) {
15093 ciEnv::current()->record_failure("CodeCache is full");
15094 return;
15095 }
15096 %}
15097
15098 ins_pipe(pipe_class_memory);
15099 %}
15100
15101 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15102 %{
15103 predicate((uint64_t)n->in(2)->get_long()
15104 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15105 match(Set dummy (ClearArray cnt base));
15106 effect(TEMP temp, USE_KILL base, KILL cr);
15107
15108 ins_cost(4 * INSN_COST);
15109 format %{ "ClearArray $cnt, $base" %}
15110
15111 ins_encode %{
15112 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15113 if (tpc == nullptr) {
15114 ciEnv::current()->record_failure("CodeCache is full");
15115 return;
15116 }
15117 %}
15118
15119 ins_pipe(pipe_class_memory);
15120 %}
15121
15122 // ============================================================================
15123 // Overflow Math Instructions
15124
15125 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15126 %{
15127 match(Set cr (OverflowAddI op1 op2));
15128
15129 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15130 ins_cost(INSN_COST);
15131 ins_encode %{
15132 __ cmnw($op1$$Register, $op2$$Register);
15133 %}
15134
15135 ins_pipe(icmp_reg_reg);
15136 %}
15137
15138 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15139 %{
15140 match(Set cr (OverflowAddI op1 op2));
15141
15142 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15143 ins_cost(INSN_COST);
15144 ins_encode %{
15145 __ cmnw($op1$$Register, $op2$$constant);
15146 %}
15147
15148 ins_pipe(icmp_reg_imm);
15149 %}
15150
15151 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15152 %{
15153 match(Set cr (OverflowAddL op1 op2));
15154
15155 format %{ "cmn $op1, $op2\t# overflow check long" %}
15156 ins_cost(INSN_COST);
15157 ins_encode %{
15158 __ cmn($op1$$Register, $op2$$Register);
15159 %}
15160
15161 ins_pipe(icmp_reg_reg);
15162 %}
15163
15164 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15165 %{
15166 match(Set cr (OverflowAddL op1 op2));
15167
15168 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15169 ins_cost(INSN_COST);
15170 ins_encode %{
15171 __ adds(zr, $op1$$Register, $op2$$constant);
15172 %}
15173
15174 ins_pipe(icmp_reg_imm);
15175 %}
15176
15177 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15178 %{
15179 match(Set cr (OverflowSubI op1 op2));
15180
15181 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15182 ins_cost(INSN_COST);
15183 ins_encode %{
15184 __ cmpw($op1$$Register, $op2$$Register);
15185 %}
15186
15187 ins_pipe(icmp_reg_reg);
15188 %}
15189
15190 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15191 %{
15192 match(Set cr (OverflowSubI op1 op2));
15193
15194 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15195 ins_cost(INSN_COST);
15196 ins_encode %{
15197 __ cmpw($op1$$Register, $op2$$constant);
15198 %}
15199
15200 ins_pipe(icmp_reg_imm);
15201 %}
15202
15203 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15204 %{
15205 match(Set cr (OverflowSubL op1 op2));
15206
15207 format %{ "cmp $op1, $op2\t# overflow check long" %}
15208 ins_cost(INSN_COST);
15209 ins_encode %{
15210 __ cmp($op1$$Register, $op2$$Register);
15211 %}
15212
15213 ins_pipe(icmp_reg_reg);
15214 %}
15215
15216 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15217 %{
15218 match(Set cr (OverflowSubL op1 op2));
15219
15220 format %{ "cmp $op1, $op2\t# overflow check long" %}
15221 ins_cost(INSN_COST);
15222 ins_encode %{
15223 __ subs(zr, $op1$$Register, $op2$$constant);
15224 %}
15225
15226 ins_pipe(icmp_reg_imm);
15227 %}
15228
15229 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15230 %{
15231 match(Set cr (OverflowSubI zero op1));
15232
15233 format %{ "cmpw zr, $op1\t# overflow check int" %}
15234 ins_cost(INSN_COST);
15235 ins_encode %{
15236 __ cmpw(zr, $op1$$Register);
15237 %}
15238
15239 ins_pipe(icmp_reg_imm);
15240 %}
15241
15242 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15243 %{
15244 match(Set cr (OverflowSubL zero op1));
15245
15246 format %{ "cmp zr, $op1\t# overflow check long" %}
15247 ins_cost(INSN_COST);
15248 ins_encode %{
15249 __ cmp(zr, $op1$$Register);
15250 %}
15251
15252 ins_pipe(icmp_reg_imm);
15253 %}
15254
15255 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15256 %{
15257 match(Set cr (OverflowMulI op1 op2));
15258
15259 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15260 "cmp rscratch1, rscratch1, sxtw\n\t"
15261 "movw rscratch1, #0x80000000\n\t"
15262 "cselw rscratch1, rscratch1, zr, NE\n\t"
15263 "cmpw rscratch1, #1" %}
15264 ins_cost(5 * INSN_COST);
15265 ins_encode %{
15266 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15267 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15268 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15269 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15270 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15271 %}
15272
15273 ins_pipe(pipe_slow);
15274 %}
15275
15276 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15277 %{
15278 match(If cmp (OverflowMulI op1 op2));
15279 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15280 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15281 effect(USE labl, KILL cr);
15282
15283 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15284 "cmp rscratch1, rscratch1, sxtw\n\t"
15285 "b$cmp $labl" %}
15286 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15287 ins_encode %{
15288 Label* L = $labl$$label;
15289 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15290 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15291 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15292 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15293 %}
15294
15295 ins_pipe(pipe_serial);
15296 %}
15297
15298 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15299 %{
15300 match(Set cr (OverflowMulL op1 op2));
15301
15302 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15303 "smulh rscratch2, $op1, $op2\n\t"
15304 "cmp rscratch2, rscratch1, ASR #63\n\t"
15305 "movw rscratch1, #0x80000000\n\t"
15306 "cselw rscratch1, rscratch1, zr, NE\n\t"
15307 "cmpw rscratch1, #1" %}
15308 ins_cost(6 * INSN_COST);
15309 ins_encode %{
15310 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15311 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15312 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15313 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15314 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15315 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15316 %}
15317
15318 ins_pipe(pipe_slow);
15319 %}
15320
15321 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15322 %{
15323 match(If cmp (OverflowMulL op1 op2));
15324 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15325 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15326 effect(USE labl, KILL cr);
15327
15328 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15329 "smulh rscratch2, $op1, $op2\n\t"
15330 "cmp rscratch2, rscratch1, ASR #63\n\t"
15331 "b$cmp $labl" %}
15332 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15333 ins_encode %{
15334 Label* L = $labl$$label;
15335 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15336 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15337 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15338 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15339 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15340 %}
15341
15342 ins_pipe(pipe_serial);
15343 %}
15344
15345 // ============================================================================
15346 // Compare Instructions
15347
15348 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15349 %{
15350 match(Set cr (CmpI op1 op2));
15351
15352 effect(DEF cr, USE op1, USE op2);
15353
15354 ins_cost(INSN_COST);
15355 format %{ "cmpw $op1, $op2" %}
15356
15357 ins_encode(aarch64_enc_cmpw(op1, op2));
15358
15359 ins_pipe(icmp_reg_reg);
15360 %}
15361
15362 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15363 %{
15364 match(Set cr (CmpI op1 zero));
15365
15366 effect(DEF cr, USE op1);
15367
15368 ins_cost(INSN_COST);
15369 format %{ "cmpw $op1, 0" %}
15370
15371 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15372
15373 ins_pipe(icmp_reg_imm);
15374 %}
15375
15376 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15377 %{
15378 match(Set cr (CmpI op1 op2));
15379
15380 effect(DEF cr, USE op1);
15381
15382 ins_cost(INSN_COST);
15383 format %{ "cmpw $op1, $op2" %}
15384
15385 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15386
15387 ins_pipe(icmp_reg_imm);
15388 %}
15389
15390 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15391 %{
15392 match(Set cr (CmpI op1 op2));
15393
15394 effect(DEF cr, USE op1);
15395
15396 ins_cost(INSN_COST * 2);
15397 format %{ "cmpw $op1, $op2" %}
15398
15399 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15400
15401 ins_pipe(icmp_reg_imm);
15402 %}
15403
15404 // Unsigned compare Instructions; really, same as signed compare
15405 // except it should only be used to feed an If or a CMovI which takes a
15406 // cmpOpU.
15407
15408 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15409 %{
15410 match(Set cr (CmpU op1 op2));
15411
15412 effect(DEF cr, USE op1, USE op2);
15413
15414 ins_cost(INSN_COST);
15415 format %{ "cmpw $op1, $op2\t# unsigned" %}
15416
15417 ins_encode(aarch64_enc_cmpw(op1, op2));
15418
15419 ins_pipe(icmp_reg_reg);
15420 %}
15421
15422 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15423 %{
15424 match(Set cr (CmpU op1 zero));
15425
15426 effect(DEF cr, USE op1);
15427
15428 ins_cost(INSN_COST);
15429 format %{ "cmpw $op1, #0\t# unsigned" %}
15430
15431 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15432
15433 ins_pipe(icmp_reg_imm);
15434 %}
15435
15436 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15437 %{
15438 match(Set cr (CmpU op1 op2));
15439
15440 effect(DEF cr, USE op1);
15441
15442 ins_cost(INSN_COST);
15443 format %{ "cmpw $op1, $op2\t# unsigned" %}
15444
15445 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15446
15447 ins_pipe(icmp_reg_imm);
15448 %}
15449
15450 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15451 %{
15452 match(Set cr (CmpU op1 op2));
15453
15454 effect(DEF cr, USE op1);
15455
15456 ins_cost(INSN_COST * 2);
15457 format %{ "cmpw $op1, $op2\t# unsigned" %}
15458
15459 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15460
15461 ins_pipe(icmp_reg_imm);
15462 %}
15463
15464 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15465 %{
15466 match(Set cr (CmpL op1 op2));
15467
15468 effect(DEF cr, USE op1, USE op2);
15469
15470 ins_cost(INSN_COST);
15471 format %{ "cmp $op1, $op2" %}
15472
15473 ins_encode(aarch64_enc_cmp(op1, op2));
15474
15475 ins_pipe(icmp_reg_reg);
15476 %}
15477
15478 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15479 %{
15480 match(Set cr (CmpL op1 zero));
15481
15482 effect(DEF cr, USE op1);
15483
15484 ins_cost(INSN_COST);
15485 format %{ "tst $op1" %}
15486
15487 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15488
15489 ins_pipe(icmp_reg_imm);
15490 %}
15491
15492 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15493 %{
15494 match(Set cr (CmpL op1 op2));
15495
15496 effect(DEF cr, USE op1);
15497
15498 ins_cost(INSN_COST);
15499 format %{ "cmp $op1, $op2" %}
15500
15501 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15502
15503 ins_pipe(icmp_reg_imm);
15504 %}
15505
15506 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15507 %{
15508 match(Set cr (CmpL op1 op2));
15509
15510 effect(DEF cr, USE op1);
15511
15512 ins_cost(INSN_COST * 2);
15513 format %{ "cmp $op1, $op2" %}
15514
15515 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15516
15517 ins_pipe(icmp_reg_imm);
15518 %}
15519
15520 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15521 %{
15522 match(Set cr (CmpUL op1 op2));
15523
15524 effect(DEF cr, USE op1, USE op2);
15525
15526 ins_cost(INSN_COST);
15527 format %{ "cmp $op1, $op2" %}
15528
15529 ins_encode(aarch64_enc_cmp(op1, op2));
15530
15531 ins_pipe(icmp_reg_reg);
15532 %}
15533
15534 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15535 %{
15536 match(Set cr (CmpUL op1 zero));
15537
15538 effect(DEF cr, USE op1);
15539
15540 ins_cost(INSN_COST);
15541 format %{ "tst $op1" %}
15542
15543 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15544
15545 ins_pipe(icmp_reg_imm);
15546 %}
15547
15548 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15549 %{
15550 match(Set cr (CmpUL op1 op2));
15551
15552 effect(DEF cr, USE op1);
15553
15554 ins_cost(INSN_COST);
15555 format %{ "cmp $op1, $op2" %}
15556
15557 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15558
15559 ins_pipe(icmp_reg_imm);
15560 %}
15561
15562 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15563 %{
15564 match(Set cr (CmpUL op1 op2));
15565
15566 effect(DEF cr, USE op1);
15567
15568 ins_cost(INSN_COST * 2);
15569 format %{ "cmp $op1, $op2" %}
15570
15571 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15572
15573 ins_pipe(icmp_reg_imm);
15574 %}
15575
15576 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15577 %{
15578 match(Set cr (CmpP op1 op2));
15579
15580 effect(DEF cr, USE op1, USE op2);
15581
15582 ins_cost(INSN_COST);
15583 format %{ "cmp $op1, $op2\t // ptr" %}
15584
15585 ins_encode(aarch64_enc_cmpp(op1, op2));
15586
15587 ins_pipe(icmp_reg_reg);
15588 %}
15589
15590 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15591 %{
15592 match(Set cr (CmpN op1 op2));
15593
15594 effect(DEF cr, USE op1, USE op2);
15595
15596 ins_cost(INSN_COST);
15597 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15598
15599 ins_encode(aarch64_enc_cmpn(op1, op2));
15600
15601 ins_pipe(icmp_reg_reg);
15602 %}
15603
15604 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15605 %{
15606 match(Set cr (CmpP op1 zero));
15607
15608 effect(DEF cr, USE op1, USE zero);
15609
15610 ins_cost(INSN_COST);
15611 format %{ "cmp $op1, 0\t // ptr" %}
15612
15613 ins_encode(aarch64_enc_testp(op1));
15614
15615 ins_pipe(icmp_reg_imm);
15616 %}
15617
15618 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15619 %{
15620 match(Set cr (CmpN op1 zero));
15621
15622 effect(DEF cr, USE op1, USE zero);
15623
15624 ins_cost(INSN_COST);
15625 format %{ "cmp $op1, 0\t // compressed ptr" %}
15626
15627 ins_encode(aarch64_enc_testn(op1));
15628
15629 ins_pipe(icmp_reg_imm);
15630 %}
15631
15632 // FP comparisons
15633 //
15634 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15635 // using normal cmpOp. See declaration of rFlagsReg for details.
15636
15637 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15638 %{
15639 match(Set cr (CmpF src1 src2));
15640
15641 ins_cost(3 * INSN_COST);
15642 format %{ "fcmps $src1, $src2" %}
15643
15644 ins_encode %{
15645 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15646 %}
15647
15648 ins_pipe(pipe_class_compare);
15649 %}
15650
15651 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15652 %{
15653 match(Set cr (CmpF src1 src2));
15654
15655 ins_cost(3 * INSN_COST);
15656 format %{ "fcmps $src1, 0.0" %}
15657
15658 ins_encode %{
15659 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15660 %}
15661
15662 ins_pipe(pipe_class_compare);
15663 %}
15664 // FROM HERE
15665
15666 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15667 %{
15668 match(Set cr (CmpD src1 src2));
15669
15670 ins_cost(3 * INSN_COST);
15671 format %{ "fcmpd $src1, $src2" %}
15672
15673 ins_encode %{
15674 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15675 %}
15676
15677 ins_pipe(pipe_class_compare);
15678 %}
15679
15680 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15681 %{
15682 match(Set cr (CmpD src1 src2));
15683
15684 ins_cost(3 * INSN_COST);
15685 format %{ "fcmpd $src1, 0.0" %}
15686
15687 ins_encode %{
15688 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15689 %}
15690
15691 ins_pipe(pipe_class_compare);
15692 %}
15693
15694 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15695 %{
15696 match(Set dst (CmpF3 src1 src2));
15697 effect(KILL cr);
15698
15699 ins_cost(5 * INSN_COST);
15700 format %{ "fcmps $src1, $src2\n\t"
15701 "csinvw($dst, zr, zr, eq\n\t"
15702 "csnegw($dst, $dst, $dst, lt)"
15703 %}
15704
15705 ins_encode %{
15706 Label done;
15707 FloatRegister s1 = as_FloatRegister($src1$$reg);
15708 FloatRegister s2 = as_FloatRegister($src2$$reg);
15709 Register d = as_Register($dst$$reg);
15710 __ fcmps(s1, s2);
15711 // installs 0 if EQ else -1
15712 __ csinvw(d, zr, zr, Assembler::EQ);
15713 // keeps -1 if less or unordered else installs 1
15714 __ csnegw(d, d, d, Assembler::LT);
15715 __ bind(done);
15716 %}
15717
15718 ins_pipe(pipe_class_default);
15719
15720 %}
15721
15722 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15723 %{
15724 match(Set dst (CmpD3 src1 src2));
15725 effect(KILL cr);
15726
15727 ins_cost(5 * INSN_COST);
15728 format %{ "fcmpd $src1, $src2\n\t"
15729 "csinvw($dst, zr, zr, eq\n\t"
15730 "csnegw($dst, $dst, $dst, lt)"
15731 %}
15732
15733 ins_encode %{
15734 Label done;
15735 FloatRegister s1 = as_FloatRegister($src1$$reg);
15736 FloatRegister s2 = as_FloatRegister($src2$$reg);
15737 Register d = as_Register($dst$$reg);
15738 __ fcmpd(s1, s2);
15739 // installs 0 if EQ else -1
15740 __ csinvw(d, zr, zr, Assembler::EQ);
15741 // keeps -1 if less or unordered else installs 1
15742 __ csnegw(d, d, d, Assembler::LT);
15743 __ bind(done);
15744 %}
15745 ins_pipe(pipe_class_default);
15746
15747 %}
15748
15749 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15750 %{
15751 match(Set dst (CmpF3 src1 zero));
15752 effect(KILL cr);
15753
15754 ins_cost(5 * INSN_COST);
15755 format %{ "fcmps $src1, 0.0\n\t"
15756 "csinvw($dst, zr, zr, eq\n\t"
15757 "csnegw($dst, $dst, $dst, lt)"
15758 %}
15759
15760 ins_encode %{
15761 Label done;
15762 FloatRegister s1 = as_FloatRegister($src1$$reg);
15763 Register d = as_Register($dst$$reg);
15764 __ fcmps(s1, 0.0);
15765 // installs 0 if EQ else -1
15766 __ csinvw(d, zr, zr, Assembler::EQ);
15767 // keeps -1 if less or unordered else installs 1
15768 __ csnegw(d, d, d, Assembler::LT);
15769 __ bind(done);
15770 %}
15771
15772 ins_pipe(pipe_class_default);
15773
15774 %}
15775
15776 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15777 %{
15778 match(Set dst (CmpD3 src1 zero));
15779 effect(KILL cr);
15780
15781 ins_cost(5 * INSN_COST);
15782 format %{ "fcmpd $src1, 0.0\n\t"
15783 "csinvw($dst, zr, zr, eq\n\t"
15784 "csnegw($dst, $dst, $dst, lt)"
15785 %}
15786
15787 ins_encode %{
15788 Label done;
15789 FloatRegister s1 = as_FloatRegister($src1$$reg);
15790 Register d = as_Register($dst$$reg);
15791 __ fcmpd(s1, 0.0);
15792 // installs 0 if EQ else -1
15793 __ csinvw(d, zr, zr, Assembler::EQ);
15794 // keeps -1 if less or unordered else installs 1
15795 __ csnegw(d, d, d, Assembler::LT);
15796 __ bind(done);
15797 %}
15798 ins_pipe(pipe_class_default);
15799
15800 %}
15801
15802 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15803 %{
15804 match(Set dst (CmpLTMask p q));
15805 effect(KILL cr);
15806
15807 ins_cost(3 * INSN_COST);
15808
15809 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15810 "csetw $dst, lt\n\t"
15811 "subw $dst, zr, $dst"
15812 %}
15813
15814 ins_encode %{
15815 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15816 __ csetw(as_Register($dst$$reg), Assembler::LT);
15817 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15818 %}
15819
15820 ins_pipe(ialu_reg_reg);
15821 %}
15822
15823 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15824 %{
15825 match(Set dst (CmpLTMask src zero));
15826 effect(KILL cr);
15827
15828 ins_cost(INSN_COST);
15829
15830 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15831
15832 ins_encode %{
15833 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15834 %}
15835
15836 ins_pipe(ialu_reg_shift);
15837 %}
15838
15839 // ============================================================================
15840 // Max and Min
15841
15842 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15843
15844 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15845 %{
15846 effect(DEF cr, USE src);
15847 ins_cost(INSN_COST);
15848 format %{ "cmpw $src, 0" %}
15849
15850 ins_encode %{
15851 __ cmpw($src$$Register, 0);
15852 %}
15853 ins_pipe(icmp_reg_imm);
15854 %}
15855
15856 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15857 %{
15858 match(Set dst (MinI src1 src2));
15859 ins_cost(INSN_COST * 3);
15860
15861 expand %{
15862 rFlagsReg cr;
15863 compI_reg_reg(cr, src1, src2);
15864 cmovI_reg_reg_lt(dst, src1, src2, cr);
15865 %}
15866 %}
15867
15868 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15869 %{
15870 match(Set dst (MaxI src1 src2));
15871 ins_cost(INSN_COST * 3);
15872
15873 expand %{
15874 rFlagsReg cr;
15875 compI_reg_reg(cr, src1, src2);
15876 cmovI_reg_reg_gt(dst, src1, src2, cr);
15877 %}
15878 %}
15879
15880
15881 // ============================================================================
15882 // Branch Instructions
15883
15884 // Direct Branch.
15885 instruct branch(label lbl)
15886 %{
15887 match(Goto);
15888
15889 effect(USE lbl);
15890
15891 ins_cost(BRANCH_COST);
15892 format %{ "b $lbl" %}
15893
15894 ins_encode(aarch64_enc_b(lbl));
15895
15896 ins_pipe(pipe_branch);
15897 %}
15898
15899 // Conditional Near Branch
15900 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15901 %{
15902 // Same match rule as `branchConFar'.
15903 match(If cmp cr);
15904
15905 effect(USE lbl);
15906
15907 ins_cost(BRANCH_COST);
15908 // If set to 1 this indicates that the current instruction is a
15909 // short variant of a long branch. This avoids using this
15910 // instruction in first-pass matching. It will then only be used in
15911 // the `Shorten_branches' pass.
15912 // ins_short_branch(1);
15913 format %{ "b$cmp $lbl" %}
15914
15915 ins_encode(aarch64_enc_br_con(cmp, lbl));
15916
15917 ins_pipe(pipe_branch_cond);
15918 %}
15919
15920 // Conditional Near Branch Unsigned
15921 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15922 %{
15923 // Same match rule as `branchConFar'.
15924 match(If cmp cr);
15925
15926 effect(USE lbl);
15927
15928 ins_cost(BRANCH_COST);
15929 // If set to 1 this indicates that the current instruction is a
15930 // short variant of a long branch. This avoids using this
15931 // instruction in first-pass matching. It will then only be used in
15932 // the `Shorten_branches' pass.
15933 // ins_short_branch(1);
15934 format %{ "b$cmp $lbl\t# unsigned" %}
15935
15936 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15937
15938 ins_pipe(pipe_branch_cond);
15939 %}
15940
15941 // Make use of CBZ and CBNZ. These instructions, as well as being
15942 // shorter than (cmp; branch), have the additional benefit of not
15943 // killing the flags.
15944
15945 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15946 match(If cmp (CmpI op1 op2));
15947 effect(USE labl);
15948
15949 ins_cost(BRANCH_COST);
15950 format %{ "cbw$cmp $op1, $labl" %}
15951 ins_encode %{
15952 Label* L = $labl$$label;
15953 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15954 if (cond == Assembler::EQ)
15955 __ cbzw($op1$$Register, *L);
15956 else
15957 __ cbnzw($op1$$Register, *L);
15958 %}
15959 ins_pipe(pipe_cmp_branch);
15960 %}
15961
15962 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15963 match(If cmp (CmpL op1 op2));
15964 effect(USE labl);
15965
15966 ins_cost(BRANCH_COST);
15967 format %{ "cb$cmp $op1, $labl" %}
15968 ins_encode %{
15969 Label* L = $labl$$label;
15970 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15971 if (cond == Assembler::EQ)
15972 __ cbz($op1$$Register, *L);
15973 else
15974 __ cbnz($op1$$Register, *L);
15975 %}
15976 ins_pipe(pipe_cmp_branch);
15977 %}
15978
15979 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15980 match(If cmp (CmpP op1 op2));
15981 effect(USE labl);
15982
15983 ins_cost(BRANCH_COST);
15984 format %{ "cb$cmp $op1, $labl" %}
15985 ins_encode %{
15986 Label* L = $labl$$label;
15987 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15988 if (cond == Assembler::EQ)
15989 __ cbz($op1$$Register, *L);
15990 else
15991 __ cbnz($op1$$Register, *L);
15992 %}
15993 ins_pipe(pipe_cmp_branch);
15994 %}
15995
15996 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15997 match(If cmp (CmpN op1 op2));
15998 effect(USE labl);
15999
16000 ins_cost(BRANCH_COST);
16001 format %{ "cbw$cmp $op1, $labl" %}
16002 ins_encode %{
16003 Label* L = $labl$$label;
16004 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16005 if (cond == Assembler::EQ)
16006 __ cbzw($op1$$Register, *L);
16007 else
16008 __ cbnzw($op1$$Register, *L);
16009 %}
16010 ins_pipe(pipe_cmp_branch);
16011 %}
16012
16013 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16014 match(If cmp (CmpP (DecodeN oop) zero));
16015 effect(USE labl);
16016
16017 ins_cost(BRANCH_COST);
16018 format %{ "cb$cmp $oop, $labl" %}
16019 ins_encode %{
16020 Label* L = $labl$$label;
16021 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16022 if (cond == Assembler::EQ)
16023 __ cbzw($oop$$Register, *L);
16024 else
16025 __ cbnzw($oop$$Register, *L);
16026 %}
16027 ins_pipe(pipe_cmp_branch);
16028 %}
16029
16030 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16031 match(If cmp (CmpU op1 op2));
16032 effect(USE labl);
16033
16034 ins_cost(BRANCH_COST);
16035 format %{ "cbw$cmp $op1, $labl" %}
16036 ins_encode %{
16037 Label* L = $labl$$label;
16038 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16039 if (cond == Assembler::EQ || cond == Assembler::LS) {
16040 __ cbzw($op1$$Register, *L);
16041 } else {
16042 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16043 __ cbnzw($op1$$Register, *L);
16044 }
16045 %}
16046 ins_pipe(pipe_cmp_branch);
16047 %}
16048
16049 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16050 match(If cmp (CmpUL op1 op2));
16051 effect(USE labl);
16052
16053 ins_cost(BRANCH_COST);
16054 format %{ "cb$cmp $op1, $labl" %}
16055 ins_encode %{
16056 Label* L = $labl$$label;
16057 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16058 if (cond == Assembler::EQ || cond == Assembler::LS) {
16059 __ cbz($op1$$Register, *L);
16060 } else {
16061 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16062 __ cbnz($op1$$Register, *L);
16063 }
16064 %}
16065 ins_pipe(pipe_cmp_branch);
16066 %}
16067
16068 // Test bit and Branch
16069
16070 // Patterns for short (< 32KiB) variants
16071 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16072 match(If cmp (CmpL op1 op2));
16073 effect(USE labl);
16074
16075 ins_cost(BRANCH_COST);
16076 format %{ "cb$cmp $op1, $labl # long" %}
16077 ins_encode %{
16078 Label* L = $labl$$label;
16079 Assembler::Condition cond =
16080 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16081 __ tbr(cond, $op1$$Register, 63, *L);
16082 %}
16083 ins_pipe(pipe_cmp_branch);
16084 ins_short_branch(1);
16085 %}
16086
16087 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16088 match(If cmp (CmpI op1 op2));
16089 effect(USE labl);
16090
16091 ins_cost(BRANCH_COST);
16092 format %{ "cb$cmp $op1, $labl # int" %}
16093 ins_encode %{
16094 Label* L = $labl$$label;
16095 Assembler::Condition cond =
16096 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16097 __ tbr(cond, $op1$$Register, 31, *L);
16098 %}
16099 ins_pipe(pipe_cmp_branch);
16100 ins_short_branch(1);
16101 %}
16102
16103 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16104 match(If cmp (CmpL (AndL op1 op2) op3));
16105 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16106 effect(USE labl);
16107
16108 ins_cost(BRANCH_COST);
16109 format %{ "tb$cmp $op1, $op2, $labl" %}
16110 ins_encode %{
16111 Label* L = $labl$$label;
16112 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16113 int bit = exact_log2_long($op2$$constant);
16114 __ tbr(cond, $op1$$Register, bit, *L);
16115 %}
16116 ins_pipe(pipe_cmp_branch);
16117 ins_short_branch(1);
16118 %}
16119
16120 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16121 match(If cmp (CmpI (AndI op1 op2) op3));
16122 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16123 effect(USE labl);
16124
16125 ins_cost(BRANCH_COST);
16126 format %{ "tb$cmp $op1, $op2, $labl" %}
16127 ins_encode %{
16128 Label* L = $labl$$label;
16129 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16130 int bit = exact_log2((juint)$op2$$constant);
16131 __ tbr(cond, $op1$$Register, bit, *L);
16132 %}
16133 ins_pipe(pipe_cmp_branch);
16134 ins_short_branch(1);
16135 %}
16136
16137 // And far variants
16138 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16139 match(If cmp (CmpL op1 op2));
16140 effect(USE labl);
16141
16142 ins_cost(BRANCH_COST);
16143 format %{ "cb$cmp $op1, $labl # long" %}
16144 ins_encode %{
16145 Label* L = $labl$$label;
16146 Assembler::Condition cond =
16147 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16148 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16149 %}
16150 ins_pipe(pipe_cmp_branch);
16151 %}
16152
16153 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16154 match(If cmp (CmpI op1 op2));
16155 effect(USE labl);
16156
16157 ins_cost(BRANCH_COST);
16158 format %{ "cb$cmp $op1, $labl # int" %}
16159 ins_encode %{
16160 Label* L = $labl$$label;
16161 Assembler::Condition cond =
16162 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16163 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16164 %}
16165 ins_pipe(pipe_cmp_branch);
16166 %}
16167
16168 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16169 match(If cmp (CmpL (AndL op1 op2) op3));
16170 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16171 effect(USE labl);
16172
16173 ins_cost(BRANCH_COST);
16174 format %{ "tb$cmp $op1, $op2, $labl" %}
16175 ins_encode %{
16176 Label* L = $labl$$label;
16177 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16178 int bit = exact_log2_long($op2$$constant);
16179 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16180 %}
16181 ins_pipe(pipe_cmp_branch);
16182 %}
16183
16184 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16185 match(If cmp (CmpI (AndI op1 op2) op3));
16186 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16187 effect(USE labl);
16188
16189 ins_cost(BRANCH_COST);
16190 format %{ "tb$cmp $op1, $op2, $labl" %}
16191 ins_encode %{
16192 Label* L = $labl$$label;
16193 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16194 int bit = exact_log2((juint)$op2$$constant);
16195 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16196 %}
16197 ins_pipe(pipe_cmp_branch);
16198 %}
16199
16200 // Test bits
16201
16202 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16203 match(Set cr (CmpL (AndL op1 op2) op3));
16204 predicate(Assembler::operand_valid_for_logical_immediate
16205 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16206
16207 ins_cost(INSN_COST);
16208 format %{ "tst $op1, $op2 # long" %}
16209 ins_encode %{
16210 __ tst($op1$$Register, $op2$$constant);
16211 %}
16212 ins_pipe(ialu_reg_reg);
16213 %}
16214
16215 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16216 match(Set cr (CmpI (AndI op1 op2) op3));
16217 predicate(Assembler::operand_valid_for_logical_immediate
16218 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16219
16220 ins_cost(INSN_COST);
16221 format %{ "tst $op1, $op2 # int" %}
16222 ins_encode %{
16223 __ tstw($op1$$Register, $op2$$constant);
16224 %}
16225 ins_pipe(ialu_reg_reg);
16226 %}
16227
16228 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16229 match(Set cr (CmpL (AndL op1 op2) op3));
16230
16231 ins_cost(INSN_COST);
16232 format %{ "tst $op1, $op2 # long" %}
16233 ins_encode %{
16234 __ tst($op1$$Register, $op2$$Register);
16235 %}
16236 ins_pipe(ialu_reg_reg);
16237 %}
16238
16239 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16240 match(Set cr (CmpI (AndI op1 op2) op3));
16241
16242 ins_cost(INSN_COST);
16243 format %{ "tstw $op1, $op2 # int" %}
16244 ins_encode %{
16245 __ tstw($op1$$Register, $op2$$Register);
16246 %}
16247 ins_pipe(ialu_reg_reg);
16248 %}
16249
16250
16251 // Conditional Far Branch
16252 // Conditional Far Branch Unsigned
16253 // TODO: fixme
16254
16255 // counted loop end branch near
16256 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16257 %{
16258 match(CountedLoopEnd cmp cr);
16259
16260 effect(USE lbl);
16261
16262 ins_cost(BRANCH_COST);
16263 // short variant.
16264 // ins_short_branch(1);
16265 format %{ "b$cmp $lbl \t// counted loop end" %}
16266
16267 ins_encode(aarch64_enc_br_con(cmp, lbl));
16268
16269 ins_pipe(pipe_branch);
16270 %}
16271
16272 // counted loop end branch far
16273 // TODO: fixme
16274
16275 // ============================================================================
16276 // inlined locking and unlocking
16277
16278 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16279 %{
16280 match(Set cr (FastLock object box));
16281 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16282
16283 ins_cost(5 * INSN_COST);
16284 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16285
16286 ins_encode %{
16287 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16288 %}
16289
16290 ins_pipe(pipe_serial);
16291 %}
16292
16293 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16294 %{
16295 match(Set cr (FastUnlock object box));
16296 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16297
16298 ins_cost(5 * INSN_COST);
16299 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16300
16301 ins_encode %{
16302 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16303 %}
16304
16305 ins_pipe(pipe_serial);
16306 %}
16307
16308 // ============================================================================
16309 // Safepoint Instructions
16310
16311 // TODO
16312 // provide a near and far version of this code
16313
16314 instruct safePoint(rFlagsReg cr, iRegP poll)
16315 %{
16316 match(SafePoint poll);
16317 effect(KILL cr);
16318
16319 format %{
16320 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16321 %}
16322 ins_encode %{
16323 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16324 %}
16325 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16326 %}
16327
16328
16329 // ============================================================================
16330 // Procedure Call/Return Instructions
16331
16332 // Call Java Static Instruction
16333
16334 instruct CallStaticJavaDirect(method meth)
16335 %{
16336 match(CallStaticJava);
16337
16338 effect(USE meth);
16339
16340 ins_cost(CALL_COST);
16341
16342 format %{ "call,static $meth \t// ==> " %}
16343
16344 ins_encode(aarch64_enc_java_static_call(meth),
16345 aarch64_enc_call_epilog);
16346
16347 ins_pipe(pipe_class_call);
16348 %}
16349
16350 // TO HERE
16351
16352 // Call Java Dynamic Instruction
16353 instruct CallDynamicJavaDirect(method meth)
16354 %{
16355 match(CallDynamicJava);
16356
16357 effect(USE meth);
16358
16359 ins_cost(CALL_COST);
16360
16361 format %{ "CALL,dynamic $meth \t// ==> " %}
16362
16363 ins_encode(aarch64_enc_java_dynamic_call(meth),
16364 aarch64_enc_call_epilog);
16365
16366 ins_pipe(pipe_class_call);
16367 %}
16368
16369 // Call Runtime Instruction
16370
16371 instruct CallRuntimeDirect(method meth)
16372 %{
16373 match(CallRuntime);
16374
16375 effect(USE meth);
16376
16377 ins_cost(CALL_COST);
16378
16379 format %{ "CALL, runtime $meth" %}
16380
16381 ins_encode( aarch64_enc_java_to_runtime(meth) );
16382
16383 ins_pipe(pipe_class_call);
16384 %}
16385
16386 // Call Runtime Instruction
16387
16388 instruct CallLeafDirect(method meth)
16389 %{
16390 match(CallLeaf);
16391
16392 effect(USE meth);
16393
16394 ins_cost(CALL_COST);
16395
16396 format %{ "CALL, runtime leaf $meth" %}
16397
16398 ins_encode( aarch64_enc_java_to_runtime(meth) );
16399
16400 ins_pipe(pipe_class_call);
16401 %}
16402
16403 // Call Runtime Instruction without safepoint and with vector arguments
16404 instruct CallLeafDirectVector(method meth)
16405 %{
16406 match(CallLeafVector);
16407
16408 effect(USE meth);
16409
16410 ins_cost(CALL_COST);
16411
16412 format %{ "CALL, runtime leaf vector $meth" %}
16413
16414 ins_encode(aarch64_enc_java_to_runtime(meth));
16415
16416 ins_pipe(pipe_class_call);
16417 %}
16418
16419 // Call Runtime Instruction
16420
16421 instruct CallLeafNoFPDirect(method meth)
16422 %{
16423 match(CallLeafNoFP);
16424
16425 effect(USE meth);
16426
16427 ins_cost(CALL_COST);
16428
16429 format %{ "CALL, runtime leaf nofp $meth" %}
16430
16431 ins_encode( aarch64_enc_java_to_runtime(meth) );
16432
16433 ins_pipe(pipe_class_call);
16434 %}
16435
16436 // Tail Call; Jump from runtime stub to Java code.
16437 // Also known as an 'interprocedural jump'.
16438 // Target of jump will eventually return to caller.
16439 // TailJump below removes the return address.
16440 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16441 // emitted just above the TailCall which has reset rfp to the caller state.
16442 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16443 %{
16444 match(TailCall jump_target method_ptr);
16445
16446 ins_cost(CALL_COST);
16447
16448 format %{ "br $jump_target\t# $method_ptr holds method" %}
16449
16450 ins_encode(aarch64_enc_tail_call(jump_target));
16451
16452 ins_pipe(pipe_class_call);
16453 %}
16454
16455 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16456 %{
16457 match(TailJump jump_target ex_oop);
16458
16459 ins_cost(CALL_COST);
16460
16461 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16462
16463 ins_encode(aarch64_enc_tail_jmp(jump_target));
16464
16465 ins_pipe(pipe_class_call);
16466 %}
16467
16468 // Forward exception.
16469 instruct ForwardExceptionjmp()
16470 %{
16471 match(ForwardException);
16472 ins_cost(CALL_COST);
16473
16474 format %{ "b forward_exception_stub" %}
16475 ins_encode %{
16476 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16477 %}
16478 ins_pipe(pipe_class_call);
16479 %}
16480
16481 // Create exception oop: created by stack-crawling runtime code.
16482 // Created exception is now available to this handler, and is setup
16483 // just prior to jumping to this handler. No code emitted.
16484 // TODO check
16485 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16486 instruct CreateException(iRegP_R0 ex_oop)
16487 %{
16488 match(Set ex_oop (CreateEx));
16489
16490 format %{ " -- \t// exception oop; no code emitted" %}
16491
16492 size(0);
16493
16494 ins_encode( /*empty*/ );
16495
16496 ins_pipe(pipe_class_empty);
16497 %}
16498
16499 // Rethrow exception: The exception oop will come in the first
16500 // argument position. Then JUMP (not call) to the rethrow stub code.
16501 instruct RethrowException() %{
16502 match(Rethrow);
16503 ins_cost(CALL_COST);
16504
16505 format %{ "b rethrow_stub" %}
16506
16507 ins_encode( aarch64_enc_rethrow() );
16508
16509 ins_pipe(pipe_class_call);
16510 %}
16511
16512
16513 // Return Instruction
16514 // epilog node loads ret address into lr as part of frame pop
16515 instruct Ret()
16516 %{
16517 match(Return);
16518
16519 format %{ "ret\t// return register" %}
16520
16521 ins_encode( aarch64_enc_ret() );
16522
16523 ins_pipe(pipe_branch);
16524 %}
16525
16526 // Die now.
16527 instruct ShouldNotReachHere() %{
16528 match(Halt);
16529
16530 ins_cost(CALL_COST);
16531 format %{ "ShouldNotReachHere" %}
16532
16533 ins_encode %{
16534 if (is_reachable()) {
16535 const char* str = __ code_string(_halt_reason);
16536 __ stop(str);
16537 }
16538 %}
16539
16540 ins_pipe(pipe_class_default);
16541 %}
16542
16543 // ============================================================================
16544 // Partial Subtype Check
16545 //
16546 // superklass array for an instance of the superklass. Set a hidden
16547 // internal cache on a hit (cache is checked with exposed code in
16548 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16549 // encoding ALSO sets flags.
16550
16551 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16552 %{
16553 match(Set result (PartialSubtypeCheck sub super));
16554 predicate(!UseSecondarySupersTable);
16555 effect(KILL cr, KILL temp);
16556
16557 ins_cost(20 * INSN_COST); // slightly larger than the next version
16558 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16559
16560 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16561
16562 opcode(0x1); // Force zero of result reg on hit
16563
16564 ins_pipe(pipe_class_memory);
16565 %}
16566
16567 // Two versions of partialSubtypeCheck, both used when we need to
16568 // search for a super class in the secondary supers array. The first
16569 // is used when we don't know _a priori_ the class being searched
16570 // for. The second, far more common, is used when we do know: this is
16571 // used for instanceof, checkcast, and any case where C2 can determine
16572 // it by constant propagation.
16573
16574 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16575 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16576 rFlagsReg cr)
16577 %{
16578 match(Set result (PartialSubtypeCheck sub super));
16579 predicate(UseSecondarySupersTable);
16580 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16581
16582 ins_cost(10 * INSN_COST); // slightly larger than the next version
16583 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16584
16585 ins_encode %{
16586 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16587 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16588 $vtemp$$FloatRegister,
16589 $result$$Register, /*L_success*/nullptr);
16590 %}
16591
16592 ins_pipe(pipe_class_memory);
16593 %}
16594
16595 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16596 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16597 rFlagsReg cr)
16598 %{
16599 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16600 predicate(UseSecondarySupersTable);
16601 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16602
16603 ins_cost(5 * INSN_COST); // smaller than the next version
16604 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16605
16606 ins_encode %{
16607 bool success = false;
16608 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16609 if (InlineSecondarySupersTest) {
16610 success =
16611 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16612 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16613 $vtemp$$FloatRegister,
16614 $result$$Register,
16615 super_klass_slot);
16616 } else {
16617 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16618 success = (call != nullptr);
16619 }
16620 if (!success) {
16621 ciEnv::current()->record_failure("CodeCache is full");
16622 return;
16623 }
16624 %}
16625
16626 ins_pipe(pipe_class_memory);
16627 %}
16628
16629 // Intrisics for String.compareTo()
16630
16631 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16632 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16633 %{
16634 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16635 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16636 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16637
16638 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16639 ins_encode %{
16640 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16641 __ string_compare($str1$$Register, $str2$$Register,
16642 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16643 $tmp1$$Register, $tmp2$$Register,
16644 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16645 %}
16646 ins_pipe(pipe_class_memory);
16647 %}
16648
16649 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16650 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16651 %{
16652 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16653 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16654 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16655
16656 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16657 ins_encode %{
16658 __ string_compare($str1$$Register, $str2$$Register,
16659 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16660 $tmp1$$Register, $tmp2$$Register,
16661 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16662 %}
16663 ins_pipe(pipe_class_memory);
16664 %}
16665
16666 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16667 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16668 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16669 %{
16670 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16671 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16672 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16673 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16674
16675 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16676 ins_encode %{
16677 __ string_compare($str1$$Register, $str2$$Register,
16678 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16679 $tmp1$$Register, $tmp2$$Register,
16680 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16681 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16682 %}
16683 ins_pipe(pipe_class_memory);
16684 %}
16685
16686 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16687 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16688 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16689 %{
16690 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16691 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16692 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16693 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16694
16695 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16696 ins_encode %{
16697 __ string_compare($str1$$Register, $str2$$Register,
16698 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16699 $tmp1$$Register, $tmp2$$Register,
16700 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16701 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16702 %}
16703 ins_pipe(pipe_class_memory);
16704 %}
16705
16706 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16707 // these string_compare variants as NEON register type for convenience so that the prototype of
16708 // string_compare can be shared with all variants.
16709
16710 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16711 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16712 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16713 pRegGov_P1 pgtmp2, rFlagsReg cr)
16714 %{
16715 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16716 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16717 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16718 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16719
16720 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16721 ins_encode %{
16722 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16723 __ string_compare($str1$$Register, $str2$$Register,
16724 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16725 $tmp1$$Register, $tmp2$$Register,
16726 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16727 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16728 StrIntrinsicNode::LL);
16729 %}
16730 ins_pipe(pipe_class_memory);
16731 %}
16732
16733 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16734 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16735 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16736 pRegGov_P1 pgtmp2, rFlagsReg cr)
16737 %{
16738 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16739 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16740 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16741 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16742
16743 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16744 ins_encode %{
16745 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16746 __ string_compare($str1$$Register, $str2$$Register,
16747 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16748 $tmp1$$Register, $tmp2$$Register,
16749 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16750 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16751 StrIntrinsicNode::LU);
16752 %}
16753 ins_pipe(pipe_class_memory);
16754 %}
16755
16756 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16757 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16758 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16759 pRegGov_P1 pgtmp2, rFlagsReg cr)
16760 %{
16761 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16762 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16763 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16764 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16765
16766 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16767 ins_encode %{
16768 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16769 __ string_compare($str1$$Register, $str2$$Register,
16770 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16771 $tmp1$$Register, $tmp2$$Register,
16772 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16773 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16774 StrIntrinsicNode::UL);
16775 %}
16776 ins_pipe(pipe_class_memory);
16777 %}
16778
16779 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16780 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16781 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16782 pRegGov_P1 pgtmp2, rFlagsReg cr)
16783 %{
16784 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16785 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16786 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16787 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16788
16789 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16790 ins_encode %{
16791 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16792 __ string_compare($str1$$Register, $str2$$Register,
16793 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16794 $tmp1$$Register, $tmp2$$Register,
16795 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16796 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16797 StrIntrinsicNode::UU);
16798 %}
16799 ins_pipe(pipe_class_memory);
16800 %}
16801
16802 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16803 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16804 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16805 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16806 %{
16807 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16808 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16809 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16810 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16811 TEMP vtmp0, TEMP vtmp1, KILL cr);
16812 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16813 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16814
16815 ins_encode %{
16816 __ string_indexof($str1$$Register, $str2$$Register,
16817 $cnt1$$Register, $cnt2$$Register,
16818 $tmp1$$Register, $tmp2$$Register,
16819 $tmp3$$Register, $tmp4$$Register,
16820 $tmp5$$Register, $tmp6$$Register,
16821 -1, $result$$Register, StrIntrinsicNode::UU);
16822 %}
16823 ins_pipe(pipe_class_memory);
16824 %}
16825
16826 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16827 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16828 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16829 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16830 %{
16831 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16832 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16833 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16834 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16835 TEMP vtmp0, TEMP vtmp1, KILL cr);
16836 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16837 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16838
16839 ins_encode %{
16840 __ string_indexof($str1$$Register, $str2$$Register,
16841 $cnt1$$Register, $cnt2$$Register,
16842 $tmp1$$Register, $tmp2$$Register,
16843 $tmp3$$Register, $tmp4$$Register,
16844 $tmp5$$Register, $tmp6$$Register,
16845 -1, $result$$Register, StrIntrinsicNode::LL);
16846 %}
16847 ins_pipe(pipe_class_memory);
16848 %}
16849
16850 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16851 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16852 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16853 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16854 %{
16855 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16856 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16857 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16858 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16859 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16860 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16861 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16862
16863 ins_encode %{
16864 __ string_indexof($str1$$Register, $str2$$Register,
16865 $cnt1$$Register, $cnt2$$Register,
16866 $tmp1$$Register, $tmp2$$Register,
16867 $tmp3$$Register, $tmp4$$Register,
16868 $tmp5$$Register, $tmp6$$Register,
16869 -1, $result$$Register, StrIntrinsicNode::UL);
16870 %}
16871 ins_pipe(pipe_class_memory);
16872 %}
16873
16874 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16875 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16876 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16877 %{
16878 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16879 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16880 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16881 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16882 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16883 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16884
16885 ins_encode %{
16886 int icnt2 = (int)$int_cnt2$$constant;
16887 __ string_indexof($str1$$Register, $str2$$Register,
16888 $cnt1$$Register, zr,
16889 $tmp1$$Register, $tmp2$$Register,
16890 $tmp3$$Register, $tmp4$$Register, zr, zr,
16891 icnt2, $result$$Register, StrIntrinsicNode::UU);
16892 %}
16893 ins_pipe(pipe_class_memory);
16894 %}
16895
16896 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16897 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16898 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16899 %{
16900 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16901 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16902 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16903 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16904 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16905 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16906
16907 ins_encode %{
16908 int icnt2 = (int)$int_cnt2$$constant;
16909 __ string_indexof($str1$$Register, $str2$$Register,
16910 $cnt1$$Register, zr,
16911 $tmp1$$Register, $tmp2$$Register,
16912 $tmp3$$Register, $tmp4$$Register, zr, zr,
16913 icnt2, $result$$Register, StrIntrinsicNode::LL);
16914 %}
16915 ins_pipe(pipe_class_memory);
16916 %}
16917
16918 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16919 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16920 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16921 %{
16922 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16923 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16924 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16925 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16926 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16927 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16928
16929 ins_encode %{
16930 int icnt2 = (int)$int_cnt2$$constant;
16931 __ string_indexof($str1$$Register, $str2$$Register,
16932 $cnt1$$Register, zr,
16933 $tmp1$$Register, $tmp2$$Register,
16934 $tmp3$$Register, $tmp4$$Register, zr, zr,
16935 icnt2, $result$$Register, StrIntrinsicNode::UL);
16936 %}
16937 ins_pipe(pipe_class_memory);
16938 %}
16939
16940 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16941 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16942 iRegINoSp tmp3, rFlagsReg cr)
16943 %{
16944 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16945 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16946 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16947 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16948
16949 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16950
16951 ins_encode %{
16952 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16953 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16954 $tmp3$$Register);
16955 %}
16956 ins_pipe(pipe_class_memory);
16957 %}
16958
16959 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16960 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16961 iRegINoSp tmp3, rFlagsReg cr)
16962 %{
16963 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16964 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16965 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16966 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16967
16968 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16969
16970 ins_encode %{
16971 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16972 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16973 $tmp3$$Register);
16974 %}
16975 ins_pipe(pipe_class_memory);
16976 %}
16977
16978 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16979 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16980 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16981 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16982 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16983 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16984 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16985 ins_encode %{
16986 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16987 $result$$Register, $ztmp1$$FloatRegister,
16988 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16989 $ptmp$$PRegister, true /* isL */);
16990 %}
16991 ins_pipe(pipe_class_memory);
16992 %}
16993
16994 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16995 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16996 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16997 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16998 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16999 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17000 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17001 ins_encode %{
17002 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17003 $result$$Register, $ztmp1$$FloatRegister,
17004 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17005 $ptmp$$PRegister, false /* isL */);
17006 %}
17007 ins_pipe(pipe_class_memory);
17008 %}
17009
17010 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17011 iRegI_R0 result, rFlagsReg cr)
17012 %{
17013 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17014 match(Set result (StrEquals (Binary str1 str2) cnt));
17015 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17016
17017 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17018 ins_encode %{
17019 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17020 __ string_equals($str1$$Register, $str2$$Register,
17021 $result$$Register, $cnt$$Register);
17022 %}
17023 ins_pipe(pipe_class_memory);
17024 %}
17025
17026 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17027 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17028 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17029 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17030 iRegP_R10 tmp, rFlagsReg cr)
17031 %{
17032 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17033 match(Set result (AryEq ary1 ary2));
17034 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17035 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17036 TEMP vtmp6, TEMP vtmp7, KILL cr);
17037
17038 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17039 ins_encode %{
17040 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17041 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17042 $result$$Register, $tmp$$Register, 1);
17043 if (tpc == nullptr) {
17044 ciEnv::current()->record_failure("CodeCache is full");
17045 return;
17046 }
17047 %}
17048 ins_pipe(pipe_class_memory);
17049 %}
17050
17051 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17052 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17053 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17054 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17055 iRegP_R10 tmp, rFlagsReg cr)
17056 %{
17057 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17058 match(Set result (AryEq ary1 ary2));
17059 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17060 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17061 TEMP vtmp6, TEMP vtmp7, KILL cr);
17062
17063 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17064 ins_encode %{
17065 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17066 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17067 $result$$Register, $tmp$$Register, 2);
17068 if (tpc == nullptr) {
17069 ciEnv::current()->record_failure("CodeCache is full");
17070 return;
17071 }
17072 %}
17073 ins_pipe(pipe_class_memory);
17074 %}
17075
17076 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17077 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17078 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17079 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17080 %{
17081 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17082 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17083 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17084
17085 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
17086 ins_encode %{
17087 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17088 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17089 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17090 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17091 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17092 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17093 (BasicType)$basic_type$$constant);
17094 if (tpc == nullptr) {
17095 ciEnv::current()->record_failure("CodeCache is full");
17096 return;
17097 }
17098 %}
17099 ins_pipe(pipe_class_memory);
17100 %}
17101
17102 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17103 %{
17104 match(Set result (CountPositives ary1 len));
17105 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17106 format %{ "count positives byte[] $ary1,$len -> $result" %}
17107 ins_encode %{
17108 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17109 if (tpc == nullptr) {
17110 ciEnv::current()->record_failure("CodeCache is full");
17111 return;
17112 }
17113 %}
17114 ins_pipe( pipe_slow );
17115 %}
17116
17117 // fast char[] to byte[] compression
17118 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17119 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17120 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17121 iRegI_R0 result, rFlagsReg cr)
17122 %{
17123 match(Set result (StrCompressedCopy src (Binary dst len)));
17124 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17125 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17126
17127 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17128 ins_encode %{
17129 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17130 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17131 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17132 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17133 %}
17134 ins_pipe(pipe_slow);
17135 %}
17136
17137 // fast byte[] to char[] inflation
17138 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17139 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17140 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17141 %{
17142 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17143 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17144 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17145 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17146
17147 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17148 ins_encode %{
17149 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17150 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17151 $vtmp2$$FloatRegister, $tmp$$Register);
17152 if (tpc == nullptr) {
17153 ciEnv::current()->record_failure("CodeCache is full");
17154 return;
17155 }
17156 %}
17157 ins_pipe(pipe_class_memory);
17158 %}
17159
17160 // encode char[] to byte[] in ISO_8859_1
17161 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17162 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17163 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17164 iRegI_R0 result, rFlagsReg cr)
17165 %{
17166 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17167 match(Set result (EncodeISOArray src (Binary dst len)));
17168 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17169 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17170
17171 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17172 ins_encode %{
17173 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17174 $result$$Register, false,
17175 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17176 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17177 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17178 %}
17179 ins_pipe(pipe_class_memory);
17180 %}
17181
17182 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17183 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17184 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17185 iRegI_R0 result, rFlagsReg cr)
17186 %{
17187 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17188 match(Set result (EncodeISOArray src (Binary dst len)));
17189 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17190 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17191
17192 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17193 ins_encode %{
17194 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17195 $result$$Register, true,
17196 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17197 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17198 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17199 %}
17200 ins_pipe(pipe_class_memory);
17201 %}
17202
17203 //----------------------------- CompressBits/ExpandBits ------------------------
17204
17205 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17206 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17207 match(Set dst (CompressBits src mask));
17208 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17209 format %{ "mov $tsrc, $src\n\t"
17210 "mov $tmask, $mask\n\t"
17211 "bext $tdst, $tsrc, $tmask\n\t"
17212 "mov $dst, $tdst"
17213 %}
17214 ins_encode %{
17215 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17216 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17217 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17218 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17219 %}
17220 ins_pipe(pipe_slow);
17221 %}
17222
17223 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17224 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17225 match(Set dst (CompressBits (LoadI mem) mask));
17226 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17227 format %{ "ldrs $tsrc, $mem\n\t"
17228 "ldrs $tmask, $mask\n\t"
17229 "bext $tdst, $tsrc, $tmask\n\t"
17230 "mov $dst, $tdst"
17231 %}
17232 ins_encode %{
17233 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17234 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17235 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17236 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17237 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17238 %}
17239 ins_pipe(pipe_slow);
17240 %}
17241
17242 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17243 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17244 match(Set dst (CompressBits src mask));
17245 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17246 format %{ "mov $tsrc, $src\n\t"
17247 "mov $tmask, $mask\n\t"
17248 "bext $tdst, $tsrc, $tmask\n\t"
17249 "mov $dst, $tdst"
17250 %}
17251 ins_encode %{
17252 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17253 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17254 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17255 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17256 %}
17257 ins_pipe(pipe_slow);
17258 %}
17259
17260 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17261 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17262 match(Set dst (CompressBits (LoadL mem) mask));
17263 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17264 format %{ "ldrd $tsrc, $mem\n\t"
17265 "ldrd $tmask, $mask\n\t"
17266 "bext $tdst, $tsrc, $tmask\n\t"
17267 "mov $dst, $tdst"
17268 %}
17269 ins_encode %{
17270 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17271 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17272 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17273 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17274 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17275 %}
17276 ins_pipe(pipe_slow);
17277 %}
17278
17279 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17280 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17281 match(Set dst (ExpandBits src mask));
17282 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17283 format %{ "mov $tsrc, $src\n\t"
17284 "mov $tmask, $mask\n\t"
17285 "bdep $tdst, $tsrc, $tmask\n\t"
17286 "mov $dst, $tdst"
17287 %}
17288 ins_encode %{
17289 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17290 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17291 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17292 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17293 %}
17294 ins_pipe(pipe_slow);
17295 %}
17296
17297 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17298 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17299 match(Set dst (ExpandBits (LoadI mem) mask));
17300 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17301 format %{ "ldrs $tsrc, $mem\n\t"
17302 "ldrs $tmask, $mask\n\t"
17303 "bdep $tdst, $tsrc, $tmask\n\t"
17304 "mov $dst, $tdst"
17305 %}
17306 ins_encode %{
17307 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17308 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17309 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17310 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17311 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17312 %}
17313 ins_pipe(pipe_slow);
17314 %}
17315
17316 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17317 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17318 match(Set dst (ExpandBits src mask));
17319 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17320 format %{ "mov $tsrc, $src\n\t"
17321 "mov $tmask, $mask\n\t"
17322 "bdep $tdst, $tsrc, $tmask\n\t"
17323 "mov $dst, $tdst"
17324 %}
17325 ins_encode %{
17326 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17327 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17328 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17329 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17330 %}
17331 ins_pipe(pipe_slow);
17332 %}
17333
17334
17335 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17336 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17337 match(Set dst (ExpandBits (LoadL mem) mask));
17338 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17339 format %{ "ldrd $tsrc, $mem\n\t"
17340 "ldrd $tmask, $mask\n\t"
17341 "bdep $tdst, $tsrc, $tmask\n\t"
17342 "mov $dst, $tdst"
17343 %}
17344 ins_encode %{
17345 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17346 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17347 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17348 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17349 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17350 %}
17351 ins_pipe(pipe_slow);
17352 %}
17353
17354 //----------------------------- Reinterpret ----------------------------------
17355 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17356 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17357 match(Set dst (ReinterpretHF2S src));
17358 format %{ "reinterpretHF2S $dst, $src" %}
17359 ins_encode %{
17360 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17361 %}
17362 ins_pipe(pipe_slow);
17363 %}
17364
17365 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17366 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17367 match(Set dst (ReinterpretS2HF src));
17368 format %{ "reinterpretS2HF $dst, $src" %}
17369 ins_encode %{
17370 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17371 %}
17372 ins_pipe(pipe_slow);
17373 %}
17374
17375 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17376 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17377 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
17378 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
17379 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
17380 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17381 // can be omitted in this pattern, resulting in -
17382 // fcvt $dst, $src // Convert float to half-precision float
17383 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17384 %{
17385 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17386 format %{ "convF2HFAndS2HF $dst, $src" %}
17387 ins_encode %{
17388 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17389 %}
17390 ins_pipe(pipe_slow);
17391 %}
17392
17393 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17394 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17395 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
17396 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17397 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
17398 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17399 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17400 // resulting in -
17401 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
17402 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17403 %{
17404 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17405 format %{ "convHF2SAndHF2F $dst, $src" %}
17406 ins_encode %{
17407 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17408 %}
17409 ins_pipe(pipe_slow);
17410 %}
17411
17412 // ============================================================================
17413 // This name is KNOWN by the ADLC and cannot be changed.
17414 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17415 // for this guy.
17416 instruct tlsLoadP(thread_RegP dst)
17417 %{
17418 match(Set dst (ThreadLocal));
17419
17420 ins_cost(0);
17421
17422 format %{ " -- \t// $dst=Thread::current(), empty" %}
17423
17424 size(0);
17425
17426 ins_encode( /*empty*/ );
17427
17428 ins_pipe(pipe_class_empty);
17429 %}
17430
17431 //----------PEEPHOLE RULES-----------------------------------------------------
17432 // These must follow all instruction definitions as they use the names
17433 // defined in the instructions definitions.
17434 //
17435 // peepmatch ( root_instr_name [preceding_instruction]* );
17436 //
17437 // peepconstraint %{
17438 // (instruction_number.operand_name relational_op instruction_number.operand_name
17439 // [, ...] );
17440 // // instruction numbers are zero-based using left to right order in peepmatch
17441 //
17442 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17443 // // provide an instruction_number.operand_name for each operand that appears
17444 // // in the replacement instruction's match rule
17445 //
17446 // ---------VM FLAGS---------------------------------------------------------
17447 //
17448 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17449 //
17450 // Each peephole rule is given an identifying number starting with zero and
17451 // increasing by one in the order seen by the parser. An individual peephole
17452 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17453 // on the command-line.
17454 //
17455 // ---------CURRENT LIMITATIONS----------------------------------------------
17456 //
17457 // Only match adjacent instructions in same basic block
17458 // Only equality constraints
17459 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17460 // Only one replacement instruction
17461 //
17462 // ---------EXAMPLE----------------------------------------------------------
17463 //
17464 // // pertinent parts of existing instructions in architecture description
17465 // instruct movI(iRegINoSp dst, iRegI src)
17466 // %{
17467 // match(Set dst (CopyI src));
17468 // %}
17469 //
17470 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17471 // %{
17472 // match(Set dst (AddI dst src));
17473 // effect(KILL cr);
17474 // %}
17475 //
17476 // // Change (inc mov) to lea
17477 // peephole %{
17478 // // increment preceded by register-register move
17479 // peepmatch ( incI_iReg movI );
17480 // // require that the destination register of the increment
17481 // // match the destination register of the move
17482 // peepconstraint ( 0.dst == 1.dst );
17483 // // construct a replacement instruction that sets
17484 // // the destination to ( move's source register + one )
17485 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17486 // %}
17487 //
17488
17489 // Implementation no longer uses movX instructions since
17490 // machine-independent system no longer uses CopyX nodes.
17491 //
17492 // peephole
17493 // %{
17494 // peepmatch (incI_iReg movI);
17495 // peepconstraint (0.dst == 1.dst);
17496 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17497 // %}
17498
17499 // peephole
17500 // %{
17501 // peepmatch (decI_iReg movI);
17502 // peepconstraint (0.dst == 1.dst);
17503 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17504 // %}
17505
17506 // peephole
17507 // %{
17508 // peepmatch (addI_iReg_imm movI);
17509 // peepconstraint (0.dst == 1.dst);
17510 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17511 // %}
17512
17513 // peephole
17514 // %{
17515 // peepmatch (incL_iReg movL);
17516 // peepconstraint (0.dst == 1.dst);
17517 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17518 // %}
17519
17520 // peephole
17521 // %{
17522 // peepmatch (decL_iReg movL);
17523 // peepconstraint (0.dst == 1.dst);
17524 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17525 // %}
17526
17527 // peephole
17528 // %{
17529 // peepmatch (addL_iReg_imm movL);
17530 // peepconstraint (0.dst == 1.dst);
17531 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17532 // %}
17533
17534 // peephole
17535 // %{
17536 // peepmatch (addP_iReg_imm movP);
17537 // peepconstraint (0.dst == 1.dst);
17538 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17539 // %}
17540
17541 // // Change load of spilled value to only a spill
17542 // instruct storeI(memory mem, iRegI src)
17543 // %{
17544 // match(Set mem (StoreI mem src));
17545 // %}
17546 //
17547 // instruct loadI(iRegINoSp dst, memory mem)
17548 // %{
17549 // match(Set dst (LoadI mem));
17550 // %}
17551 //
17552
17553 //----------SMARTSPILL RULES---------------------------------------------------
17554 // These must follow all instruction definitions as they use the names
17555 // defined in the instructions definitions.
17556
17557 // Local Variables:
17558 // mode: c++
17559 // End: