1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for vector register V10
885 reg_class v10_veca_reg(
886 V10, V10_H, V10_J, V10_K
887 );
888
889 // Class for vector register V11
890 reg_class v11_veca_reg(
891 V11, V11_H, V11_J, V11_K
892 );
893
894 // Class for vector register V12
895 reg_class v12_veca_reg(
896 V12, V12_H, V12_J, V12_K
897 );
898
899 // Class for vector register V13
900 reg_class v13_veca_reg(
901 V13, V13_H, V13_J, V13_K
902 );
903
904 // Class for vector register V17
905 reg_class v17_veca_reg(
906 V17, V17_H, V17_J, V17_K
907 );
908
909 // Class for vector register V18
910 reg_class v18_veca_reg(
911 V18, V18_H, V18_J, V18_K
912 );
913
914 // Class for vector register V23
915 reg_class v23_veca_reg(
916 V23, V23_H, V23_J, V23_K
917 );
918
919 // Class for vector register V24
920 reg_class v24_veca_reg(
921 V24, V24_H, V24_J, V24_K
922 );
923
924 // Class for 128 bit register v0
925 reg_class v0_reg(
926 V0, V0_H
927 );
928
929 // Class for 128 bit register v1
930 reg_class v1_reg(
931 V1, V1_H
932 );
933
934 // Class for 128 bit register v2
935 reg_class v2_reg(
936 V2, V2_H
937 );
938
939 // Class for 128 bit register v3
940 reg_class v3_reg(
941 V3, V3_H
942 );
943
944 // Class for 128 bit register v4
945 reg_class v4_reg(
946 V4, V4_H
947 );
948
949 // Class for 128 bit register v5
950 reg_class v5_reg(
951 V5, V5_H
952 );
953
954 // Class for 128 bit register v6
955 reg_class v6_reg(
956 V6, V6_H
957 );
958
959 // Class for 128 bit register v7
960 reg_class v7_reg(
961 V7, V7_H
962 );
963
964 // Class for 128 bit register v8
965 reg_class v8_reg(
966 V8, V8_H
967 );
968
969 // Class for 128 bit register v9
970 reg_class v9_reg(
971 V9, V9_H
972 );
973
974 // Class for 128 bit register v10
975 reg_class v10_reg(
976 V10, V10_H
977 );
978
979 // Class for 128 bit register v11
980 reg_class v11_reg(
981 V11, V11_H
982 );
983
984 // Class for 128 bit register v12
985 reg_class v12_reg(
986 V12, V12_H
987 );
988
989 // Class for 128 bit register v13
990 reg_class v13_reg(
991 V13, V13_H
992 );
993
994 // Class for 128 bit register v14
995 reg_class v14_reg(
996 V14, V14_H
997 );
998
999 // Class for 128 bit register v15
1000 reg_class v15_reg(
1001 V15, V15_H
1002 );
1003
1004 // Class for 128 bit register v16
1005 reg_class v16_reg(
1006 V16, V16_H
1007 );
1008
1009 // Class for 128 bit register v17
1010 reg_class v17_reg(
1011 V17, V17_H
1012 );
1013
1014 // Class for 128 bit register v18
1015 reg_class v18_reg(
1016 V18, V18_H
1017 );
1018
1019 // Class for 128 bit register v19
1020 reg_class v19_reg(
1021 V19, V19_H
1022 );
1023
1024 // Class for 128 bit register v20
1025 reg_class v20_reg(
1026 V20, V20_H
1027 );
1028
1029 // Class for 128 bit register v21
1030 reg_class v21_reg(
1031 V21, V21_H
1032 );
1033
1034 // Class for 128 bit register v22
1035 reg_class v22_reg(
1036 V22, V22_H
1037 );
1038
1039 // Class for 128 bit register v23
1040 reg_class v23_reg(
1041 V23, V23_H
1042 );
1043
1044 // Class for 128 bit register v24
1045 reg_class v24_reg(
1046 V24, V24_H
1047 );
1048
1049 // Class for 128 bit register v25
1050 reg_class v25_reg(
1051 V25, V25_H
1052 );
1053
1054 // Class for 128 bit register v26
1055 reg_class v26_reg(
1056 V26, V26_H
1057 );
1058
1059 // Class for 128 bit register v27
1060 reg_class v27_reg(
1061 V27, V27_H
1062 );
1063
1064 // Class for 128 bit register v28
1065 reg_class v28_reg(
1066 V28, V28_H
1067 );
1068
1069 // Class for 128 bit register v29
1070 reg_class v29_reg(
1071 V29, V29_H
1072 );
1073
1074 // Class for 128 bit register v30
1075 reg_class v30_reg(
1076 V30, V30_H
1077 );
1078
1079 // Class for 128 bit register v31
1080 reg_class v31_reg(
1081 V31, V31_H
1082 );
1083
1084 // Class for all SVE predicate registers.
1085 reg_class pr_reg (
1086 P0,
1087 P1,
1088 P2,
1089 P3,
1090 P4,
1091 P5,
1092 P6,
1093 // P7, non-allocatable, preserved with all elements preset to TRUE.
1094 P8,
1095 P9,
1096 P10,
1097 P11,
1098 P12,
1099 P13,
1100 P14,
1101 P15
1102 );
1103
1104 // Class for SVE governing predicate registers, which are used
1105 // to determine the active elements of a predicated instruction.
1106 reg_class gov_pr (
1107 P0,
1108 P1,
1109 P2,
1110 P3,
1111 P4,
1112 P5,
1113 P6,
1114 // P7, non-allocatable, preserved with all elements preset to TRUE.
1115 );
1116
1117 reg_class p0_reg(P0);
1118 reg_class p1_reg(P1);
1119
1120 // Singleton class for condition codes
1121 reg_class int_flags(RFLAGS);
1122
1123 %}
1124
1125 //----------DEFINITION BLOCK---------------------------------------------------
1126 // Define name --> value mappings to inform the ADLC of an integer valued name
1127 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1128 // Format:
1129 // int_def <name> ( <int_value>, <expression>);
1130 // Generated Code in ad_<arch>.hpp
1131 // #define <name> (<expression>)
1132 // // value == <int_value>
1133 // Generated code in ad_<arch>.cpp adlc_verification()
1134 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1135 //
1136
1137 // we follow the ppc-aix port in using a simple cost model which ranks
1138 // register operations as cheap, memory ops as more expensive and
1139 // branches as most expensive. the first two have a low as well as a
1140 // normal cost. huge cost appears to be a way of saying don't do
1141 // something
1142
1143 definitions %{
1144 // The default cost (of a register move instruction).
1145 int_def INSN_COST ( 100, 100);
1146 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1147 int_def CALL_COST ( 200, 2 * INSN_COST);
1148 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1149 %}
1150
1151
1152 //----------SOURCE BLOCK-------------------------------------------------------
1153 // This is a block of C++ code which provides values, functions, and
1154 // definitions necessary in the rest of the architecture description
1155
1156 source_hpp %{
1157
1158 #include "asm/macroAssembler.hpp"
1159 #include "gc/shared/barrierSetAssembler.hpp"
1160 #include "gc/shared/cardTable.hpp"
1161 #include "gc/shared/cardTableBarrierSet.hpp"
1162 #include "gc/shared/collectedHeap.hpp"
1163 #include "opto/addnode.hpp"
1164 #include "opto/convertnode.hpp"
1165 #include "runtime/objectMonitor.hpp"
1166
1167 extern RegMask _ANY_REG32_mask;
1168 extern RegMask _ANY_REG_mask;
1169 extern RegMask _PTR_REG_mask;
1170 extern RegMask _NO_SPECIAL_REG32_mask;
1171 extern RegMask _NO_SPECIAL_REG_mask;
1172 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1173 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1174
1175 class CallStubImpl {
1176
1177 //--------------------------------------------------------------
1178 //---< Used for optimization in Compile::shorten_branches >---
1179 //--------------------------------------------------------------
1180
1181 public:
1182 // Size of call trampoline stub.
1183 static uint size_call_trampoline() {
1184 return 0; // no call trampolines on this platform
1185 }
1186
1187 // number of relocations needed by a call trampoline stub
1188 static uint reloc_call_trampoline() {
1189 return 0; // no call trampolines on this platform
1190 }
1191 };
1192
1193 class HandlerImpl {
1194
1195 public:
1196
1197 static int emit_exception_handler(C2_MacroAssembler *masm);
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_exception_handler() {
1201 return MacroAssembler::far_codestub_branch_size();
1202 }
1203
1204 static uint size_deopt_handler() {
1205 // count one adr and one far branch instruction
1206 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1207 }
1208 };
1209
1210 class Node::PD {
1211 public:
1212 enum NodeFlags {
1213 _last_flag = Node::_last_flag
1214 };
1215 };
1216
1217 bool is_CAS(int opcode, bool maybe_volatile);
1218
1219 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1220
1221 bool unnecessary_acquire(const Node *barrier);
1222 bool needs_acquiring_load(const Node *load);
1223
1224 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1225
1226 bool unnecessary_release(const Node *barrier);
1227 bool unnecessary_volatile(const Node *barrier);
1228 bool needs_releasing_store(const Node *store);
1229
1230 // predicate controlling translation of CompareAndSwapX
1231 bool needs_acquiring_load_exclusive(const Node *load);
1232
1233 // predicate controlling addressing modes
1234 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1235
1236 // Convert BootTest condition to Assembler condition.
1237 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1238 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1239 %}
1240
1241 source %{
1242
1243 // Derived RegMask with conditionally allocatable registers
1244
1245 void PhaseOutput::pd_perform_mach_node_analysis() {
1246 }
1247
1248 int MachNode::pd_alignment_required() const {
1249 return 1;
1250 }
1251
1252 int MachNode::compute_padding(int current_offset) const {
1253 return 0;
1254 }
1255
1256 RegMask _ANY_REG32_mask;
1257 RegMask _ANY_REG_mask;
1258 RegMask _PTR_REG_mask;
1259 RegMask _NO_SPECIAL_REG32_mask;
1260 RegMask _NO_SPECIAL_REG_mask;
1261 RegMask _NO_SPECIAL_PTR_REG_mask;
1262 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1263
1264 void reg_mask_init() {
1265 // We derive below RegMask(s) from the ones which are auto-generated from
1266 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1267 // registers conditionally reserved.
1268
1269 _ANY_REG32_mask = _ALL_REG32_mask;
1270 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1271
1272 _ANY_REG_mask = _ALL_REG_mask;
1273
1274 _PTR_REG_mask = _ALL_REG_mask;
1275
1276 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1277 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1278
1279 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1280 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1281
1282 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1283 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1284
1285 // r27 is not allocatable when compressed oops is on and heapbase is not
1286 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1287 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1288 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1289 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1290 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1291 }
1292
1293 // r29 is not allocatable when PreserveFramePointer is on
1294 if (PreserveFramePointer) {
1295 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1296 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1297 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1301 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1302 }
1303
1304 // Optimizaton of volatile gets and puts
1305 // -------------------------------------
1306 //
1307 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1308 // use to implement volatile reads and writes. For a volatile read
1309 // we simply need
1310 //
1311 // ldar<x>
1312 //
1313 // and for a volatile write we need
1314 //
1315 // stlr<x>
1316 //
1317 // Alternatively, we can implement them by pairing a normal
1318 // load/store with a memory barrier. For a volatile read we need
1319 //
1320 // ldr<x>
1321 // dmb ishld
1322 //
1323 // for a volatile write
1324 //
1325 // dmb ish
1326 // str<x>
1327 // dmb ish
1328 //
1329 // We can also use ldaxr and stlxr to implement compare and swap CAS
1330 // sequences. These are normally translated to an instruction
1331 // sequence like the following
1332 //
1333 // dmb ish
1334 // retry:
1335 // ldxr<x> rval raddr
1336 // cmp rval rold
1337 // b.ne done
1338 // stlxr<x> rval, rnew, rold
1339 // cbnz rval retry
1340 // done:
1341 // cset r0, eq
1342 // dmb ishld
1343 //
1344 // Note that the exclusive store is already using an stlxr
1345 // instruction. That is required to ensure visibility to other
1346 // threads of the exclusive write (assuming it succeeds) before that
1347 // of any subsequent writes.
1348 //
1349 // The following instruction sequence is an improvement on the above
1350 //
1351 // retry:
1352 // ldaxr<x> rval raddr
1353 // cmp rval rold
1354 // b.ne done
1355 // stlxr<x> rval, rnew, rold
1356 // cbnz rval retry
1357 // done:
1358 // cset r0, eq
1359 //
1360 // We don't need the leading dmb ish since the stlxr guarantees
1361 // visibility of prior writes in the case that the swap is
1362 // successful. Crucially we don't have to worry about the case where
1363 // the swap is not successful since no valid program should be
1364 // relying on visibility of prior changes by the attempting thread
1365 // in the case where the CAS fails.
1366 //
1367 // Similarly, we don't need the trailing dmb ishld if we substitute
1368 // an ldaxr instruction since that will provide all the guarantees we
1369 // require regarding observation of changes made by other threads
1370 // before any change to the CAS address observed by the load.
1371 //
1372 // In order to generate the desired instruction sequence we need to
1373 // be able to identify specific 'signature' ideal graph node
1374 // sequences which i) occur as a translation of a volatile reads or
1375 // writes or CAS operations and ii) do not occur through any other
1376 // translation or graph transformation. We can then provide
1377 // alternative aldc matching rules which translate these node
1378 // sequences to the desired machine code sequences. Selection of the
1379 // alternative rules can be implemented by predicates which identify
1380 // the relevant node sequences.
1381 //
1382 // The ideal graph generator translates a volatile read to the node
1383 // sequence
1384 //
1385 // LoadX[mo_acquire]
1386 // MemBarAcquire
1387 //
1388 // As a special case when using the compressed oops optimization we
1389 // may also see this variant
1390 //
1391 // LoadN[mo_acquire]
1392 // DecodeN
1393 // MemBarAcquire
1394 //
1395 // A volatile write is translated to the node sequence
1396 //
1397 // MemBarRelease
1398 // StoreX[mo_release] {CardMark}-optional
1399 // MemBarVolatile
1400 //
1401 // n.b. the above node patterns are generated with a strict
1402 // 'signature' configuration of input and output dependencies (see
1403 // the predicates below for exact details). The card mark may be as
1404 // simple as a few extra nodes or, in a few GC configurations, may
1405 // include more complex control flow between the leading and
1406 // trailing memory barriers. However, whatever the card mark
1407 // configuration these signatures are unique to translated volatile
1408 // reads/stores -- they will not appear as a result of any other
1409 // bytecode translation or inlining nor as a consequence of
1410 // optimizing transforms.
1411 //
1412 // We also want to catch inlined unsafe volatile gets and puts and
1413 // be able to implement them using either ldar<x>/stlr<x> or some
1414 // combination of ldr<x>/stlr<x> and dmb instructions.
1415 //
1416 // Inlined unsafe volatiles puts manifest as a minor variant of the
1417 // normal volatile put node sequence containing an extra cpuorder
1418 // membar
1419 //
1420 // MemBarRelease
1421 // MemBarCPUOrder
1422 // StoreX[mo_release] {CardMark}-optional
1423 // MemBarCPUOrder
1424 // MemBarVolatile
1425 //
1426 // n.b. as an aside, a cpuorder membar is not itself subject to
1427 // matching and translation by adlc rules. However, the rule
1428 // predicates need to detect its presence in order to correctly
1429 // select the desired adlc rules.
1430 //
1431 // Inlined unsafe volatile gets manifest as a slightly different
1432 // node sequence to a normal volatile get because of the
1433 // introduction of some CPUOrder memory barriers to bracket the
1434 // Load. However, but the same basic skeleton of a LoadX feeding a
1435 // MemBarAcquire, possibly through an optional DecodeN, is still
1436 // present
1437 //
1438 // MemBarCPUOrder
1439 // || \\
1440 // MemBarCPUOrder LoadX[mo_acquire]
1441 // || |
1442 // || {DecodeN} optional
1443 // || /
1444 // MemBarAcquire
1445 //
1446 // In this case the acquire membar does not directly depend on the
1447 // load. However, we can be sure that the load is generated from an
1448 // inlined unsafe volatile get if we see it dependent on this unique
1449 // sequence of membar nodes. Similarly, given an acquire membar we
1450 // can know that it was added because of an inlined unsafe volatile
1451 // get if it is fed and feeds a cpuorder membar and if its feed
1452 // membar also feeds an acquiring load.
1453 //
1454 // Finally an inlined (Unsafe) CAS operation is translated to the
1455 // following ideal graph
1456 //
1457 // MemBarRelease
1458 // MemBarCPUOrder
1459 // CompareAndSwapX {CardMark}-optional
1460 // MemBarCPUOrder
1461 // MemBarAcquire
1462 //
1463 // So, where we can identify these volatile read and write
1464 // signatures we can choose to plant either of the above two code
1465 // sequences. For a volatile read we can simply plant a normal
1466 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1467 // also choose to inhibit translation of the MemBarAcquire and
1468 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1469 //
1470 // When we recognise a volatile store signature we can choose to
1471 // plant at a dmb ish as a translation for the MemBarRelease, a
1472 // normal str<x> and then a dmb ish for the MemBarVolatile.
1473 // Alternatively, we can inhibit translation of the MemBarRelease
1474 // and MemBarVolatile and instead plant a simple stlr<x>
1475 // instruction.
1476 //
1477 // when we recognise a CAS signature we can choose to plant a dmb
1478 // ish as a translation for the MemBarRelease, the conventional
1479 // macro-instruction sequence for the CompareAndSwap node (which
1480 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1481 // Alternatively, we can elide generation of the dmb instructions
1482 // and plant the alternative CompareAndSwap macro-instruction
1483 // sequence (which uses ldaxr<x>).
1484 //
1485 // Of course, the above only applies when we see these signature
1486 // configurations. We still want to plant dmb instructions in any
1487 // other cases where we may see a MemBarAcquire, MemBarRelease or
1488 // MemBarVolatile. For example, at the end of a constructor which
1489 // writes final/volatile fields we will see a MemBarRelease
1490 // instruction and this needs a 'dmb ish' lest we risk the
1491 // constructed object being visible without making the
1492 // final/volatile field writes visible.
1493 //
1494 // n.b. the translation rules below which rely on detection of the
1495 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1496 // If we see anything other than the signature configurations we
1497 // always just translate the loads and stores to ldr<x> and str<x>
1498 // and translate acquire, release and volatile membars to the
1499 // relevant dmb instructions.
1500 //
1501
1502 // is_CAS(int opcode, bool maybe_volatile)
1503 //
1504 // return true if opcode is one of the possible CompareAndSwapX
1505 // values otherwise false.
1506
1507 bool is_CAS(int opcode, bool maybe_volatile)
1508 {
1509 switch(opcode) {
1510 // We handle these
1511 case Op_CompareAndSwapI:
1512 case Op_CompareAndSwapL:
1513 case Op_CompareAndSwapP:
1514 case Op_CompareAndSwapN:
1515 case Op_ShenandoahCompareAndSwapP:
1516 case Op_ShenandoahCompareAndSwapN:
1517 case Op_CompareAndSwapB:
1518 case Op_CompareAndSwapS:
1519 case Op_GetAndSetI:
1520 case Op_GetAndSetL:
1521 case Op_GetAndSetP:
1522 case Op_GetAndSetN:
1523 case Op_GetAndAddI:
1524 case Op_GetAndAddL:
1525 return true;
1526 case Op_CompareAndExchangeI:
1527 case Op_CompareAndExchangeN:
1528 case Op_CompareAndExchangeB:
1529 case Op_CompareAndExchangeS:
1530 case Op_CompareAndExchangeL:
1531 case Op_CompareAndExchangeP:
1532 case Op_WeakCompareAndSwapB:
1533 case Op_WeakCompareAndSwapS:
1534 case Op_WeakCompareAndSwapI:
1535 case Op_WeakCompareAndSwapL:
1536 case Op_WeakCompareAndSwapP:
1537 case Op_WeakCompareAndSwapN:
1538 case Op_ShenandoahWeakCompareAndSwapP:
1539 case Op_ShenandoahWeakCompareAndSwapN:
1540 case Op_ShenandoahCompareAndExchangeP:
1541 case Op_ShenandoahCompareAndExchangeN:
1542 return maybe_volatile;
1543 default:
1544 return false;
1545 }
1546 }
1547
1548 // helper to determine the maximum number of Phi nodes we may need to
1549 // traverse when searching from a card mark membar for the merge mem
1550 // feeding a trailing membar or vice versa
1551
1552 // predicates controlling emit of ldr<x>/ldar<x>
1553
1554 bool unnecessary_acquire(const Node *barrier)
1555 {
1556 assert(barrier->is_MemBar(), "expecting a membar");
1557
1558 MemBarNode* mb = barrier->as_MemBar();
1559
1560 if (mb->trailing_load()) {
1561 return true;
1562 }
1563
1564 if (mb->trailing_load_store()) {
1565 Node* load_store = mb->in(MemBarNode::Precedent);
1566 assert(load_store->is_LoadStore(), "unexpected graph shape");
1567 return is_CAS(load_store->Opcode(), true);
1568 }
1569
1570 return false;
1571 }
1572
1573 bool needs_acquiring_load(const Node *n)
1574 {
1575 assert(n->is_Load(), "expecting a load");
1576 LoadNode *ld = n->as_Load();
1577 return ld->is_acquire();
1578 }
1579
1580 bool unnecessary_release(const Node *n)
1581 {
1582 assert((n->is_MemBar() &&
1583 n->Opcode() == Op_MemBarRelease),
1584 "expecting a release membar");
1585
1586 MemBarNode *barrier = n->as_MemBar();
1587 if (!barrier->leading()) {
1588 return false;
1589 } else {
1590 Node* trailing = barrier->trailing_membar();
1591 MemBarNode* trailing_mb = trailing->as_MemBar();
1592 assert(trailing_mb->trailing(), "Not a trailing membar?");
1593 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1594
1595 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1596 if (mem->is_Store()) {
1597 assert(mem->as_Store()->is_release(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1599 return true;
1600 } else {
1601 assert(mem->is_LoadStore(), "");
1602 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1603 return is_CAS(mem->Opcode(), true);
1604 }
1605 }
1606 return false;
1607 }
1608
1609 bool unnecessary_volatile(const Node *n)
1610 {
1611 // assert n->is_MemBar();
1612 MemBarNode *mbvol = n->as_MemBar();
1613
1614 bool release = mbvol->trailing_store();
1615 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1616 #ifdef ASSERT
1617 if (release) {
1618 Node* leading = mbvol->leading_membar();
1619 assert(leading->Opcode() == Op_MemBarRelease, "");
1620 assert(leading->as_MemBar()->leading_store(), "");
1621 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1622 }
1623 #endif
1624
1625 return release;
1626 }
1627
1628 // predicates controlling emit of str<x>/stlr<x>
1629
1630 bool needs_releasing_store(const Node *n)
1631 {
1632 // assert n->is_Store();
1633 StoreNode *st = n->as_Store();
1634 return st->trailing_membar() != nullptr;
1635 }
1636
1637 // predicate controlling translation of CAS
1638 //
1639 // returns true if CAS needs to use an acquiring load otherwise false
1640
1641 bool needs_acquiring_load_exclusive(const Node *n)
1642 {
1643 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1644 LoadStoreNode* ldst = n->as_LoadStore();
1645 if (is_CAS(n->Opcode(), false)) {
1646 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1647 } else {
1648 return ldst->trailing_membar() != nullptr;
1649 }
1650
1651 // so we can just return true here
1652 return true;
1653 }
1654
1655 #define __ masm->
1656
1657 // advance declarations for helper functions to convert register
1658 // indices to register objects
1659
1660 // the ad file has to provide implementations of certain methods
1661 // expected by the generic code
1662 //
1663 // REQUIRED FUNCTIONALITY
1664
1665 //=============================================================================
1666
1667 // !!!!! Special hack to get all types of calls to specify the byte offset
1668 // from the start of the call to the point where the return address
1669 // will point.
1670
1671 int MachCallStaticJavaNode::ret_addr_offset()
1672 {
1673 // call should be a simple bl
1674 int off = 4;
1675 return off;
1676 }
1677
1678 int MachCallDynamicJavaNode::ret_addr_offset()
1679 {
1680 return 16; // movz, movk, movk, bl
1681 }
1682
1683 int MachCallRuntimeNode::ret_addr_offset() {
1684 // for generated stubs the call will be
1685 // bl(addr)
1686 // or with far branches
1687 // bl(trampoline_stub)
1688 // for real runtime callouts it will be six instructions
1689 // see aarch64_enc_java_to_runtime
1690 // adr(rscratch2, retaddr)
1691 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1692 // lea(rscratch1, RuntimeAddress(addr)
1693 // blr(rscratch1)
1694 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1695 if (cb) {
1696 return 1 * NativeInstruction::instruction_size;
1697 } else {
1698 return 6 * NativeInstruction::instruction_size;
1699 }
1700 }
1701
1702 //=============================================================================
1703
1704 #ifndef PRODUCT
1705 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1706 st->print("BREAKPOINT");
1707 }
1708 #endif
1709
1710 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1711 __ brk(0);
1712 }
1713
1714 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1715 return MachNode::size(ra_);
1716 }
1717
1718 //=============================================================================
1719
1720 #ifndef PRODUCT
1721 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1722 st->print("nop \t# %d bytes pad for loops and calls", _count);
1723 }
1724 #endif
1725
1726 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1727 for (int i = 0; i < _count; i++) {
1728 __ nop();
1729 }
1730 }
1731
1732 uint MachNopNode::size(PhaseRegAlloc*) const {
1733 return _count * NativeInstruction::instruction_size;
1734 }
1735
1736 //=============================================================================
1737 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1738
1739 int ConstantTable::calculate_table_base_offset() const {
1740 return 0; // absolute addressing, no offset
1741 }
1742
1743 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1744 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1745 ShouldNotReachHere();
1746 }
1747
1748 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1749 // Empty encoding
1750 }
1751
1752 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1753 return 0;
1754 }
1755
1756 #ifndef PRODUCT
1757 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1758 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1759 }
1760 #endif
1761
1762 #ifndef PRODUCT
1763 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1764 Compile* C = ra_->C;
1765
1766 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1767
1768 if (C->output()->need_stack_bang(framesize))
1769 st->print("# stack bang size=%d\n\t", framesize);
1770
1771 if (VM_Version::use_rop_protection()) {
1772 st->print("ldr zr, [lr]\n\t");
1773 st->print("paciaz\n\t");
1774 }
1775 if (framesize < ((1 << 9) + 2 * wordSize)) {
1776 st->print("sub sp, sp, #%d\n\t", framesize);
1777 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1778 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1779 } else {
1780 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1781 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1782 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1783 st->print("sub sp, sp, rscratch1");
1784 }
1785 if (C->stub_function() == nullptr) {
1786 st->print("\n\t");
1787 st->print("ldr rscratch1, [guard]\n\t");
1788 st->print("dmb ishld\n\t");
1789 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1790 st->print("cmp rscratch1, rscratch2\n\t");
1791 st->print("b.eq skip");
1792 st->print("\n\t");
1793 st->print("blr #nmethod_entry_barrier_stub\n\t");
1794 st->print("b skip\n\t");
1795 st->print("guard: int\n\t");
1796 st->print("\n\t");
1797 st->print("skip:\n\t");
1798 }
1799 }
1800 #endif
1801
1802 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1803 Compile* C = ra_->C;
1804
1805 // n.b. frame size includes space for return pc and rfp
1806 const int framesize = C->output()->frame_size_in_bytes();
1807
1808 if (C->clinit_barrier_on_entry()) {
1809 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1810
1811 Label L_skip_barrier;
1812
1813 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1814 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1815 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1816 __ bind(L_skip_barrier);
1817 }
1818
1819 if (C->max_vector_size() > 0) {
1820 __ reinitialize_ptrue();
1821 }
1822
1823 int bangsize = C->output()->bang_size_in_bytes();
1824 if (C->output()->need_stack_bang(bangsize))
1825 __ generate_stack_overflow_check(bangsize);
1826
1827 __ build_frame(framesize);
1828
1829 if (C->stub_function() == nullptr) {
1830 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1831 // Dummy labels for just measuring the code size
1832 Label dummy_slow_path;
1833 Label dummy_continuation;
1834 Label dummy_guard;
1835 Label* slow_path = &dummy_slow_path;
1836 Label* continuation = &dummy_continuation;
1837 Label* guard = &dummy_guard;
1838 if (!Compile::current()->output()->in_scratch_emit_size()) {
1839 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1840 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1841 Compile::current()->output()->add_stub(stub);
1842 slow_path = &stub->entry();
1843 continuation = &stub->continuation();
1844 guard = &stub->guard();
1845 }
1846 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1847 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1848 }
1849
1850 if (VerifyStackAtCalls) {
1851 Unimplemented();
1852 }
1853
1854 C->output()->set_frame_complete(__ offset());
1855
1856 if (C->has_mach_constant_base_node()) {
1857 // NOTE: We set the table base offset here because users might be
1858 // emitted before MachConstantBaseNode.
1859 ConstantTable& constant_table = C->output()->constant_table();
1860 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1861 }
1862 }
1863
1864 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1865 {
1866 return MachNode::size(ra_); // too many variables; just compute it
1867 // the hard way
1868 }
1869
1870 int MachPrologNode::reloc() const
1871 {
1872 return 0;
1873 }
1874
1875 //=============================================================================
1876
1877 #ifndef PRODUCT
1878 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1879 Compile* C = ra_->C;
1880 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1881
1882 st->print("# pop frame %d\n\t",framesize);
1883
1884 if (framesize == 0) {
1885 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1886 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1887 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1888 st->print("add sp, sp, #%d\n\t", framesize);
1889 } else {
1890 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1891 st->print("add sp, sp, rscratch1\n\t");
1892 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1893 }
1894 if (VM_Version::use_rop_protection()) {
1895 st->print("autiaz\n\t");
1896 st->print("ldr zr, [lr]\n\t");
1897 }
1898
1899 if (do_polling() && C->is_method_compilation()) {
1900 st->print("# test polling word\n\t");
1901 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1902 st->print("cmp sp, rscratch1\n\t");
1903 st->print("bhi #slow_path");
1904 }
1905 }
1906 #endif
1907
1908 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1909 Compile* C = ra_->C;
1910 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1911
1912 __ remove_frame(framesize);
1913
1914 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1915 __ reserved_stack_check();
1916 }
1917
1918 if (do_polling() && C->is_method_compilation()) {
1919 Label dummy_label;
1920 Label* code_stub = &dummy_label;
1921 if (!C->output()->in_scratch_emit_size()) {
1922 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1923 C->output()->add_stub(stub);
1924 code_stub = &stub->entry();
1925 }
1926 __ relocate(relocInfo::poll_return_type);
1927 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1928 }
1929 }
1930
1931 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1932 // Variable size. Determine dynamically.
1933 return MachNode::size(ra_);
1934 }
1935
1936 int MachEpilogNode::reloc() const {
1937 // Return number of relocatable values contained in this instruction.
1938 return 1; // 1 for polling page.
1939 }
1940
1941 const Pipeline * MachEpilogNode::pipeline() const {
1942 return MachNode::pipeline_class();
1943 }
1944
1945 //=============================================================================
1946
1947 static enum RC rc_class(OptoReg::Name reg) {
1948
1949 if (reg == OptoReg::Bad) {
1950 return rc_bad;
1951 }
1952
1953 // we have 32 int registers * 2 halves
1954 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1955
1956 if (reg < slots_of_int_registers) {
1957 return rc_int;
1958 }
1959
1960 // we have 32 float register * 8 halves
1961 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1962 if (reg < slots_of_int_registers + slots_of_float_registers) {
1963 return rc_float;
1964 }
1965
1966 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1967 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1968 return rc_predicate;
1969 }
1970
1971 // Between predicate regs & stack is the flags.
1972 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1973
1974 return rc_stack;
1975 }
1976
1977 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1978 Compile* C = ra_->C;
1979
1980 // Get registers to move.
1981 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1982 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1983 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1984 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1985
1986 enum RC src_hi_rc = rc_class(src_hi);
1987 enum RC src_lo_rc = rc_class(src_lo);
1988 enum RC dst_hi_rc = rc_class(dst_hi);
1989 enum RC dst_lo_rc = rc_class(dst_lo);
1990
1991 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1992
1993 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1994 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1995 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1996 "expected aligned-adjacent pairs");
1997 }
1998
1999 if (src_lo == dst_lo && src_hi == dst_hi) {
2000 return 0; // Self copy, no move.
2001 }
2002
2003 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2004 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2005 int src_offset = ra_->reg2offset(src_lo);
2006 int dst_offset = ra_->reg2offset(dst_lo);
2007
2008 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2009 uint ireg = ideal_reg();
2010 if (ireg == Op_VecA && masm) {
2011 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2012 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2013 // stack->stack
2014 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2015 sve_vector_reg_size_in_bytes);
2016 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2017 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2018 sve_vector_reg_size_in_bytes);
2019 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2020 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2021 sve_vector_reg_size_in_bytes);
2022 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2023 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2024 as_FloatRegister(Matcher::_regEncode[src_lo]),
2025 as_FloatRegister(Matcher::_regEncode[src_lo]));
2026 } else {
2027 ShouldNotReachHere();
2028 }
2029 } else if (masm) {
2030 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2031 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2032 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2033 // stack->stack
2034 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2035 if (ireg == Op_VecD) {
2036 __ unspill(rscratch1, true, src_offset);
2037 __ spill(rscratch1, true, dst_offset);
2038 } else {
2039 __ spill_copy128(src_offset, dst_offset);
2040 }
2041 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2042 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2043 ireg == Op_VecD ? __ T8B : __ T16B,
2044 as_FloatRegister(Matcher::_regEncode[src_lo]));
2045 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2046 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2047 ireg == Op_VecD ? __ D : __ Q,
2048 ra_->reg2offset(dst_lo));
2049 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2050 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2051 ireg == Op_VecD ? __ D : __ Q,
2052 ra_->reg2offset(src_lo));
2053 } else {
2054 ShouldNotReachHere();
2055 }
2056 }
2057 } else if (masm) {
2058 switch (src_lo_rc) {
2059 case rc_int:
2060 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2061 if (is64) {
2062 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2063 as_Register(Matcher::_regEncode[src_lo]));
2064 } else {
2065 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2066 as_Register(Matcher::_regEncode[src_lo]));
2067 }
2068 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2069 if (is64) {
2070 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2071 as_Register(Matcher::_regEncode[src_lo]));
2072 } else {
2073 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2074 as_Register(Matcher::_regEncode[src_lo]));
2075 }
2076 } else { // gpr --> stack spill
2077 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2078 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2079 }
2080 break;
2081 case rc_float:
2082 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2083 if (is64) {
2084 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2085 as_FloatRegister(Matcher::_regEncode[src_lo]));
2086 } else {
2087 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2088 as_FloatRegister(Matcher::_regEncode[src_lo]));
2089 }
2090 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2091 if (is64) {
2092 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2093 as_FloatRegister(Matcher::_regEncode[src_lo]));
2094 } else {
2095 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2096 as_FloatRegister(Matcher::_regEncode[src_lo]));
2097 }
2098 } else { // fpr --> stack spill
2099 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2100 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2101 is64 ? __ D : __ S, dst_offset);
2102 }
2103 break;
2104 case rc_stack:
2105 if (dst_lo_rc == rc_int) { // stack --> gpr load
2106 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2107 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2108 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2109 is64 ? __ D : __ S, src_offset);
2110 } else if (dst_lo_rc == rc_predicate) {
2111 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2112 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2113 } else { // stack --> stack copy
2114 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2115 if (ideal_reg() == Op_RegVectMask) {
2116 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2117 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2118 } else {
2119 __ unspill(rscratch1, is64, src_offset);
2120 __ spill(rscratch1, is64, dst_offset);
2121 }
2122 }
2123 break;
2124 case rc_predicate:
2125 if (dst_lo_rc == rc_predicate) {
2126 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2127 } else if (dst_lo_rc == rc_stack) {
2128 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2129 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2130 } else {
2131 assert(false, "bad src and dst rc_class combination.");
2132 ShouldNotReachHere();
2133 }
2134 break;
2135 default:
2136 assert(false, "bad rc_class for spill");
2137 ShouldNotReachHere();
2138 }
2139 }
2140
2141 if (st) {
2142 st->print("spill ");
2143 if (src_lo_rc == rc_stack) {
2144 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2145 } else {
2146 st->print("%s -> ", Matcher::regName[src_lo]);
2147 }
2148 if (dst_lo_rc == rc_stack) {
2149 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2150 } else {
2151 st->print("%s", Matcher::regName[dst_lo]);
2152 }
2153 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2154 int vsize = 0;
2155 switch (ideal_reg()) {
2156 case Op_VecD:
2157 vsize = 64;
2158 break;
2159 case Op_VecX:
2160 vsize = 128;
2161 break;
2162 case Op_VecA:
2163 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2164 break;
2165 default:
2166 assert(false, "bad register type for spill");
2167 ShouldNotReachHere();
2168 }
2169 st->print("\t# vector spill size = %d", vsize);
2170 } else if (ideal_reg() == Op_RegVectMask) {
2171 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2172 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2173 st->print("\t# predicate spill size = %d", vsize);
2174 } else {
2175 st->print("\t# spill size = %d", is64 ? 64 : 32);
2176 }
2177 }
2178
2179 return 0;
2180
2181 }
2182
2183 #ifndef PRODUCT
2184 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2185 if (!ra_)
2186 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2187 else
2188 implementation(nullptr, ra_, false, st);
2189 }
2190 #endif
2191
2192 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2193 implementation(masm, ra_, false, nullptr);
2194 }
2195
2196 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2197 return MachNode::size(ra_);
2198 }
2199
2200 //=============================================================================
2201
2202 #ifndef PRODUCT
2203 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2204 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2205 int reg = ra_->get_reg_first(this);
2206 st->print("add %s, rsp, #%d]\t# box lock",
2207 Matcher::regName[reg], offset);
2208 }
2209 #endif
2210
2211 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2212 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2213 int reg = ra_->get_encode(this);
2214
2215 // This add will handle any 24-bit signed offset. 24 bits allows an
2216 // 8 megabyte stack frame.
2217 __ add(as_Register(reg), sp, offset);
2218 }
2219
2220 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2221 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2222 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2223
2224 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2225 return NativeInstruction::instruction_size;
2226 } else {
2227 return 2 * NativeInstruction::instruction_size;
2228 }
2229 }
2230
2231 //=============================================================================
2232
2233 #ifndef PRODUCT
2234 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2235 {
2236 st->print_cr("# MachUEPNode");
2237 if (UseCompressedClassPointers) {
2238 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2239 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2240 st->print_cr("\tcmpw rscratch1, r10");
2241 } else {
2242 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2243 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2244 st->print_cr("\tcmp rscratch1, r10");
2245 }
2246 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2247 }
2248 #endif
2249
2250 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2251 {
2252 __ ic_check(InteriorEntryAlignment);
2253 }
2254
2255 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2256 {
2257 return MachNode::size(ra_);
2258 }
2259
2260 // REQUIRED EMIT CODE
2261
2262 //=============================================================================
2263
2264 // Emit exception handler code.
2265 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2266 {
2267 // mov rscratch1 #exception_blob_entry_point
2268 // br rscratch1
2269 // Note that the code buffer's insts_mark is always relative to insts.
2270 // That's why we must use the macroassembler to generate a handler.
2271 address base = __ start_a_stub(size_exception_handler());
2272 if (base == nullptr) {
2273 ciEnv::current()->record_failure("CodeCache is full");
2274 return 0; // CodeBuffer::expand failed
2275 }
2276 int offset = __ offset();
2277 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2278 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2279 __ end_a_stub();
2280 return offset;
2281 }
2282
2283 // Emit deopt handler code.
2284 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2285 {
2286 // Note that the code buffer's insts_mark is always relative to insts.
2287 // That's why we must use the macroassembler to generate a handler.
2288 address base = __ start_a_stub(size_deopt_handler());
2289 if (base == nullptr) {
2290 ciEnv::current()->record_failure("CodeCache is full");
2291 return 0; // CodeBuffer::expand failed
2292 }
2293 int offset = __ offset();
2294
2295 __ adr(lr, __ pc());
2296 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2297
2298 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2299 __ end_a_stub();
2300 return offset;
2301 }
2302
2303 // REQUIRED MATCHER CODE
2304
2305 //=============================================================================
2306
2307 bool Matcher::match_rule_supported(int opcode) {
2308 if (!has_match_rule(opcode))
2309 return false;
2310
2311 switch (opcode) {
2312 case Op_OnSpinWait:
2313 return VM_Version::supports_on_spin_wait();
2314 case Op_CacheWB:
2315 case Op_CacheWBPreSync:
2316 case Op_CacheWBPostSync:
2317 if (!VM_Version::supports_data_cache_line_flush()) {
2318 return false;
2319 }
2320 break;
2321 case Op_ExpandBits:
2322 case Op_CompressBits:
2323 if (!VM_Version::supports_svebitperm()) {
2324 return false;
2325 }
2326 break;
2327 case Op_FmaF:
2328 case Op_FmaD:
2329 case Op_FmaVF:
2330 case Op_FmaVD:
2331 if (!UseFMA) {
2332 return false;
2333 }
2334 break;
2335 case Op_FmaHF:
2336 // UseFMA flag also needs to be checked along with FEAT_FP16
2337 if (!UseFMA || !is_feat_fp16_supported()) {
2338 return false;
2339 }
2340 break;
2341 case Op_AddHF:
2342 case Op_SubHF:
2343 case Op_MulHF:
2344 case Op_DivHF:
2345 case Op_MinHF:
2346 case Op_MaxHF:
2347 case Op_SqrtHF:
2348 // Half-precision floating point scalar operations require FEAT_FP16
2349 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2350 // features are supported.
2351 if (!is_feat_fp16_supported()) {
2352 return false;
2353 }
2354 break;
2355 }
2356
2357 return true; // Per default match rules are supported.
2358 }
2359
2360 const RegMask* Matcher::predicate_reg_mask(void) {
2361 return &_PR_REG_mask;
2362 }
2363
2364 bool Matcher::supports_vector_calling_convention(void) {
2365 return EnableVectorSupport;
2366 }
2367
2368 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2369 assert(EnableVectorSupport, "sanity");
2370 int lo = V0_num;
2371 int hi = V0_H_num;
2372 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2373 hi = V0_K_num;
2374 }
2375 return OptoRegPair(hi, lo);
2376 }
2377
2378 // Is this branch offset short enough that a short branch can be used?
2379 //
2380 // NOTE: If the platform does not provide any short branch variants, then
2381 // this method should return false for offset 0.
2382 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2383 // The passed offset is relative to address of the branch.
2384
2385 return (-32768 <= offset && offset < 32768);
2386 }
2387
2388 // Vector width in bytes.
2389 int Matcher::vector_width_in_bytes(BasicType bt) {
2390 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2391 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2392 // Minimum 2 values in vector
2393 if (size < 2*type2aelembytes(bt)) size = 0;
2394 // But never < 4
2395 if (size < 4) size = 0;
2396 return size;
2397 }
2398
2399 // Limits on vector size (number of elements) loaded into vector.
2400 int Matcher::max_vector_size(const BasicType bt) {
2401 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2402 }
2403
2404 int Matcher::min_vector_size(const BasicType bt) {
2405 // Usually, the shortest vector length supported by AArch64 ISA and
2406 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2407 // vectors in a few special cases.
2408 int size;
2409 switch(bt) {
2410 case T_BOOLEAN:
2411 // Load/store a vector mask with only 2 elements for vector types
2412 // such as "2I/2F/2L/2D".
2413 size = 2;
2414 break;
2415 case T_BYTE:
2416 // Generate a "4B" vector, to support vector cast between "8B/16B"
2417 // and "4S/4I/4L/4F/4D".
2418 size = 4;
2419 break;
2420 case T_SHORT:
2421 // Generate a "2S" vector, to support vector cast between "4S/8S"
2422 // and "2I/2L/2F/2D".
2423 size = 2;
2424 break;
2425 default:
2426 // Limit the min vector length to 64-bit.
2427 size = 8 / type2aelembytes(bt);
2428 // The number of elements in a vector should be at least 2.
2429 size = MAX2(size, 2);
2430 }
2431
2432 int max_size = max_vector_size(bt);
2433 return MIN2(size, max_size);
2434 }
2435
2436 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2437 return Matcher::max_vector_size(bt);
2438 }
2439
2440 // Actual max scalable vector register length.
2441 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2442 return Matcher::max_vector_size(bt);
2443 }
2444
2445 // Vector ideal reg.
2446 uint Matcher::vector_ideal_reg(int len) {
2447 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2448 return Op_VecA;
2449 }
2450 switch(len) {
2451 // For 16-bit/32-bit mask vector, reuse VecD.
2452 case 2:
2453 case 4:
2454 case 8: return Op_VecD;
2455 case 16: return Op_VecX;
2456 }
2457 ShouldNotReachHere();
2458 return 0;
2459 }
2460
2461 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2462 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2463 switch (ideal_reg) {
2464 case Op_VecA: return new vecAOper();
2465 case Op_VecD: return new vecDOper();
2466 case Op_VecX: return new vecXOper();
2467 }
2468 ShouldNotReachHere();
2469 return nullptr;
2470 }
2471
2472 bool Matcher::is_reg2reg_move(MachNode* m) {
2473 return false;
2474 }
2475
2476 bool Matcher::is_generic_vector(MachOper* opnd) {
2477 return opnd->opcode() == VREG;
2478 }
2479
2480 // Return whether or not this register is ever used as an argument.
2481 // This function is used on startup to build the trampoline stubs in
2482 // generateOptoStub. Registers not mentioned will be killed by the VM
2483 // call in the trampoline, and arguments in those registers not be
2484 // available to the callee.
2485 bool Matcher::can_be_java_arg(int reg)
2486 {
2487 return
2488 reg == R0_num || reg == R0_H_num ||
2489 reg == R1_num || reg == R1_H_num ||
2490 reg == R2_num || reg == R2_H_num ||
2491 reg == R3_num || reg == R3_H_num ||
2492 reg == R4_num || reg == R4_H_num ||
2493 reg == R5_num || reg == R5_H_num ||
2494 reg == R6_num || reg == R6_H_num ||
2495 reg == R7_num || reg == R7_H_num ||
2496 reg == V0_num || reg == V0_H_num ||
2497 reg == V1_num || reg == V1_H_num ||
2498 reg == V2_num || reg == V2_H_num ||
2499 reg == V3_num || reg == V3_H_num ||
2500 reg == V4_num || reg == V4_H_num ||
2501 reg == V5_num || reg == V5_H_num ||
2502 reg == V6_num || reg == V6_H_num ||
2503 reg == V7_num || reg == V7_H_num;
2504 }
2505
2506 bool Matcher::is_spillable_arg(int reg)
2507 {
2508 return can_be_java_arg(reg);
2509 }
2510
2511 uint Matcher::int_pressure_limit()
2512 {
2513 // JDK-8183543: When taking the number of available registers as int
2514 // register pressure threshold, the jtreg test:
2515 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2516 // failed due to C2 compilation failure with
2517 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2518 //
2519 // A derived pointer is live at CallNode and then is flagged by RA
2520 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2521 // derived pointers and lastly fail to spill after reaching maximum
2522 // number of iterations. Lowering the default pressure threshold to
2523 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2524 // a high register pressure area of the code so that split_DEF can
2525 // generate DefinitionSpillCopy for the derived pointer.
2526 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2527 if (!PreserveFramePointer) {
2528 // When PreserveFramePointer is off, frame pointer is allocatable,
2529 // but different from other SOC registers, it is excluded from
2530 // fatproj's mask because its save type is No-Save. Decrease 1 to
2531 // ensure high pressure at fatproj when PreserveFramePointer is off.
2532 // See check_pressure_at_fatproj().
2533 default_int_pressure_threshold--;
2534 }
2535 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2536 }
2537
2538 uint Matcher::float_pressure_limit()
2539 {
2540 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2541 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2542 }
2543
2544 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2545 return false;
2546 }
2547
2548 RegMask Matcher::divI_proj_mask() {
2549 ShouldNotReachHere();
2550 return RegMask();
2551 }
2552
2553 // Register for MODI projection of divmodI.
2554 RegMask Matcher::modI_proj_mask() {
2555 ShouldNotReachHere();
2556 return RegMask();
2557 }
2558
2559 // Register for DIVL projection of divmodL.
2560 RegMask Matcher::divL_proj_mask() {
2561 ShouldNotReachHere();
2562 return RegMask();
2563 }
2564
2565 // Register for MODL projection of divmodL.
2566 RegMask Matcher::modL_proj_mask() {
2567 ShouldNotReachHere();
2568 return RegMask();
2569 }
2570
2571 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2572 return FP_REG_mask();
2573 }
2574
2575 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2576 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2577 Node* u = addp->fast_out(i);
2578 if (u->is_LoadStore()) {
2579 // On AArch64, LoadStoreNodes (i.e. compare and swap
2580 // instructions) only take register indirect as an operand, so
2581 // any attempt to use an AddPNode as an input to a LoadStoreNode
2582 // must fail.
2583 return false;
2584 }
2585 if (u->is_Mem()) {
2586 int opsize = u->as_Mem()->memory_size();
2587 assert(opsize > 0, "unexpected memory operand size");
2588 if (u->as_Mem()->memory_size() != (1<<shift)) {
2589 return false;
2590 }
2591 }
2592 }
2593 return true;
2594 }
2595
2596 // Convert BootTest condition to Assembler condition.
2597 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2598 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2599 Assembler::Condition result;
2600 switch(cond) {
2601 case BoolTest::eq:
2602 result = Assembler::EQ; break;
2603 case BoolTest::ne:
2604 result = Assembler::NE; break;
2605 case BoolTest::le:
2606 result = Assembler::LE; break;
2607 case BoolTest::ge:
2608 result = Assembler::GE; break;
2609 case BoolTest::lt:
2610 result = Assembler::LT; break;
2611 case BoolTest::gt:
2612 result = Assembler::GT; break;
2613 case BoolTest::ule:
2614 result = Assembler::LS; break;
2615 case BoolTest::uge:
2616 result = Assembler::HS; break;
2617 case BoolTest::ult:
2618 result = Assembler::LO; break;
2619 case BoolTest::ugt:
2620 result = Assembler::HI; break;
2621 case BoolTest::overflow:
2622 result = Assembler::VS; break;
2623 case BoolTest::no_overflow:
2624 result = Assembler::VC; break;
2625 default:
2626 ShouldNotReachHere();
2627 return Assembler::Condition(-1);
2628 }
2629
2630 // Check conversion
2631 if (cond & BoolTest::unsigned_compare) {
2632 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2633 } else {
2634 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2635 }
2636
2637 return result;
2638 }
2639
2640 // Binary src (Replicate con)
2641 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2642 if (n == nullptr || m == nullptr) {
2643 return false;
2644 }
2645
2646 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2647 return false;
2648 }
2649
2650 Node* imm_node = m->in(1);
2651 if (!imm_node->is_Con()) {
2652 return false;
2653 }
2654
2655 const Type* t = imm_node->bottom_type();
2656 if (!(t->isa_int() || t->isa_long())) {
2657 return false;
2658 }
2659
2660 switch (n->Opcode()) {
2661 case Op_AndV:
2662 case Op_OrV:
2663 case Op_XorV: {
2664 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2665 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2666 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2667 }
2668 case Op_AddVB:
2669 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2670 case Op_AddVS:
2671 case Op_AddVI:
2672 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2673 case Op_AddVL:
2674 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2675 default:
2676 return false;
2677 }
2678 }
2679
2680 // (XorV src (Replicate m1))
2681 // (XorVMask src (MaskAll m1))
2682 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2683 if (n != nullptr && m != nullptr) {
2684 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2685 VectorNode::is_all_ones_vector(m);
2686 }
2687 return false;
2688 }
2689
2690 // Should the matcher clone input 'm' of node 'n'?
2691 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2692 if (is_vshift_con_pattern(n, m) ||
2693 is_vector_bitwise_not_pattern(n, m) ||
2694 is_valid_sve_arith_imm_pattern(n, m) ||
2695 is_encode_and_store_pattern(n, m)) {
2696 mstack.push(m, Visit);
2697 return true;
2698 }
2699 return false;
2700 }
2701
2702 // Should the Matcher clone shifts on addressing modes, expecting them
2703 // to be subsumed into complex addressing expressions or compute them
2704 // into registers?
2705 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2706
2707 // Loads and stores with indirect memory input (e.g., volatile loads and
2708 // stores) do not subsume the input into complex addressing expressions. If
2709 // the addressing expression is input to at least one such load or store, do
2710 // not clone the addressing expression. Query needs_acquiring_load and
2711 // needs_releasing_store as a proxy for indirect memory input, as it is not
2712 // possible to directly query for indirect memory input at this stage.
2713 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2714 Node* n = m->fast_out(i);
2715 if (n->is_Load() && needs_acquiring_load(n)) {
2716 return false;
2717 }
2718 if (n->is_Store() && needs_releasing_store(n)) {
2719 return false;
2720 }
2721 }
2722
2723 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2724 return true;
2725 }
2726
2727 Node *off = m->in(AddPNode::Offset);
2728 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2729 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2730 // Are there other uses besides address expressions?
2731 !is_visited(off)) {
2732 address_visited.set(off->_idx); // Flag as address_visited
2733 mstack.push(off->in(2), Visit);
2734 Node *conv = off->in(1);
2735 if (conv->Opcode() == Op_ConvI2L &&
2736 // Are there other uses besides address expressions?
2737 !is_visited(conv)) {
2738 address_visited.set(conv->_idx); // Flag as address_visited
2739 mstack.push(conv->in(1), Pre_Visit);
2740 } else {
2741 mstack.push(conv, Pre_Visit);
2742 }
2743 address_visited.test_set(m->_idx); // Flag as address_visited
2744 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2745 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2746 return true;
2747 } else if (off->Opcode() == Op_ConvI2L &&
2748 // Are there other uses besides address expressions?
2749 !is_visited(off)) {
2750 address_visited.test_set(m->_idx); // Flag as address_visited
2751 address_visited.set(off->_idx); // Flag as address_visited
2752 mstack.push(off->in(1), Pre_Visit);
2753 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2754 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2755 return true;
2756 }
2757 return false;
2758 }
2759
2760 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2761 { \
2762 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2763 guarantee(DISP == 0, "mode not permitted for volatile"); \
2764 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2765 __ INSN(REG, as_Register(BASE)); \
2766 }
2767
2768
2769 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2770 {
2771 Address::extend scale;
2772
2773 // Hooboy, this is fugly. We need a way to communicate to the
2774 // encoder that the index needs to be sign extended, so we have to
2775 // enumerate all the cases.
2776 switch (opcode) {
2777 case INDINDEXSCALEDI2L:
2778 case INDINDEXSCALEDI2LN:
2779 case INDINDEXI2L:
2780 case INDINDEXI2LN:
2781 scale = Address::sxtw(size);
2782 break;
2783 default:
2784 scale = Address::lsl(size);
2785 }
2786
2787 if (index == -1) {
2788 return Address(base, disp);
2789 } else {
2790 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2791 return Address(base, as_Register(index), scale);
2792 }
2793 }
2794
2795
2796 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2797 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2798 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2799 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2800 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2801
2802 // Used for all non-volatile memory accesses. The use of
2803 // $mem->opcode() to discover whether this pattern uses sign-extended
2804 // offsets is something of a kludge.
2805 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2806 Register reg, int opcode,
2807 Register base, int index, int scale, int disp,
2808 int size_in_memory)
2809 {
2810 Address addr = mem2address(opcode, base, index, scale, disp);
2811 if (addr.getMode() == Address::base_plus_offset) {
2812 /* Fix up any out-of-range offsets. */
2813 assert_different_registers(rscratch1, base);
2814 assert_different_registers(rscratch1, reg);
2815 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2816 }
2817 (masm->*insn)(reg, addr);
2818 }
2819
2820 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2821 FloatRegister reg, int opcode,
2822 Register base, int index, int size, int disp,
2823 int size_in_memory)
2824 {
2825 Address::extend scale;
2826
2827 switch (opcode) {
2828 case INDINDEXSCALEDI2L:
2829 case INDINDEXSCALEDI2LN:
2830 scale = Address::sxtw(size);
2831 break;
2832 default:
2833 scale = Address::lsl(size);
2834 }
2835
2836 if (index == -1) {
2837 // Fix up any out-of-range offsets.
2838 assert_different_registers(rscratch1, base);
2839 Address addr = Address(base, disp);
2840 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2841 (masm->*insn)(reg, addr);
2842 } else {
2843 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2844 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2845 }
2846 }
2847
2848 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2849 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2850 int opcode, Register base, int index, int size, int disp)
2851 {
2852 if (index == -1) {
2853 (masm->*insn)(reg, T, Address(base, disp));
2854 } else {
2855 assert(disp == 0, "unsupported address mode");
2856 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2857 }
2858 }
2859
2860 %}
2861
2862
2863
2864 //----------ENCODING BLOCK-----------------------------------------------------
2865 // This block specifies the encoding classes used by the compiler to
2866 // output byte streams. Encoding classes are parameterized macros
2867 // used by Machine Instruction Nodes in order to generate the bit
2868 // encoding of the instruction. Operands specify their base encoding
2869 // interface with the interface keyword. There are currently
2870 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2871 // COND_INTER. REG_INTER causes an operand to generate a function
2872 // which returns its register number when queried. CONST_INTER causes
2873 // an operand to generate a function which returns the value of the
2874 // constant when queried. MEMORY_INTER causes an operand to generate
2875 // four functions which return the Base Register, the Index Register,
2876 // the Scale Value, and the Offset Value of the operand when queried.
2877 // COND_INTER causes an operand to generate six functions which return
2878 // the encoding code (ie - encoding bits for the instruction)
2879 // associated with each basic boolean condition for a conditional
2880 // instruction.
2881 //
2882 // Instructions specify two basic values for encoding. Again, a
2883 // function is available to check if the constant displacement is an
2884 // oop. They use the ins_encode keyword to specify their encoding
2885 // classes (which must be a sequence of enc_class names, and their
2886 // parameters, specified in the encoding block), and they use the
2887 // opcode keyword to specify, in order, their primary, secondary, and
2888 // tertiary opcode. Only the opcode sections which a particular
2889 // instruction needs for encoding need to be specified.
2890 encode %{
2891 // Build emit functions for each basic byte or larger field in the
2892 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2893 // from C++ code in the enc_class source block. Emit functions will
2894 // live in the main source block for now. In future, we can
2895 // generalize this by adding a syntax that specifies the sizes of
2896 // fields in an order, so that the adlc can build the emit functions
2897 // automagically
2898
2899 // catch all for unimplemented encodings
2900 enc_class enc_unimplemented %{
2901 __ unimplemented("C2 catch all");
2902 %}
2903
2904 // BEGIN Non-volatile memory access
2905
2906 // This encoding class is generated automatically from ad_encode.m4.
2907 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2908 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2909 Register dst_reg = as_Register($dst$$reg);
2910 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2911 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2912 %}
2913
2914 // This encoding class is generated automatically from ad_encode.m4.
2915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2916 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2917 Register dst_reg = as_Register($dst$$reg);
2918 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2919 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2920 %}
2921
2922 // This encoding class is generated automatically from ad_encode.m4.
2923 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2924 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2925 Register dst_reg = as_Register($dst$$reg);
2926 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2927 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2928 %}
2929
2930 // This encoding class is generated automatically from ad_encode.m4.
2931 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2932 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2933 Register dst_reg = as_Register($dst$$reg);
2934 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2935 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2936 %}
2937
2938 // This encoding class is generated automatically from ad_encode.m4.
2939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2940 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2941 Register dst_reg = as_Register($dst$$reg);
2942 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2943 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2944 %}
2945
2946 // This encoding class is generated automatically from ad_encode.m4.
2947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2948 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2949 Register dst_reg = as_Register($dst$$reg);
2950 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2951 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2952 %}
2953
2954 // This encoding class is generated automatically from ad_encode.m4.
2955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2956 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2957 Register dst_reg = as_Register($dst$$reg);
2958 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2959 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2960 %}
2961
2962 // This encoding class is generated automatically from ad_encode.m4.
2963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2964 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2965 Register dst_reg = as_Register($dst$$reg);
2966 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2967 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2968 %}
2969
2970 // This encoding class is generated automatically from ad_encode.m4.
2971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2972 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2973 Register dst_reg = as_Register($dst$$reg);
2974 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2975 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2976 %}
2977
2978 // This encoding class is generated automatically from ad_encode.m4.
2979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2980 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2981 Register dst_reg = as_Register($dst$$reg);
2982 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2983 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2984 %}
2985
2986 // This encoding class is generated automatically from ad_encode.m4.
2987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2988 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2989 Register dst_reg = as_Register($dst$$reg);
2990 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2991 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2992 %}
2993
2994 // This encoding class is generated automatically from ad_encode.m4.
2995 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2996 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2997 Register dst_reg = as_Register($dst$$reg);
2998 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2999 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3000 %}
3001
3002 // This encoding class is generated automatically from ad_encode.m4.
3003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3004 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
3005 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3006 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
3007 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3008 %}
3009
3010 // This encoding class is generated automatically from ad_encode.m4.
3011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3012 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
3013 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3014 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
3015 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3016 %}
3017
3018 // This encoding class is generated automatically from ad_encode.m4.
3019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3020 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3021 Register src_reg = as_Register($src$$reg);
3022 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3023 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3024 %}
3025
3026 // This encoding class is generated automatically from ad_encode.m4.
3027 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3028 enc_class aarch64_enc_strb0(memory1 mem) %{
3029 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3030 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3031 %}
3032
3033 // This encoding class is generated automatically from ad_encode.m4.
3034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3035 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3036 Register src_reg = as_Register($src$$reg);
3037 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3038 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3039 %}
3040
3041 // This encoding class is generated automatically from ad_encode.m4.
3042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3043 enc_class aarch64_enc_strh0(memory2 mem) %{
3044 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3045 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3046 %}
3047
3048 // This encoding class is generated automatically from ad_encode.m4.
3049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3050 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3051 Register src_reg = as_Register($src$$reg);
3052 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3053 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3054 %}
3055
3056 // This encoding class is generated automatically from ad_encode.m4.
3057 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3058 enc_class aarch64_enc_strw0(memory4 mem) %{
3059 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3060 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3061 %}
3062
3063 // This encoding class is generated automatically from ad_encode.m4.
3064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3065 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3066 Register src_reg = as_Register($src$$reg);
3067 // we sometimes get asked to store the stack pointer into the
3068 // current thread -- we cannot do that directly on AArch64
3069 if (src_reg == r31_sp) {
3070 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3071 __ mov(rscratch2, sp);
3072 src_reg = rscratch2;
3073 }
3074 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3075 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3076 %}
3077
3078 // This encoding class is generated automatically from ad_encode.m4.
3079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3080 enc_class aarch64_enc_str0(memory8 mem) %{
3081 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3082 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3083 %}
3084
3085 // This encoding class is generated automatically from ad_encode.m4.
3086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3087 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3088 FloatRegister src_reg = as_FloatRegister($src$$reg);
3089 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3090 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3091 %}
3092
3093 // This encoding class is generated automatically from ad_encode.m4.
3094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3095 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3096 FloatRegister src_reg = as_FloatRegister($src$$reg);
3097 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3098 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3099 %}
3100
3101 // This encoding class is generated automatically from ad_encode.m4.
3102 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3103 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3104 __ membar(Assembler::StoreStore);
3105 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3106 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3107 %}
3108
3109 // END Non-volatile memory access
3110
3111 // Vector loads and stores
3112 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3113 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3114 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3115 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3116 %}
3117
3118 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3119 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3120 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3121 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3122 %}
3123
3124 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3125 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3126 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3127 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3128 %}
3129
3130 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3131 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3132 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3133 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3134 %}
3135
3136 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3137 FloatRegister src_reg = as_FloatRegister($src$$reg);
3138 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3139 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3140 %}
3141
3142 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3143 FloatRegister src_reg = as_FloatRegister($src$$reg);
3144 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3145 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3146 %}
3147
3148 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3149 FloatRegister src_reg = as_FloatRegister($src$$reg);
3150 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3151 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3152 %}
3153
3154 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3155 FloatRegister src_reg = as_FloatRegister($src$$reg);
3156 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3157 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3158 %}
3159
3160 // volatile loads and stores
3161
3162 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3163 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3164 rscratch1, stlrb);
3165 %}
3166
3167 enc_class aarch64_enc_stlrb0(memory mem) %{
3168 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3169 rscratch1, stlrb);
3170 %}
3171
3172 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3173 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3174 rscratch1, stlrh);
3175 %}
3176
3177 enc_class aarch64_enc_stlrh0(memory mem) %{
3178 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3179 rscratch1, stlrh);
3180 %}
3181
3182 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3183 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3184 rscratch1, stlrw);
3185 %}
3186
3187 enc_class aarch64_enc_stlrw0(memory mem) %{
3188 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3189 rscratch1, stlrw);
3190 %}
3191
3192 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3193 Register dst_reg = as_Register($dst$$reg);
3194 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3195 rscratch1, ldarb);
3196 __ sxtbw(dst_reg, dst_reg);
3197 %}
3198
3199 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3200 Register dst_reg = as_Register($dst$$reg);
3201 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3202 rscratch1, ldarb);
3203 __ sxtb(dst_reg, dst_reg);
3204 %}
3205
3206 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3207 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3208 rscratch1, ldarb);
3209 %}
3210
3211 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3212 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3213 rscratch1, ldarb);
3214 %}
3215
3216 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3217 Register dst_reg = as_Register($dst$$reg);
3218 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3219 rscratch1, ldarh);
3220 __ sxthw(dst_reg, dst_reg);
3221 %}
3222
3223 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3224 Register dst_reg = as_Register($dst$$reg);
3225 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3226 rscratch1, ldarh);
3227 __ sxth(dst_reg, dst_reg);
3228 %}
3229
3230 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3231 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3232 rscratch1, ldarh);
3233 %}
3234
3235 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3236 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3237 rscratch1, ldarh);
3238 %}
3239
3240 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3241 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3242 rscratch1, ldarw);
3243 %}
3244
3245 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3246 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3247 rscratch1, ldarw);
3248 %}
3249
3250 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3251 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3252 rscratch1, ldar);
3253 %}
3254
3255 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3256 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3257 rscratch1, ldarw);
3258 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3259 %}
3260
3261 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3262 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3263 rscratch1, ldar);
3264 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3265 %}
3266
3267 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3268 Register src_reg = as_Register($src$$reg);
3269 // we sometimes get asked to store the stack pointer into the
3270 // current thread -- we cannot do that directly on AArch64
3271 if (src_reg == r31_sp) {
3272 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3273 __ mov(rscratch2, sp);
3274 src_reg = rscratch2;
3275 }
3276 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3277 rscratch1, stlr);
3278 %}
3279
3280 enc_class aarch64_enc_stlr0(memory mem) %{
3281 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3282 rscratch1, stlr);
3283 %}
3284
3285 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3286 {
3287 FloatRegister src_reg = as_FloatRegister($src$$reg);
3288 __ fmovs(rscratch2, src_reg);
3289 }
3290 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3291 rscratch1, stlrw);
3292 %}
3293
3294 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3295 {
3296 FloatRegister src_reg = as_FloatRegister($src$$reg);
3297 __ fmovd(rscratch2, src_reg);
3298 }
3299 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3300 rscratch1, stlr);
3301 %}
3302
3303 // synchronized read/update encodings
3304
3305 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3306 Register dst_reg = as_Register($dst$$reg);
3307 Register base = as_Register($mem$$base);
3308 int index = $mem$$index;
3309 int scale = $mem$$scale;
3310 int disp = $mem$$disp;
3311 if (index == -1) {
3312 if (disp != 0) {
3313 __ lea(rscratch1, Address(base, disp));
3314 __ ldaxr(dst_reg, rscratch1);
3315 } else {
3316 // TODO
3317 // should we ever get anything other than this case?
3318 __ ldaxr(dst_reg, base);
3319 }
3320 } else {
3321 Register index_reg = as_Register(index);
3322 if (disp == 0) {
3323 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3324 __ ldaxr(dst_reg, rscratch1);
3325 } else {
3326 __ lea(rscratch1, Address(base, disp));
3327 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3328 __ ldaxr(dst_reg, rscratch1);
3329 }
3330 }
3331 %}
3332
3333 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3334 Register src_reg = as_Register($src$$reg);
3335 Register base = as_Register($mem$$base);
3336 int index = $mem$$index;
3337 int scale = $mem$$scale;
3338 int disp = $mem$$disp;
3339 if (index == -1) {
3340 if (disp != 0) {
3341 __ lea(rscratch2, Address(base, disp));
3342 __ stlxr(rscratch1, src_reg, rscratch2);
3343 } else {
3344 // TODO
3345 // should we ever get anything other than this case?
3346 __ stlxr(rscratch1, src_reg, base);
3347 }
3348 } else {
3349 Register index_reg = as_Register(index);
3350 if (disp == 0) {
3351 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3352 __ stlxr(rscratch1, src_reg, rscratch2);
3353 } else {
3354 __ lea(rscratch2, Address(base, disp));
3355 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3356 __ stlxr(rscratch1, src_reg, rscratch2);
3357 }
3358 }
3359 __ cmpw(rscratch1, zr);
3360 %}
3361
3362 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3363 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3364 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3365 Assembler::xword, /*acquire*/ false, /*release*/ true,
3366 /*weak*/ false, noreg);
3367 %}
3368
3369 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3370 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3371 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3372 Assembler::word, /*acquire*/ false, /*release*/ true,
3373 /*weak*/ false, noreg);
3374 %}
3375
3376 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3377 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3378 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3379 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3380 /*weak*/ false, noreg);
3381 %}
3382
3383 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3384 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3385 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3386 Assembler::byte, /*acquire*/ false, /*release*/ true,
3387 /*weak*/ false, noreg);
3388 %}
3389
3390
3391 // The only difference between aarch64_enc_cmpxchg and
3392 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3393 // CompareAndSwap sequence to serve as a barrier on acquiring a
3394 // lock.
3395 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3396 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3397 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3398 Assembler::xword, /*acquire*/ true, /*release*/ true,
3399 /*weak*/ false, noreg);
3400 %}
3401
3402 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3403 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3404 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3405 Assembler::word, /*acquire*/ true, /*release*/ true,
3406 /*weak*/ false, noreg);
3407 %}
3408
3409 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3410 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3411 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3412 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3413 /*weak*/ false, noreg);
3414 %}
3415
3416 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3417 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3418 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3419 Assembler::byte, /*acquire*/ true, /*release*/ true,
3420 /*weak*/ false, noreg);
3421 %}
3422
3423 // auxiliary used for CompareAndSwapX to set result register
3424 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3425 Register res_reg = as_Register($res$$reg);
3426 __ cset(res_reg, Assembler::EQ);
3427 %}
3428
3429 // prefetch encodings
3430
3431 enc_class aarch64_enc_prefetchw(memory mem) %{
3432 Register base = as_Register($mem$$base);
3433 int index = $mem$$index;
3434 int scale = $mem$$scale;
3435 int disp = $mem$$disp;
3436 if (index == -1) {
3437 // Fix up any out-of-range offsets.
3438 assert_different_registers(rscratch1, base);
3439 Address addr = Address(base, disp);
3440 addr = __ legitimize_address(addr, 8, rscratch1);
3441 __ prfm(addr, PSTL1KEEP);
3442 } else {
3443 Register index_reg = as_Register(index);
3444 if (disp == 0) {
3445 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3446 } else {
3447 __ lea(rscratch1, Address(base, disp));
3448 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3449 }
3450 }
3451 %}
3452
3453 // mov encodings
3454
3455 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3456 uint32_t con = (uint32_t)$src$$constant;
3457 Register dst_reg = as_Register($dst$$reg);
3458 if (con == 0) {
3459 __ movw(dst_reg, zr);
3460 } else {
3461 __ movw(dst_reg, con);
3462 }
3463 %}
3464
3465 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3466 Register dst_reg = as_Register($dst$$reg);
3467 uint64_t con = (uint64_t)$src$$constant;
3468 if (con == 0) {
3469 __ mov(dst_reg, zr);
3470 } else {
3471 __ mov(dst_reg, con);
3472 }
3473 %}
3474
3475 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3476 Register dst_reg = as_Register($dst$$reg);
3477 address con = (address)$src$$constant;
3478 if (con == nullptr || con == (address)1) {
3479 ShouldNotReachHere();
3480 } else {
3481 relocInfo::relocType rtype = $src->constant_reloc();
3482 if (rtype == relocInfo::oop_type) {
3483 __ movoop(dst_reg, (jobject)con);
3484 } else if (rtype == relocInfo::metadata_type) {
3485 __ mov_metadata(dst_reg, (Metadata*)con);
3486 } else {
3487 assert(rtype == relocInfo::none, "unexpected reloc type");
3488 if (! __ is_valid_AArch64_address(con) ||
3489 con < (address)(uintptr_t)os::vm_page_size()) {
3490 __ mov(dst_reg, con);
3491 } else {
3492 uint64_t offset;
3493 __ adrp(dst_reg, con, offset);
3494 __ add(dst_reg, dst_reg, offset);
3495 }
3496 }
3497 }
3498 %}
3499
3500 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3501 Register dst_reg = as_Register($dst$$reg);
3502 __ mov(dst_reg, zr);
3503 %}
3504
3505 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3506 Register dst_reg = as_Register($dst$$reg);
3507 __ mov(dst_reg, (uint64_t)1);
3508 %}
3509
3510 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3511 Register dst_reg = as_Register($dst$$reg);
3512 address con = (address)$src$$constant;
3513 if (con == nullptr) {
3514 ShouldNotReachHere();
3515 } else {
3516 relocInfo::relocType rtype = $src->constant_reloc();
3517 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3518 __ set_narrow_oop(dst_reg, (jobject)con);
3519 }
3520 %}
3521
3522 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3523 Register dst_reg = as_Register($dst$$reg);
3524 __ mov(dst_reg, zr);
3525 %}
3526
3527 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3528 Register dst_reg = as_Register($dst$$reg);
3529 address con = (address)$src$$constant;
3530 if (con == nullptr) {
3531 ShouldNotReachHere();
3532 } else {
3533 relocInfo::relocType rtype = $src->constant_reloc();
3534 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3535 __ set_narrow_klass(dst_reg, (Klass *)con);
3536 }
3537 %}
3538
3539 // arithmetic encodings
3540
3541 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3542 Register dst_reg = as_Register($dst$$reg);
3543 Register src_reg = as_Register($src1$$reg);
3544 int32_t con = (int32_t)$src2$$constant;
3545 // add has primary == 0, subtract has primary == 1
3546 if ($primary) { con = -con; }
3547 if (con < 0) {
3548 __ subw(dst_reg, src_reg, -con);
3549 } else {
3550 __ addw(dst_reg, src_reg, con);
3551 }
3552 %}
3553
3554 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3555 Register dst_reg = as_Register($dst$$reg);
3556 Register src_reg = as_Register($src1$$reg);
3557 int32_t con = (int32_t)$src2$$constant;
3558 // add has primary == 0, subtract has primary == 1
3559 if ($primary) { con = -con; }
3560 if (con < 0) {
3561 __ sub(dst_reg, src_reg, -con);
3562 } else {
3563 __ add(dst_reg, src_reg, con);
3564 }
3565 %}
3566
3567 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3568 Register dst_reg = as_Register($dst$$reg);
3569 Register src1_reg = as_Register($src1$$reg);
3570 Register src2_reg = as_Register($src2$$reg);
3571 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3572 %}
3573
3574 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3575 Register dst_reg = as_Register($dst$$reg);
3576 Register src1_reg = as_Register($src1$$reg);
3577 Register src2_reg = as_Register($src2$$reg);
3578 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3579 %}
3580
3581 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3582 Register dst_reg = as_Register($dst$$reg);
3583 Register src1_reg = as_Register($src1$$reg);
3584 Register src2_reg = as_Register($src2$$reg);
3585 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3586 %}
3587
3588 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3589 Register dst_reg = as_Register($dst$$reg);
3590 Register src1_reg = as_Register($src1$$reg);
3591 Register src2_reg = as_Register($src2$$reg);
3592 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3593 %}
3594
3595 // compare instruction encodings
3596
3597 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3598 Register reg1 = as_Register($src1$$reg);
3599 Register reg2 = as_Register($src2$$reg);
3600 __ cmpw(reg1, reg2);
3601 %}
3602
3603 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3604 Register reg = as_Register($src1$$reg);
3605 int32_t val = $src2$$constant;
3606 if (val >= 0) {
3607 __ subsw(zr, reg, val);
3608 } else {
3609 __ addsw(zr, reg, -val);
3610 }
3611 %}
3612
3613 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3614 Register reg1 = as_Register($src1$$reg);
3615 uint32_t val = (uint32_t)$src2$$constant;
3616 __ movw(rscratch1, val);
3617 __ cmpw(reg1, rscratch1);
3618 %}
3619
3620 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3621 Register reg1 = as_Register($src1$$reg);
3622 Register reg2 = as_Register($src2$$reg);
3623 __ cmp(reg1, reg2);
3624 %}
3625
3626 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3627 Register reg = as_Register($src1$$reg);
3628 int64_t val = $src2$$constant;
3629 if (val >= 0) {
3630 __ subs(zr, reg, val);
3631 } else if (val != -val) {
3632 __ adds(zr, reg, -val);
3633 } else {
3634 // aargh, Long.MIN_VALUE is a special case
3635 __ orr(rscratch1, zr, (uint64_t)val);
3636 __ subs(zr, reg, rscratch1);
3637 }
3638 %}
3639
3640 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3641 Register reg1 = as_Register($src1$$reg);
3642 uint64_t val = (uint64_t)$src2$$constant;
3643 __ mov(rscratch1, val);
3644 __ cmp(reg1, rscratch1);
3645 %}
3646
3647 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3648 Register reg1 = as_Register($src1$$reg);
3649 Register reg2 = as_Register($src2$$reg);
3650 __ cmp(reg1, reg2);
3651 %}
3652
3653 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3654 Register reg1 = as_Register($src1$$reg);
3655 Register reg2 = as_Register($src2$$reg);
3656 __ cmpw(reg1, reg2);
3657 %}
3658
3659 enc_class aarch64_enc_testp(iRegP src) %{
3660 Register reg = as_Register($src$$reg);
3661 __ cmp(reg, zr);
3662 %}
3663
3664 enc_class aarch64_enc_testn(iRegN src) %{
3665 Register reg = as_Register($src$$reg);
3666 __ cmpw(reg, zr);
3667 %}
3668
3669 enc_class aarch64_enc_b(label lbl) %{
3670 Label *L = $lbl$$label;
3671 __ b(*L);
3672 %}
3673
3674 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3675 Label *L = $lbl$$label;
3676 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3677 %}
3678
3679 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3680 Label *L = $lbl$$label;
3681 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3682 %}
3683
3684 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3685 %{
3686 Register sub_reg = as_Register($sub$$reg);
3687 Register super_reg = as_Register($super$$reg);
3688 Register temp_reg = as_Register($temp$$reg);
3689 Register result_reg = as_Register($result$$reg);
3690
3691 Label miss;
3692 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3693 nullptr, &miss,
3694 /*set_cond_codes:*/ true);
3695 if ($primary) {
3696 __ mov(result_reg, zr);
3697 }
3698 __ bind(miss);
3699 %}
3700
3701 enc_class aarch64_enc_java_static_call(method meth) %{
3702 address addr = (address)$meth$$method;
3703 address call;
3704 if (!_method) {
3705 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3706 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3707 if (call == nullptr) {
3708 ciEnv::current()->record_failure("CodeCache is full");
3709 return;
3710 }
3711 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3712 // The NOP here is purely to ensure that eliding a call to
3713 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3714 __ nop();
3715 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3716 } else {
3717 int method_index = resolved_method_index(masm);
3718 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3719 : static_call_Relocation::spec(method_index);
3720 call = __ trampoline_call(Address(addr, rspec));
3721 if (call == nullptr) {
3722 ciEnv::current()->record_failure("CodeCache is full");
3723 return;
3724 }
3725 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3726 // Calls of the same statically bound method can share
3727 // a stub to the interpreter.
3728 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3729 } else {
3730 // Emit stub for static call
3731 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3732 if (stub == nullptr) {
3733 ciEnv::current()->record_failure("CodeCache is full");
3734 return;
3735 }
3736 }
3737 }
3738
3739 __ post_call_nop();
3740
3741 // Only non uncommon_trap calls need to reinitialize ptrue.
3742 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3743 __ reinitialize_ptrue();
3744 }
3745 %}
3746
3747 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3748 int method_index = resolved_method_index(masm);
3749 address call = __ ic_call((address)$meth$$method, method_index);
3750 if (call == nullptr) {
3751 ciEnv::current()->record_failure("CodeCache is full");
3752 return;
3753 }
3754 __ post_call_nop();
3755 if (Compile::current()->max_vector_size() > 0) {
3756 __ reinitialize_ptrue();
3757 }
3758 %}
3759
3760 enc_class aarch64_enc_call_epilog() %{
3761 if (VerifyStackAtCalls) {
3762 // Check that stack depth is unchanged: find majik cookie on stack
3763 __ call_Unimplemented();
3764 }
3765 %}
3766
3767 enc_class aarch64_enc_java_to_runtime(method meth) %{
3768 // some calls to generated routines (arraycopy code) are scheduled
3769 // by C2 as runtime calls. if so we can call them using a br (they
3770 // will be in a reachable segment) otherwise we have to use a blr
3771 // which loads the absolute address into a register.
3772 address entry = (address)$meth$$method;
3773 CodeBlob *cb = CodeCache::find_blob(entry);
3774 if (cb) {
3775 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3776 if (call == nullptr) {
3777 ciEnv::current()->record_failure("CodeCache is full");
3778 return;
3779 }
3780 __ post_call_nop();
3781 } else {
3782 Label retaddr;
3783 // Make the anchor frame walkable
3784 __ adr(rscratch2, retaddr);
3785 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3786 __ lea(rscratch1, RuntimeAddress(entry));
3787 __ blr(rscratch1);
3788 __ bind(retaddr);
3789 __ post_call_nop();
3790 }
3791 if (Compile::current()->max_vector_size() > 0) {
3792 __ reinitialize_ptrue();
3793 }
3794 %}
3795
3796 enc_class aarch64_enc_rethrow() %{
3797 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3798 %}
3799
3800 enc_class aarch64_enc_ret() %{
3801 #ifdef ASSERT
3802 if (Compile::current()->max_vector_size() > 0) {
3803 __ verify_ptrue();
3804 }
3805 #endif
3806 __ ret(lr);
3807 %}
3808
3809 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3810 Register target_reg = as_Register($jump_target$$reg);
3811 __ br(target_reg);
3812 %}
3813
3814 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3815 Register target_reg = as_Register($jump_target$$reg);
3816 // exception oop should be in r0
3817 // ret addr has been popped into lr
3818 // callee expects it in r3
3819 __ mov(r3, lr);
3820 __ br(target_reg);
3821 %}
3822
3823 %}
3824
3825 //----------FRAME--------------------------------------------------------------
3826 // Definition of frame structure and management information.
3827 //
3828 // S T A C K L A Y O U T Allocators stack-slot number
3829 // | (to get allocators register number
3830 // G Owned by | | v add OptoReg::stack0())
3831 // r CALLER | |
3832 // o | +--------+ pad to even-align allocators stack-slot
3833 // w V | pad0 | numbers; owned by CALLER
3834 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3835 // h ^ | in | 5
3836 // | | args | 4 Holes in incoming args owned by SELF
3837 // | | | | 3
3838 // | | +--------+
3839 // V | | old out| Empty on Intel, window on Sparc
3840 // | old |preserve| Must be even aligned.
3841 // | SP-+--------+----> Matcher::_old_SP, even aligned
3842 // | | in | 3 area for Intel ret address
3843 // Owned by |preserve| Empty on Sparc.
3844 // SELF +--------+
3845 // | | pad2 | 2 pad to align old SP
3846 // | +--------+ 1
3847 // | | locks | 0
3848 // | +--------+----> OptoReg::stack0(), even aligned
3849 // | | pad1 | 11 pad to align new SP
3850 // | +--------+
3851 // | | | 10
3852 // | | spills | 9 spills
3853 // V | | 8 (pad0 slot for callee)
3854 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3855 // ^ | out | 7
3856 // | | args | 6 Holes in outgoing args owned by CALLEE
3857 // Owned by +--------+
3858 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3859 // | new |preserve| Must be even-aligned.
3860 // | SP-+--------+----> Matcher::_new_SP, even aligned
3861 // | | |
3862 //
3863 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3864 // known from SELF's arguments and the Java calling convention.
3865 // Region 6-7 is determined per call site.
3866 // Note 2: If the calling convention leaves holes in the incoming argument
3867 // area, those holes are owned by SELF. Holes in the outgoing area
3868 // are owned by the CALLEE. Holes should not be necessary in the
3869 // incoming area, as the Java calling convention is completely under
3870 // the control of the AD file. Doubles can be sorted and packed to
3871 // avoid holes. Holes in the outgoing arguments may be necessary for
3872 // varargs C calling conventions.
3873 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3874 // even aligned with pad0 as needed.
3875 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3876 // (the latter is true on Intel but is it false on AArch64?)
3877 // region 6-11 is even aligned; it may be padded out more so that
3878 // the region from SP to FP meets the minimum stack alignment.
3879 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3880 // alignment. Region 11, pad1, may be dynamically extended so that
3881 // SP meets the minimum alignment.
3882
3883 frame %{
3884 // These three registers define part of the calling convention
3885 // between compiled code and the interpreter.
3886
3887 // Inline Cache Register or Method for I2C.
3888 inline_cache_reg(R12);
3889
3890 // Number of stack slots consumed by locking an object
3891 sync_stack_slots(2);
3892
3893 // Compiled code's Frame Pointer
3894 frame_pointer(R31);
3895
3896 // Interpreter stores its frame pointer in a register which is
3897 // stored to the stack by I2CAdaptors.
3898 // I2CAdaptors convert from interpreted java to compiled java.
3899 interpreter_frame_pointer(R29);
3900
3901 // Stack alignment requirement
3902 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3903
3904 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3905 // for calls to C. Supports the var-args backing area for register parms.
3906 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3907
3908 // The after-PROLOG location of the return address. Location of
3909 // return address specifies a type (REG or STACK) and a number
3910 // representing the register number (i.e. - use a register name) or
3911 // stack slot.
3912 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3913 // Otherwise, it is above the locks and verification slot and alignment word
3914 // TODO this may well be correct but need to check why that - 2 is there
3915 // ppc port uses 0 but we definitely need to allow for fixed_slots
3916 // which folds in the space used for monitors
3917 return_addr(STACK - 2 +
3918 align_up((Compile::current()->in_preserve_stack_slots() +
3919 Compile::current()->fixed_slots()),
3920 stack_alignment_in_slots()));
3921
3922 // Location of compiled Java return values. Same as C for now.
3923 return_value
3924 %{
3925 // TODO do we allow ideal_reg == Op_RegN???
3926 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3927 "only return normal values");
3928
3929 static const int lo[Op_RegL + 1] = { // enum name
3930 0, // Op_Node
3931 0, // Op_Set
3932 R0_num, // Op_RegN
3933 R0_num, // Op_RegI
3934 R0_num, // Op_RegP
3935 V0_num, // Op_RegF
3936 V0_num, // Op_RegD
3937 R0_num // Op_RegL
3938 };
3939
3940 static const int hi[Op_RegL + 1] = { // enum name
3941 0, // Op_Node
3942 0, // Op_Set
3943 OptoReg::Bad, // Op_RegN
3944 OptoReg::Bad, // Op_RegI
3945 R0_H_num, // Op_RegP
3946 OptoReg::Bad, // Op_RegF
3947 V0_H_num, // Op_RegD
3948 R0_H_num // Op_RegL
3949 };
3950
3951 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3952 %}
3953 %}
3954
3955 //----------ATTRIBUTES---------------------------------------------------------
3956 //----------Operand Attributes-------------------------------------------------
3957 op_attrib op_cost(1); // Required cost attribute
3958
3959 //----------Instruction Attributes---------------------------------------------
3960 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3961 ins_attrib ins_size(32); // Required size attribute (in bits)
3962 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3963 // a non-matching short branch variant
3964 // of some long branch?
3965 ins_attrib ins_alignment(4); // Required alignment attribute (must
3966 // be a power of 2) specifies the
3967 // alignment that some part of the
3968 // instruction (not necessarily the
3969 // start) requires. If > 1, a
3970 // compute_padding() function must be
3971 // provided for the instruction
3972
3973 // Whether this node is expanded during code emission into a sequence of
3974 // instructions and the first instruction can perform an implicit null check.
3975 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3976
3977 //----------OPERANDS-----------------------------------------------------------
3978 // Operand definitions must precede instruction definitions for correct parsing
3979 // in the ADLC because operands constitute user defined types which are used in
3980 // instruction definitions.
3981
3982 //----------Simple Operands----------------------------------------------------
3983
3984 // Integer operands 32 bit
3985 // 32 bit immediate
3986 operand immI()
3987 %{
3988 match(ConI);
3989
3990 op_cost(0);
3991 format %{ %}
3992 interface(CONST_INTER);
3993 %}
3994
3995 // 32 bit zero
3996 operand immI0()
3997 %{
3998 predicate(n->get_int() == 0);
3999 match(ConI);
4000
4001 op_cost(0);
4002 format %{ %}
4003 interface(CONST_INTER);
4004 %}
4005
4006 // 32 bit unit increment
4007 operand immI_1()
4008 %{
4009 predicate(n->get_int() == 1);
4010 match(ConI);
4011
4012 op_cost(0);
4013 format %{ %}
4014 interface(CONST_INTER);
4015 %}
4016
4017 // 32 bit unit decrement
4018 operand immI_M1()
4019 %{
4020 predicate(n->get_int() == -1);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 // Shift values for add/sub extension shift
4029 operand immIExt()
4030 %{
4031 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4032 match(ConI);
4033
4034 op_cost(0);
4035 format %{ %}
4036 interface(CONST_INTER);
4037 %}
4038
4039 operand immI_gt_1()
4040 %{
4041 predicate(n->get_int() > 1);
4042 match(ConI);
4043
4044 op_cost(0);
4045 format %{ %}
4046 interface(CONST_INTER);
4047 %}
4048
4049 operand immI_le_4()
4050 %{
4051 predicate(n->get_int() <= 4);
4052 match(ConI);
4053
4054 op_cost(0);
4055 format %{ %}
4056 interface(CONST_INTER);
4057 %}
4058
4059 operand immI_16()
4060 %{
4061 predicate(n->get_int() == 16);
4062 match(ConI);
4063
4064 op_cost(0);
4065 format %{ %}
4066 interface(CONST_INTER);
4067 %}
4068
4069 operand immI_24()
4070 %{
4071 predicate(n->get_int() == 24);
4072 match(ConI);
4073
4074 op_cost(0);
4075 format %{ %}
4076 interface(CONST_INTER);
4077 %}
4078
4079 operand immI_32()
4080 %{
4081 predicate(n->get_int() == 32);
4082 match(ConI);
4083
4084 op_cost(0);
4085 format %{ %}
4086 interface(CONST_INTER);
4087 %}
4088
4089 operand immI_48()
4090 %{
4091 predicate(n->get_int() == 48);
4092 match(ConI);
4093
4094 op_cost(0);
4095 format %{ %}
4096 interface(CONST_INTER);
4097 %}
4098
4099 operand immI_56()
4100 %{
4101 predicate(n->get_int() == 56);
4102 match(ConI);
4103
4104 op_cost(0);
4105 format %{ %}
4106 interface(CONST_INTER);
4107 %}
4108
4109 operand immI_255()
4110 %{
4111 predicate(n->get_int() == 255);
4112 match(ConI);
4113
4114 op_cost(0);
4115 format %{ %}
4116 interface(CONST_INTER);
4117 %}
4118
4119 operand immI_65535()
4120 %{
4121 predicate(n->get_int() == 65535);
4122 match(ConI);
4123
4124 op_cost(0);
4125 format %{ %}
4126 interface(CONST_INTER);
4127 %}
4128
4129 operand immI_positive()
4130 %{
4131 predicate(n->get_int() > 0);
4132 match(ConI);
4133
4134 op_cost(0);
4135 format %{ %}
4136 interface(CONST_INTER);
4137 %}
4138
4139 // BoolTest condition for signed compare
4140 operand immI_cmp_cond()
4141 %{
4142 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4143 match(ConI);
4144
4145 op_cost(0);
4146 format %{ %}
4147 interface(CONST_INTER);
4148 %}
4149
4150 // BoolTest condition for unsigned compare
4151 operand immI_cmpU_cond()
4152 %{
4153 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4154 match(ConI);
4155
4156 op_cost(0);
4157 format %{ %}
4158 interface(CONST_INTER);
4159 %}
4160
4161 operand immL_255()
4162 %{
4163 predicate(n->get_long() == 255L);
4164 match(ConL);
4165
4166 op_cost(0);
4167 format %{ %}
4168 interface(CONST_INTER);
4169 %}
4170
4171 operand immL_65535()
4172 %{
4173 predicate(n->get_long() == 65535L);
4174 match(ConL);
4175
4176 op_cost(0);
4177 format %{ %}
4178 interface(CONST_INTER);
4179 %}
4180
4181 operand immL_4294967295()
4182 %{
4183 predicate(n->get_long() == 4294967295L);
4184 match(ConL);
4185
4186 op_cost(0);
4187 format %{ %}
4188 interface(CONST_INTER);
4189 %}
4190
4191 operand immL_bitmask()
4192 %{
4193 predicate((n->get_long() != 0)
4194 && ((n->get_long() & 0xc000000000000000l) == 0)
4195 && is_power_of_2(n->get_long() + 1));
4196 match(ConL);
4197
4198 op_cost(0);
4199 format %{ %}
4200 interface(CONST_INTER);
4201 %}
4202
4203 operand immI_bitmask()
4204 %{
4205 predicate((n->get_int() != 0)
4206 && ((n->get_int() & 0xc0000000) == 0)
4207 && is_power_of_2(n->get_int() + 1));
4208 match(ConI);
4209
4210 op_cost(0);
4211 format %{ %}
4212 interface(CONST_INTER);
4213 %}
4214
4215 operand immL_positive_bitmaskI()
4216 %{
4217 predicate((n->get_long() != 0)
4218 && ((julong)n->get_long() < 0x80000000ULL)
4219 && is_power_of_2(n->get_long() + 1));
4220 match(ConL);
4221
4222 op_cost(0);
4223 format %{ %}
4224 interface(CONST_INTER);
4225 %}
4226
4227 // Scale values for scaled offset addressing modes (up to long but not quad)
4228 operand immIScale()
4229 %{
4230 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4231 match(ConI);
4232
4233 op_cost(0);
4234 format %{ %}
4235 interface(CONST_INTER);
4236 %}
4237
4238 // 5 bit signed integer
4239 operand immI5()
4240 %{
4241 predicate(Assembler::is_simm(n->get_int(), 5));
4242 match(ConI);
4243
4244 op_cost(0);
4245 format %{ %}
4246 interface(CONST_INTER);
4247 %}
4248
4249 // 7 bit unsigned integer
4250 operand immIU7()
4251 %{
4252 predicate(Assembler::is_uimm(n->get_int(), 7));
4253 match(ConI);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 // Offset for scaled or unscaled immediate loads and stores
4261 operand immIOffset()
4262 %{
4263 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4264 match(ConI);
4265
4266 op_cost(0);
4267 format %{ %}
4268 interface(CONST_INTER);
4269 %}
4270
4271 operand immIOffset1()
4272 %{
4273 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4274 match(ConI);
4275
4276 op_cost(0);
4277 format %{ %}
4278 interface(CONST_INTER);
4279 %}
4280
4281 operand immIOffset2()
4282 %{
4283 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4284 match(ConI);
4285
4286 op_cost(0);
4287 format %{ %}
4288 interface(CONST_INTER);
4289 %}
4290
4291 operand immIOffset4()
4292 %{
4293 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4294 match(ConI);
4295
4296 op_cost(0);
4297 format %{ %}
4298 interface(CONST_INTER);
4299 %}
4300
4301 operand immIOffset8()
4302 %{
4303 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4304 match(ConI);
4305
4306 op_cost(0);
4307 format %{ %}
4308 interface(CONST_INTER);
4309 %}
4310
4311 operand immIOffset16()
4312 %{
4313 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4314 match(ConI);
4315
4316 op_cost(0);
4317 format %{ %}
4318 interface(CONST_INTER);
4319 %}
4320
4321 operand immLOffset()
4322 %{
4323 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4324 match(ConL);
4325
4326 op_cost(0);
4327 format %{ %}
4328 interface(CONST_INTER);
4329 %}
4330
4331 operand immLoffset1()
4332 %{
4333 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4334 match(ConL);
4335
4336 op_cost(0);
4337 format %{ %}
4338 interface(CONST_INTER);
4339 %}
4340
4341 operand immLoffset2()
4342 %{
4343 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4344 match(ConL);
4345
4346 op_cost(0);
4347 format %{ %}
4348 interface(CONST_INTER);
4349 %}
4350
4351 operand immLoffset4()
4352 %{
4353 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4354 match(ConL);
4355
4356 op_cost(0);
4357 format %{ %}
4358 interface(CONST_INTER);
4359 %}
4360
4361 operand immLoffset8()
4362 %{
4363 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4364 match(ConL);
4365
4366 op_cost(0);
4367 format %{ %}
4368 interface(CONST_INTER);
4369 %}
4370
4371 operand immLoffset16()
4372 %{
4373 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4374 match(ConL);
4375
4376 op_cost(0);
4377 format %{ %}
4378 interface(CONST_INTER);
4379 %}
4380
4381 // 5 bit signed long integer
4382 operand immL5()
4383 %{
4384 predicate(Assembler::is_simm(n->get_long(), 5));
4385 match(ConL);
4386
4387 op_cost(0);
4388 format %{ %}
4389 interface(CONST_INTER);
4390 %}
4391
4392 // 7 bit unsigned long integer
4393 operand immLU7()
4394 %{
4395 predicate(Assembler::is_uimm(n->get_long(), 7));
4396 match(ConL);
4397
4398 op_cost(0);
4399 format %{ %}
4400 interface(CONST_INTER);
4401 %}
4402
4403 // 8 bit signed value.
4404 operand immI8()
4405 %{
4406 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4407 match(ConI);
4408
4409 op_cost(0);
4410 format %{ %}
4411 interface(CONST_INTER);
4412 %}
4413
4414 // 8 bit signed value (simm8), or #simm8 LSL 8.
4415 operand immI8_shift8()
4416 %{
4417 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4418 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4419 match(ConI);
4420
4421 op_cost(0);
4422 format %{ %}
4423 interface(CONST_INTER);
4424 %}
4425
4426 // 8 bit signed value (simm8), or #simm8 LSL 8.
4427 operand immL8_shift8()
4428 %{
4429 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4430 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4431 match(ConL);
4432
4433 op_cost(0);
4434 format %{ %}
4435 interface(CONST_INTER);
4436 %}
4437
4438 // 8 bit integer valid for vector add sub immediate
4439 operand immBAddSubV()
4440 %{
4441 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4442 match(ConI);
4443
4444 op_cost(0);
4445 format %{ %}
4446 interface(CONST_INTER);
4447 %}
4448
4449 // 32 bit integer valid for add sub immediate
4450 operand immIAddSub()
4451 %{
4452 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4453 match(ConI);
4454 op_cost(0);
4455 format %{ %}
4456 interface(CONST_INTER);
4457 %}
4458
4459 // 32 bit integer valid for vector add sub immediate
4460 operand immIAddSubV()
4461 %{
4462 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4463 match(ConI);
4464
4465 op_cost(0);
4466 format %{ %}
4467 interface(CONST_INTER);
4468 %}
4469
4470 // 32 bit unsigned integer valid for logical immediate
4471
4472 operand immBLog()
4473 %{
4474 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4475 match(ConI);
4476
4477 op_cost(0);
4478 format %{ %}
4479 interface(CONST_INTER);
4480 %}
4481
4482 operand immSLog()
4483 %{
4484 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4485 match(ConI);
4486
4487 op_cost(0);
4488 format %{ %}
4489 interface(CONST_INTER);
4490 %}
4491
4492 operand immILog()
4493 %{
4494 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4495 match(ConI);
4496
4497 op_cost(0);
4498 format %{ %}
4499 interface(CONST_INTER);
4500 %}
4501
4502 // Integer operands 64 bit
4503 // 64 bit immediate
4504 operand immL()
4505 %{
4506 match(ConL);
4507
4508 op_cost(0);
4509 format %{ %}
4510 interface(CONST_INTER);
4511 %}
4512
4513 // 64 bit zero
4514 operand immL0()
4515 %{
4516 predicate(n->get_long() == 0);
4517 match(ConL);
4518
4519 op_cost(0);
4520 format %{ %}
4521 interface(CONST_INTER);
4522 %}
4523
4524 // 64 bit unit decrement
4525 operand immL_M1()
4526 %{
4527 predicate(n->get_long() == -1);
4528 match(ConL);
4529
4530 op_cost(0);
4531 format %{ %}
4532 interface(CONST_INTER);
4533 %}
4534
4535 // 64 bit integer valid for add sub immediate
4536 operand immLAddSub()
4537 %{
4538 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4539 match(ConL);
4540 op_cost(0);
4541 format %{ %}
4542 interface(CONST_INTER);
4543 %}
4544
4545 // 64 bit integer valid for addv subv immediate
4546 operand immLAddSubV()
4547 %{
4548 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4549 match(ConL);
4550
4551 op_cost(0);
4552 format %{ %}
4553 interface(CONST_INTER);
4554 %}
4555
4556 // 64 bit integer valid for logical immediate
4557 operand immLLog()
4558 %{
4559 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4560 match(ConL);
4561 op_cost(0);
4562 format %{ %}
4563 interface(CONST_INTER);
4564 %}
4565
4566 // Long Immediate: low 32-bit mask
4567 operand immL_32bits()
4568 %{
4569 predicate(n->get_long() == 0xFFFFFFFFL);
4570 match(ConL);
4571 op_cost(0);
4572 format %{ %}
4573 interface(CONST_INTER);
4574 %}
4575
4576 // Pointer operands
4577 // Pointer Immediate
4578 operand immP()
4579 %{
4580 match(ConP);
4581
4582 op_cost(0);
4583 format %{ %}
4584 interface(CONST_INTER);
4585 %}
4586
4587 // nullptr Pointer Immediate
4588 operand immP0()
4589 %{
4590 predicate(n->get_ptr() == 0);
4591 match(ConP);
4592
4593 op_cost(0);
4594 format %{ %}
4595 interface(CONST_INTER);
4596 %}
4597
4598 // Pointer Immediate One
4599 // this is used in object initialization (initial object header)
4600 operand immP_1()
4601 %{
4602 predicate(n->get_ptr() == 1);
4603 match(ConP);
4604
4605 op_cost(0);
4606 format %{ %}
4607 interface(CONST_INTER);
4608 %}
4609
4610 // Float and Double operands
4611 // Double Immediate
4612 operand immD()
4613 %{
4614 match(ConD);
4615 op_cost(0);
4616 format %{ %}
4617 interface(CONST_INTER);
4618 %}
4619
4620 // Double Immediate: +0.0d
4621 operand immD0()
4622 %{
4623 predicate(jlong_cast(n->getd()) == 0);
4624 match(ConD);
4625
4626 op_cost(0);
4627 format %{ %}
4628 interface(CONST_INTER);
4629 %}
4630
4631 // constant 'double +0.0'.
4632 operand immDPacked()
4633 %{
4634 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4635 match(ConD);
4636 op_cost(0);
4637 format %{ %}
4638 interface(CONST_INTER);
4639 %}
4640
4641 // Float Immediate
4642 operand immF()
4643 %{
4644 match(ConF);
4645 op_cost(0);
4646 format %{ %}
4647 interface(CONST_INTER);
4648 %}
4649
4650 // Float Immediate: +0.0f.
4651 operand immF0()
4652 %{
4653 predicate(jint_cast(n->getf()) == 0);
4654 match(ConF);
4655
4656 op_cost(0);
4657 format %{ %}
4658 interface(CONST_INTER);
4659 %}
4660
4661 // Half Float (FP16) Immediate
4662 operand immH()
4663 %{
4664 match(ConH);
4665 op_cost(0);
4666 format %{ %}
4667 interface(CONST_INTER);
4668 %}
4669
4670 //
4671 operand immFPacked()
4672 %{
4673 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4674 match(ConF);
4675 op_cost(0);
4676 format %{ %}
4677 interface(CONST_INTER);
4678 %}
4679
4680 // Narrow pointer operands
4681 // Narrow Pointer Immediate
4682 operand immN()
4683 %{
4684 match(ConN);
4685
4686 op_cost(0);
4687 format %{ %}
4688 interface(CONST_INTER);
4689 %}
4690
4691 // Narrow nullptr Pointer Immediate
4692 operand immN0()
4693 %{
4694 predicate(n->get_narrowcon() == 0);
4695 match(ConN);
4696
4697 op_cost(0);
4698 format %{ %}
4699 interface(CONST_INTER);
4700 %}
4701
4702 operand immNKlass()
4703 %{
4704 match(ConNKlass);
4705
4706 op_cost(0);
4707 format %{ %}
4708 interface(CONST_INTER);
4709 %}
4710
4711 // Integer 32 bit Register Operands
4712 // Integer 32 bitRegister (excludes SP)
4713 operand iRegI()
4714 %{
4715 constraint(ALLOC_IN_RC(any_reg32));
4716 match(RegI);
4717 match(iRegINoSp);
4718 op_cost(0);
4719 format %{ %}
4720 interface(REG_INTER);
4721 %}
4722
4723 // Integer 32 bit Register not Special
4724 operand iRegINoSp()
4725 %{
4726 constraint(ALLOC_IN_RC(no_special_reg32));
4727 match(RegI);
4728 op_cost(0);
4729 format %{ %}
4730 interface(REG_INTER);
4731 %}
4732
4733 // Integer 64 bit Register Operands
4734 // Integer 64 bit Register (includes SP)
4735 operand iRegL()
4736 %{
4737 constraint(ALLOC_IN_RC(any_reg));
4738 match(RegL);
4739 match(iRegLNoSp);
4740 op_cost(0);
4741 format %{ %}
4742 interface(REG_INTER);
4743 %}
4744
4745 // Integer 64 bit Register not Special
4746 operand iRegLNoSp()
4747 %{
4748 constraint(ALLOC_IN_RC(no_special_reg));
4749 match(RegL);
4750 match(iRegL_R0);
4751 format %{ %}
4752 interface(REG_INTER);
4753 %}
4754
4755 // Pointer Register Operands
4756 // Pointer Register
4757 operand iRegP()
4758 %{
4759 constraint(ALLOC_IN_RC(ptr_reg));
4760 match(RegP);
4761 match(iRegPNoSp);
4762 match(iRegP_R0);
4763 //match(iRegP_R2);
4764 //match(iRegP_R4);
4765 match(iRegP_R5);
4766 match(thread_RegP);
4767 op_cost(0);
4768 format %{ %}
4769 interface(REG_INTER);
4770 %}
4771
4772 // Pointer 64 bit Register not Special
4773 operand iRegPNoSp()
4774 %{
4775 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4776 match(RegP);
4777 // match(iRegP);
4778 // match(iRegP_R0);
4779 // match(iRegP_R2);
4780 // match(iRegP_R4);
4781 // match(iRegP_R5);
4782 // match(thread_RegP);
4783 op_cost(0);
4784 format %{ %}
4785 interface(REG_INTER);
4786 %}
4787
4788 // This operand is not allowed to use rfp even if
4789 // rfp is not used to hold the frame pointer.
4790 operand iRegPNoSpNoRfp()
4791 %{
4792 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4793 match(RegP);
4794 match(iRegPNoSp);
4795 op_cost(0);
4796 format %{ %}
4797 interface(REG_INTER);
4798 %}
4799
4800 // Pointer 64 bit Register R0 only
4801 operand iRegP_R0()
4802 %{
4803 constraint(ALLOC_IN_RC(r0_reg));
4804 match(RegP);
4805 // match(iRegP);
4806 match(iRegPNoSp);
4807 op_cost(0);
4808 format %{ %}
4809 interface(REG_INTER);
4810 %}
4811
4812 // Pointer 64 bit Register R1 only
4813 operand iRegP_R1()
4814 %{
4815 constraint(ALLOC_IN_RC(r1_reg));
4816 match(RegP);
4817 // match(iRegP);
4818 match(iRegPNoSp);
4819 op_cost(0);
4820 format %{ %}
4821 interface(REG_INTER);
4822 %}
4823
4824 // Pointer 64 bit Register R2 only
4825 operand iRegP_R2()
4826 %{
4827 constraint(ALLOC_IN_RC(r2_reg));
4828 match(RegP);
4829 // match(iRegP);
4830 match(iRegPNoSp);
4831 op_cost(0);
4832 format %{ %}
4833 interface(REG_INTER);
4834 %}
4835
4836 // Pointer 64 bit Register R3 only
4837 operand iRegP_R3()
4838 %{
4839 constraint(ALLOC_IN_RC(r3_reg));
4840 match(RegP);
4841 // match(iRegP);
4842 match(iRegPNoSp);
4843 op_cost(0);
4844 format %{ %}
4845 interface(REG_INTER);
4846 %}
4847
4848 // Pointer 64 bit Register R4 only
4849 operand iRegP_R4()
4850 %{
4851 constraint(ALLOC_IN_RC(r4_reg));
4852 match(RegP);
4853 // match(iRegP);
4854 match(iRegPNoSp);
4855 op_cost(0);
4856 format %{ %}
4857 interface(REG_INTER);
4858 %}
4859
4860 // Pointer 64 bit Register R5 only
4861 operand iRegP_R5()
4862 %{
4863 constraint(ALLOC_IN_RC(r5_reg));
4864 match(RegP);
4865 // match(iRegP);
4866 match(iRegPNoSp);
4867 op_cost(0);
4868 format %{ %}
4869 interface(REG_INTER);
4870 %}
4871
4872 // Pointer 64 bit Register R10 only
4873 operand iRegP_R10()
4874 %{
4875 constraint(ALLOC_IN_RC(r10_reg));
4876 match(RegP);
4877 // match(iRegP);
4878 match(iRegPNoSp);
4879 op_cost(0);
4880 format %{ %}
4881 interface(REG_INTER);
4882 %}
4883
4884 // Long 64 bit Register R0 only
4885 operand iRegL_R0()
4886 %{
4887 constraint(ALLOC_IN_RC(r0_reg));
4888 match(RegL);
4889 match(iRegLNoSp);
4890 op_cost(0);
4891 format %{ %}
4892 interface(REG_INTER);
4893 %}
4894
4895 // Long 64 bit Register R11 only
4896 operand iRegL_R11()
4897 %{
4898 constraint(ALLOC_IN_RC(r11_reg));
4899 match(RegL);
4900 match(iRegLNoSp);
4901 op_cost(0);
4902 format %{ %}
4903 interface(REG_INTER);
4904 %}
4905
4906 // Register R0 only
4907 operand iRegI_R0()
4908 %{
4909 constraint(ALLOC_IN_RC(int_r0_reg));
4910 match(RegI);
4911 match(iRegINoSp);
4912 op_cost(0);
4913 format %{ %}
4914 interface(REG_INTER);
4915 %}
4916
4917 // Register R2 only
4918 operand iRegI_R2()
4919 %{
4920 constraint(ALLOC_IN_RC(int_r2_reg));
4921 match(RegI);
4922 match(iRegINoSp);
4923 op_cost(0);
4924 format %{ %}
4925 interface(REG_INTER);
4926 %}
4927
4928 // Register R3 only
4929 operand iRegI_R3()
4930 %{
4931 constraint(ALLOC_IN_RC(int_r3_reg));
4932 match(RegI);
4933 match(iRegINoSp);
4934 op_cost(0);
4935 format %{ %}
4936 interface(REG_INTER);
4937 %}
4938
4939
4940 // Register R4 only
4941 operand iRegI_R4()
4942 %{
4943 constraint(ALLOC_IN_RC(int_r4_reg));
4944 match(RegI);
4945 match(iRegINoSp);
4946 op_cost(0);
4947 format %{ %}
4948 interface(REG_INTER);
4949 %}
4950
4951
4952 // Pointer Register Operands
4953 // Narrow Pointer Register
4954 operand iRegN()
4955 %{
4956 constraint(ALLOC_IN_RC(any_reg32));
4957 match(RegN);
4958 match(iRegNNoSp);
4959 op_cost(0);
4960 format %{ %}
4961 interface(REG_INTER);
4962 %}
4963
4964 // Integer 64 bit Register not Special
4965 operand iRegNNoSp()
4966 %{
4967 constraint(ALLOC_IN_RC(no_special_reg32));
4968 match(RegN);
4969 op_cost(0);
4970 format %{ %}
4971 interface(REG_INTER);
4972 %}
4973
4974 // Float Register
4975 // Float register operands
4976 operand vRegF()
4977 %{
4978 constraint(ALLOC_IN_RC(float_reg));
4979 match(RegF);
4980
4981 op_cost(0);
4982 format %{ %}
4983 interface(REG_INTER);
4984 %}
4985
4986 // Double Register
4987 // Double register operands
4988 operand vRegD()
4989 %{
4990 constraint(ALLOC_IN_RC(double_reg));
4991 match(RegD);
4992
4993 op_cost(0);
4994 format %{ %}
4995 interface(REG_INTER);
4996 %}
4997
4998 // Generic vector class. This will be used for
4999 // all vector operands, including NEON and SVE.
5000 operand vReg()
5001 %{
5002 constraint(ALLOC_IN_RC(dynamic));
5003 match(VecA);
5004 match(VecD);
5005 match(VecX);
5006
5007 op_cost(0);
5008 format %{ %}
5009 interface(REG_INTER);
5010 %}
5011
5012 operand vReg_V10()
5013 %{
5014 constraint(ALLOC_IN_RC(v10_veca_reg));
5015 match(vReg);
5016
5017 op_cost(0);
5018 format %{ %}
5019 interface(REG_INTER);
5020 %}
5021
5022 operand vReg_V11()
5023 %{
5024 constraint(ALLOC_IN_RC(v11_veca_reg));
5025 match(vReg);
5026
5027 op_cost(0);
5028 format %{ %}
5029 interface(REG_INTER);
5030 %}
5031
5032 operand vReg_V12()
5033 %{
5034 constraint(ALLOC_IN_RC(v12_veca_reg));
5035 match(vReg);
5036
5037 op_cost(0);
5038 format %{ %}
5039 interface(REG_INTER);
5040 %}
5041
5042 operand vReg_V13()
5043 %{
5044 constraint(ALLOC_IN_RC(v13_veca_reg));
5045 match(vReg);
5046
5047 op_cost(0);
5048 format %{ %}
5049 interface(REG_INTER);
5050 %}
5051
5052 operand vReg_V17()
5053 %{
5054 constraint(ALLOC_IN_RC(v17_veca_reg));
5055 match(vReg);
5056
5057 op_cost(0);
5058 format %{ %}
5059 interface(REG_INTER);
5060 %}
5061
5062 operand vReg_V18()
5063 %{
5064 constraint(ALLOC_IN_RC(v18_veca_reg));
5065 match(vReg);
5066
5067 op_cost(0);
5068 format %{ %}
5069 interface(REG_INTER);
5070 %}
5071
5072 operand vReg_V23()
5073 %{
5074 constraint(ALLOC_IN_RC(v23_veca_reg));
5075 match(vReg);
5076
5077 op_cost(0);
5078 format %{ %}
5079 interface(REG_INTER);
5080 %}
5081
5082 operand vReg_V24()
5083 %{
5084 constraint(ALLOC_IN_RC(v24_veca_reg));
5085 match(vReg);
5086
5087 op_cost(0);
5088 format %{ %}
5089 interface(REG_INTER);
5090 %}
5091
5092 operand vecA()
5093 %{
5094 constraint(ALLOC_IN_RC(vectora_reg));
5095 match(VecA);
5096
5097 op_cost(0);
5098 format %{ %}
5099 interface(REG_INTER);
5100 %}
5101
5102 operand vecD()
5103 %{
5104 constraint(ALLOC_IN_RC(vectord_reg));
5105 match(VecD);
5106
5107 op_cost(0);
5108 format %{ %}
5109 interface(REG_INTER);
5110 %}
5111
5112 operand vecX()
5113 %{
5114 constraint(ALLOC_IN_RC(vectorx_reg));
5115 match(VecX);
5116
5117 op_cost(0);
5118 format %{ %}
5119 interface(REG_INTER);
5120 %}
5121
5122 operand vRegD_V0()
5123 %{
5124 constraint(ALLOC_IN_RC(v0_reg));
5125 match(RegD);
5126 op_cost(0);
5127 format %{ %}
5128 interface(REG_INTER);
5129 %}
5130
5131 operand vRegD_V1()
5132 %{
5133 constraint(ALLOC_IN_RC(v1_reg));
5134 match(RegD);
5135 op_cost(0);
5136 format %{ %}
5137 interface(REG_INTER);
5138 %}
5139
5140 operand vRegD_V2()
5141 %{
5142 constraint(ALLOC_IN_RC(v2_reg));
5143 match(RegD);
5144 op_cost(0);
5145 format %{ %}
5146 interface(REG_INTER);
5147 %}
5148
5149 operand vRegD_V3()
5150 %{
5151 constraint(ALLOC_IN_RC(v3_reg));
5152 match(RegD);
5153 op_cost(0);
5154 format %{ %}
5155 interface(REG_INTER);
5156 %}
5157
5158 operand vRegD_V4()
5159 %{
5160 constraint(ALLOC_IN_RC(v4_reg));
5161 match(RegD);
5162 op_cost(0);
5163 format %{ %}
5164 interface(REG_INTER);
5165 %}
5166
5167 operand vRegD_V5()
5168 %{
5169 constraint(ALLOC_IN_RC(v5_reg));
5170 match(RegD);
5171 op_cost(0);
5172 format %{ %}
5173 interface(REG_INTER);
5174 %}
5175
5176 operand vRegD_V6()
5177 %{
5178 constraint(ALLOC_IN_RC(v6_reg));
5179 match(RegD);
5180 op_cost(0);
5181 format %{ %}
5182 interface(REG_INTER);
5183 %}
5184
5185 operand vRegD_V7()
5186 %{
5187 constraint(ALLOC_IN_RC(v7_reg));
5188 match(RegD);
5189 op_cost(0);
5190 format %{ %}
5191 interface(REG_INTER);
5192 %}
5193
5194 operand vRegD_V12()
5195 %{
5196 constraint(ALLOC_IN_RC(v12_reg));
5197 match(RegD);
5198 op_cost(0);
5199 format %{ %}
5200 interface(REG_INTER);
5201 %}
5202
5203 operand vRegD_V13()
5204 %{
5205 constraint(ALLOC_IN_RC(v13_reg));
5206 match(RegD);
5207 op_cost(0);
5208 format %{ %}
5209 interface(REG_INTER);
5210 %}
5211
5212 operand pReg()
5213 %{
5214 constraint(ALLOC_IN_RC(pr_reg));
5215 match(RegVectMask);
5216 match(pRegGov);
5217 op_cost(0);
5218 format %{ %}
5219 interface(REG_INTER);
5220 %}
5221
5222 operand pRegGov()
5223 %{
5224 constraint(ALLOC_IN_RC(gov_pr));
5225 match(RegVectMask);
5226 match(pReg);
5227 op_cost(0);
5228 format %{ %}
5229 interface(REG_INTER);
5230 %}
5231
5232 operand pRegGov_P0()
5233 %{
5234 constraint(ALLOC_IN_RC(p0_reg));
5235 match(RegVectMask);
5236 op_cost(0);
5237 format %{ %}
5238 interface(REG_INTER);
5239 %}
5240
5241 operand pRegGov_P1()
5242 %{
5243 constraint(ALLOC_IN_RC(p1_reg));
5244 match(RegVectMask);
5245 op_cost(0);
5246 format %{ %}
5247 interface(REG_INTER);
5248 %}
5249
5250 // Flags register, used as output of signed compare instructions
5251
5252 // note that on AArch64 we also use this register as the output for
5253 // for floating point compare instructions (CmpF CmpD). this ensures
5254 // that ordered inequality tests use GT, GE, LT or LE none of which
5255 // pass through cases where the result is unordered i.e. one or both
5256 // inputs to the compare is a NaN. this means that the ideal code can
5257 // replace e.g. a GT with an LE and not end up capturing the NaN case
5258 // (where the comparison should always fail). EQ and NE tests are
5259 // always generated in ideal code so that unordered folds into the NE
5260 // case, matching the behaviour of AArch64 NE.
5261 //
5262 // This differs from x86 where the outputs of FP compares use a
5263 // special FP flags registers and where compares based on this
5264 // register are distinguished into ordered inequalities (cmpOpUCF) and
5265 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5266 // to explicitly handle the unordered case in branches. x86 also has
5267 // to include extra CMoveX rules to accept a cmpOpUCF input.
5268
5269 operand rFlagsReg()
5270 %{
5271 constraint(ALLOC_IN_RC(int_flags));
5272 match(RegFlags);
5273
5274 op_cost(0);
5275 format %{ "RFLAGS" %}
5276 interface(REG_INTER);
5277 %}
5278
5279 // Flags register, used as output of unsigned compare instructions
5280 operand rFlagsRegU()
5281 %{
5282 constraint(ALLOC_IN_RC(int_flags));
5283 match(RegFlags);
5284
5285 op_cost(0);
5286 format %{ "RFLAGSU" %}
5287 interface(REG_INTER);
5288 %}
5289
5290 // Special Registers
5291
5292 // Method Register
5293 operand inline_cache_RegP(iRegP reg)
5294 %{
5295 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5296 match(reg);
5297 match(iRegPNoSp);
5298 op_cost(0);
5299 format %{ %}
5300 interface(REG_INTER);
5301 %}
5302
5303 // Thread Register
5304 operand thread_RegP(iRegP reg)
5305 %{
5306 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5307 match(reg);
5308 op_cost(0);
5309 format %{ %}
5310 interface(REG_INTER);
5311 %}
5312
5313 //----------Memory Operands----------------------------------------------------
5314
5315 operand indirect(iRegP reg)
5316 %{
5317 constraint(ALLOC_IN_RC(ptr_reg));
5318 match(reg);
5319 op_cost(0);
5320 format %{ "[$reg]" %}
5321 interface(MEMORY_INTER) %{
5322 base($reg);
5323 index(0xffffffff);
5324 scale(0x0);
5325 disp(0x0);
5326 %}
5327 %}
5328
5329 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5330 %{
5331 constraint(ALLOC_IN_RC(ptr_reg));
5332 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5333 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5334 op_cost(0);
5335 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5336 interface(MEMORY_INTER) %{
5337 base($reg);
5338 index($ireg);
5339 scale($scale);
5340 disp(0x0);
5341 %}
5342 %}
5343
5344 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5345 %{
5346 constraint(ALLOC_IN_RC(ptr_reg));
5347 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5348 match(AddP reg (LShiftL lreg scale));
5349 op_cost(0);
5350 format %{ "$reg, $lreg lsl($scale)" %}
5351 interface(MEMORY_INTER) %{
5352 base($reg);
5353 index($lreg);
5354 scale($scale);
5355 disp(0x0);
5356 %}
5357 %}
5358
5359 operand indIndexI2L(iRegP reg, iRegI ireg)
5360 %{
5361 constraint(ALLOC_IN_RC(ptr_reg));
5362 match(AddP reg (ConvI2L ireg));
5363 op_cost(0);
5364 format %{ "$reg, $ireg, 0, I2L" %}
5365 interface(MEMORY_INTER) %{
5366 base($reg);
5367 index($ireg);
5368 scale(0x0);
5369 disp(0x0);
5370 %}
5371 %}
5372
5373 operand indIndex(iRegP reg, iRegL lreg)
5374 %{
5375 constraint(ALLOC_IN_RC(ptr_reg));
5376 match(AddP reg lreg);
5377 op_cost(0);
5378 format %{ "$reg, $lreg" %}
5379 interface(MEMORY_INTER) %{
5380 base($reg);
5381 index($lreg);
5382 scale(0x0);
5383 disp(0x0);
5384 %}
5385 %}
5386
5387 operand indOffI1(iRegP reg, immIOffset1 off)
5388 %{
5389 constraint(ALLOC_IN_RC(ptr_reg));
5390 match(AddP reg off);
5391 op_cost(0);
5392 format %{ "[$reg, $off]" %}
5393 interface(MEMORY_INTER) %{
5394 base($reg);
5395 index(0xffffffff);
5396 scale(0x0);
5397 disp($off);
5398 %}
5399 %}
5400
5401 operand indOffI2(iRegP reg, immIOffset2 off)
5402 %{
5403 constraint(ALLOC_IN_RC(ptr_reg));
5404 match(AddP reg off);
5405 op_cost(0);
5406 format %{ "[$reg, $off]" %}
5407 interface(MEMORY_INTER) %{
5408 base($reg);
5409 index(0xffffffff);
5410 scale(0x0);
5411 disp($off);
5412 %}
5413 %}
5414
5415 operand indOffI4(iRegP reg, immIOffset4 off)
5416 %{
5417 constraint(ALLOC_IN_RC(ptr_reg));
5418 match(AddP reg off);
5419 op_cost(0);
5420 format %{ "[$reg, $off]" %}
5421 interface(MEMORY_INTER) %{
5422 base($reg);
5423 index(0xffffffff);
5424 scale(0x0);
5425 disp($off);
5426 %}
5427 %}
5428
5429 operand indOffI8(iRegP reg, immIOffset8 off)
5430 %{
5431 constraint(ALLOC_IN_RC(ptr_reg));
5432 match(AddP reg off);
5433 op_cost(0);
5434 format %{ "[$reg, $off]" %}
5435 interface(MEMORY_INTER) %{
5436 base($reg);
5437 index(0xffffffff);
5438 scale(0x0);
5439 disp($off);
5440 %}
5441 %}
5442
5443 operand indOffI16(iRegP reg, immIOffset16 off)
5444 %{
5445 constraint(ALLOC_IN_RC(ptr_reg));
5446 match(AddP reg off);
5447 op_cost(0);
5448 format %{ "[$reg, $off]" %}
5449 interface(MEMORY_INTER) %{
5450 base($reg);
5451 index(0xffffffff);
5452 scale(0x0);
5453 disp($off);
5454 %}
5455 %}
5456
5457 operand indOffL1(iRegP reg, immLoffset1 off)
5458 %{
5459 constraint(ALLOC_IN_RC(ptr_reg));
5460 match(AddP reg off);
5461 op_cost(0);
5462 format %{ "[$reg, $off]" %}
5463 interface(MEMORY_INTER) %{
5464 base($reg);
5465 index(0xffffffff);
5466 scale(0x0);
5467 disp($off);
5468 %}
5469 %}
5470
5471 operand indOffL2(iRegP reg, immLoffset2 off)
5472 %{
5473 constraint(ALLOC_IN_RC(ptr_reg));
5474 match(AddP reg off);
5475 op_cost(0);
5476 format %{ "[$reg, $off]" %}
5477 interface(MEMORY_INTER) %{
5478 base($reg);
5479 index(0xffffffff);
5480 scale(0x0);
5481 disp($off);
5482 %}
5483 %}
5484
5485 operand indOffL4(iRegP reg, immLoffset4 off)
5486 %{
5487 constraint(ALLOC_IN_RC(ptr_reg));
5488 match(AddP reg off);
5489 op_cost(0);
5490 format %{ "[$reg, $off]" %}
5491 interface(MEMORY_INTER) %{
5492 base($reg);
5493 index(0xffffffff);
5494 scale(0x0);
5495 disp($off);
5496 %}
5497 %}
5498
5499 operand indOffL8(iRegP reg, immLoffset8 off)
5500 %{
5501 constraint(ALLOC_IN_RC(ptr_reg));
5502 match(AddP reg off);
5503 op_cost(0);
5504 format %{ "[$reg, $off]" %}
5505 interface(MEMORY_INTER) %{
5506 base($reg);
5507 index(0xffffffff);
5508 scale(0x0);
5509 disp($off);
5510 %}
5511 %}
5512
5513 operand indOffL16(iRegP reg, immLoffset16 off)
5514 %{
5515 constraint(ALLOC_IN_RC(ptr_reg));
5516 match(AddP reg off);
5517 op_cost(0);
5518 format %{ "[$reg, $off]" %}
5519 interface(MEMORY_INTER) %{
5520 base($reg);
5521 index(0xffffffff);
5522 scale(0x0);
5523 disp($off);
5524 %}
5525 %}
5526
5527 operand indirectX2P(iRegL reg)
5528 %{
5529 constraint(ALLOC_IN_RC(ptr_reg));
5530 match(CastX2P reg);
5531 op_cost(0);
5532 format %{ "[$reg]\t# long -> ptr" %}
5533 interface(MEMORY_INTER) %{
5534 base($reg);
5535 index(0xffffffff);
5536 scale(0x0);
5537 disp(0x0);
5538 %}
5539 %}
5540
5541 operand indOffX2P(iRegL reg, immLOffset off)
5542 %{
5543 constraint(ALLOC_IN_RC(ptr_reg));
5544 match(AddP (CastX2P reg) off);
5545 op_cost(0);
5546 format %{ "[$reg, $off]\t# long -> ptr" %}
5547 interface(MEMORY_INTER) %{
5548 base($reg);
5549 index(0xffffffff);
5550 scale(0x0);
5551 disp($off);
5552 %}
5553 %}
5554
5555 operand indirectN(iRegN reg)
5556 %{
5557 predicate(CompressedOops::shift() == 0);
5558 constraint(ALLOC_IN_RC(ptr_reg));
5559 match(DecodeN reg);
5560 op_cost(0);
5561 format %{ "[$reg]\t# narrow" %}
5562 interface(MEMORY_INTER) %{
5563 base($reg);
5564 index(0xffffffff);
5565 scale(0x0);
5566 disp(0x0);
5567 %}
5568 %}
5569
5570 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5571 %{
5572 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5573 constraint(ALLOC_IN_RC(ptr_reg));
5574 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5575 op_cost(0);
5576 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5577 interface(MEMORY_INTER) %{
5578 base($reg);
5579 index($ireg);
5580 scale($scale);
5581 disp(0x0);
5582 %}
5583 %}
5584
5585 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5586 %{
5587 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5588 constraint(ALLOC_IN_RC(ptr_reg));
5589 match(AddP (DecodeN reg) (LShiftL lreg scale));
5590 op_cost(0);
5591 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5592 interface(MEMORY_INTER) %{
5593 base($reg);
5594 index($lreg);
5595 scale($scale);
5596 disp(0x0);
5597 %}
5598 %}
5599
5600 operand indIndexI2LN(iRegN reg, iRegI ireg)
5601 %{
5602 predicate(CompressedOops::shift() == 0);
5603 constraint(ALLOC_IN_RC(ptr_reg));
5604 match(AddP (DecodeN reg) (ConvI2L ireg));
5605 op_cost(0);
5606 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5607 interface(MEMORY_INTER) %{
5608 base($reg);
5609 index($ireg);
5610 scale(0x0);
5611 disp(0x0);
5612 %}
5613 %}
5614
5615 operand indIndexN(iRegN reg, iRegL lreg)
5616 %{
5617 predicate(CompressedOops::shift() == 0);
5618 constraint(ALLOC_IN_RC(ptr_reg));
5619 match(AddP (DecodeN reg) lreg);
5620 op_cost(0);
5621 format %{ "$reg, $lreg\t# narrow" %}
5622 interface(MEMORY_INTER) %{
5623 base($reg);
5624 index($lreg);
5625 scale(0x0);
5626 disp(0x0);
5627 %}
5628 %}
5629
5630 operand indOffIN(iRegN reg, immIOffset off)
5631 %{
5632 predicate(CompressedOops::shift() == 0);
5633 constraint(ALLOC_IN_RC(ptr_reg));
5634 match(AddP (DecodeN reg) off);
5635 op_cost(0);
5636 format %{ "[$reg, $off]\t# narrow" %}
5637 interface(MEMORY_INTER) %{
5638 base($reg);
5639 index(0xffffffff);
5640 scale(0x0);
5641 disp($off);
5642 %}
5643 %}
5644
5645 operand indOffLN(iRegN reg, immLOffset off)
5646 %{
5647 predicate(CompressedOops::shift() == 0);
5648 constraint(ALLOC_IN_RC(ptr_reg));
5649 match(AddP (DecodeN reg) off);
5650 op_cost(0);
5651 format %{ "[$reg, $off]\t# narrow" %}
5652 interface(MEMORY_INTER) %{
5653 base($reg);
5654 index(0xffffffff);
5655 scale(0x0);
5656 disp($off);
5657 %}
5658 %}
5659
5660
5661 //----------Special Memory Operands--------------------------------------------
5662 // Stack Slot Operand - This operand is used for loading and storing temporary
5663 // values on the stack where a match requires a value to
5664 // flow through memory.
5665 operand stackSlotP(sRegP reg)
5666 %{
5667 constraint(ALLOC_IN_RC(stack_slots));
5668 op_cost(100);
5669 // No match rule because this operand is only generated in matching
5670 // match(RegP);
5671 format %{ "[$reg]" %}
5672 interface(MEMORY_INTER) %{
5673 base(0x1e); // RSP
5674 index(0x0); // No Index
5675 scale(0x0); // No Scale
5676 disp($reg); // Stack Offset
5677 %}
5678 %}
5679
5680 operand stackSlotI(sRegI reg)
5681 %{
5682 constraint(ALLOC_IN_RC(stack_slots));
5683 // No match rule because this operand is only generated in matching
5684 // match(RegI);
5685 format %{ "[$reg]" %}
5686 interface(MEMORY_INTER) %{
5687 base(0x1e); // RSP
5688 index(0x0); // No Index
5689 scale(0x0); // No Scale
5690 disp($reg); // Stack Offset
5691 %}
5692 %}
5693
5694 operand stackSlotF(sRegF reg)
5695 %{
5696 constraint(ALLOC_IN_RC(stack_slots));
5697 // No match rule because this operand is only generated in matching
5698 // match(RegF);
5699 format %{ "[$reg]" %}
5700 interface(MEMORY_INTER) %{
5701 base(0x1e); // RSP
5702 index(0x0); // No Index
5703 scale(0x0); // No Scale
5704 disp($reg); // Stack Offset
5705 %}
5706 %}
5707
5708 operand stackSlotD(sRegD reg)
5709 %{
5710 constraint(ALLOC_IN_RC(stack_slots));
5711 // No match rule because this operand is only generated in matching
5712 // match(RegD);
5713 format %{ "[$reg]" %}
5714 interface(MEMORY_INTER) %{
5715 base(0x1e); // RSP
5716 index(0x0); // No Index
5717 scale(0x0); // No Scale
5718 disp($reg); // Stack Offset
5719 %}
5720 %}
5721
5722 operand stackSlotL(sRegL reg)
5723 %{
5724 constraint(ALLOC_IN_RC(stack_slots));
5725 // No match rule because this operand is only generated in matching
5726 // match(RegL);
5727 format %{ "[$reg]" %}
5728 interface(MEMORY_INTER) %{
5729 base(0x1e); // RSP
5730 index(0x0); // No Index
5731 scale(0x0); // No Scale
5732 disp($reg); // Stack Offset
5733 %}
5734 %}
5735
5736 // Operands for expressing Control Flow
5737 // NOTE: Label is a predefined operand which should not be redefined in
5738 // the AD file. It is generically handled within the ADLC.
5739
5740 //----------Conditional Branch Operands----------------------------------------
5741 // Comparison Op - This is the operation of the comparison, and is limited to
5742 // the following set of codes:
5743 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5744 //
5745 // Other attributes of the comparison, such as unsignedness, are specified
5746 // by the comparison instruction that sets a condition code flags register.
5747 // That result is represented by a flags operand whose subtype is appropriate
5748 // to the unsignedness (etc.) of the comparison.
5749 //
5750 // Later, the instruction which matches both the Comparison Op (a Bool) and
5751 // the flags (produced by the Cmp) specifies the coding of the comparison op
5752 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5753
5754 // used for signed integral comparisons and fp comparisons
5755
5756 operand cmpOp()
5757 %{
5758 match(Bool);
5759
5760 format %{ "" %}
5761 interface(COND_INTER) %{
5762 equal(0x0, "eq");
5763 not_equal(0x1, "ne");
5764 less(0xb, "lt");
5765 greater_equal(0xa, "ge");
5766 less_equal(0xd, "le");
5767 greater(0xc, "gt");
5768 overflow(0x6, "vs");
5769 no_overflow(0x7, "vc");
5770 %}
5771 %}
5772
5773 // used for unsigned integral comparisons
5774
5775 operand cmpOpU()
5776 %{
5777 match(Bool);
5778
5779 format %{ "" %}
5780 interface(COND_INTER) %{
5781 equal(0x0, "eq");
5782 not_equal(0x1, "ne");
5783 less(0x3, "lo");
5784 greater_equal(0x2, "hs");
5785 less_equal(0x9, "ls");
5786 greater(0x8, "hi");
5787 overflow(0x6, "vs");
5788 no_overflow(0x7, "vc");
5789 %}
5790 %}
5791
5792 // used for certain integral comparisons which can be
5793 // converted to cbxx or tbxx instructions
5794
5795 operand cmpOpEqNe()
5796 %{
5797 match(Bool);
5798 op_cost(0);
5799 predicate(n->as_Bool()->_test._test == BoolTest::ne
5800 || n->as_Bool()->_test._test == BoolTest::eq);
5801
5802 format %{ "" %}
5803 interface(COND_INTER) %{
5804 equal(0x0, "eq");
5805 not_equal(0x1, "ne");
5806 less(0xb, "lt");
5807 greater_equal(0xa, "ge");
5808 less_equal(0xd, "le");
5809 greater(0xc, "gt");
5810 overflow(0x6, "vs");
5811 no_overflow(0x7, "vc");
5812 %}
5813 %}
5814
5815 // used for certain integral comparisons which can be
5816 // converted to cbxx or tbxx instructions
5817
5818 operand cmpOpLtGe()
5819 %{
5820 match(Bool);
5821 op_cost(0);
5822
5823 predicate(n->as_Bool()->_test._test == BoolTest::lt
5824 || n->as_Bool()->_test._test == BoolTest::ge);
5825
5826 format %{ "" %}
5827 interface(COND_INTER) %{
5828 equal(0x0, "eq");
5829 not_equal(0x1, "ne");
5830 less(0xb, "lt");
5831 greater_equal(0xa, "ge");
5832 less_equal(0xd, "le");
5833 greater(0xc, "gt");
5834 overflow(0x6, "vs");
5835 no_overflow(0x7, "vc");
5836 %}
5837 %}
5838
5839 // used for certain unsigned integral comparisons which can be
5840 // converted to cbxx or tbxx instructions
5841
5842 operand cmpOpUEqNeLeGt()
5843 %{
5844 match(Bool);
5845 op_cost(0);
5846
5847 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5848 n->as_Bool()->_test._test == BoolTest::ne ||
5849 n->as_Bool()->_test._test == BoolTest::le ||
5850 n->as_Bool()->_test._test == BoolTest::gt);
5851
5852 format %{ "" %}
5853 interface(COND_INTER) %{
5854 equal(0x0, "eq");
5855 not_equal(0x1, "ne");
5856 less(0x3, "lo");
5857 greater_equal(0x2, "hs");
5858 less_equal(0x9, "ls");
5859 greater(0x8, "hi");
5860 overflow(0x6, "vs");
5861 no_overflow(0x7, "vc");
5862 %}
5863 %}
5864
5865 // Special operand allowing long args to int ops to be truncated for free
5866
5867 operand iRegL2I(iRegL reg) %{
5868
5869 op_cost(0);
5870
5871 match(ConvL2I reg);
5872
5873 format %{ "l2i($reg)" %}
5874
5875 interface(REG_INTER)
5876 %}
5877
5878 operand iRegL2P(iRegL reg) %{
5879
5880 op_cost(0);
5881
5882 match(CastX2P reg);
5883
5884 format %{ "l2p($reg)" %}
5885
5886 interface(REG_INTER)
5887 %}
5888
5889 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5890 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5891 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5892 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5893
5894 //----------OPERAND CLASSES----------------------------------------------------
5895 // Operand Classes are groups of operands that are used as to simplify
5896 // instruction definitions by not requiring the AD writer to specify
5897 // separate instructions for every form of operand when the
5898 // instruction accepts multiple operand types with the same basic
5899 // encoding and format. The classic case of this is memory operands.
5900
5901 // memory is used to define read/write location for load/store
5902 // instruction defs. we can turn a memory op into an Address
5903
5904 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5905 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5906
5907 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5908 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5909
5910 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5911 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5912
5913 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5914 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5915
5916 // All of the memory operands. For the pipeline description.
5917 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5918 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5919 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5920
5921
5922 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5923 // operations. it allows the src to be either an iRegI or a (ConvL2I
5924 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5925 // can be elided because the 32-bit instruction will just employ the
5926 // lower 32 bits anyway.
5927 //
5928 // n.b. this does not elide all L2I conversions. if the truncated
5929 // value is consumed by more than one operation then the ConvL2I
5930 // cannot be bundled into the consuming nodes so an l2i gets planted
5931 // (actually a movw $dst $src) and the downstream instructions consume
5932 // the result of the l2i as an iRegI input. That's a shame since the
5933 // movw is actually redundant but its not too costly.
5934
5935 opclass iRegIorL2I(iRegI, iRegL2I);
5936 opclass iRegPorL2P(iRegP, iRegL2P);
5937
5938 //----------PIPELINE-----------------------------------------------------------
5939 // Rules which define the behavior of the target architectures pipeline.
5940
5941 // For specific pipelines, eg A53, define the stages of that pipeline
5942 //pipe_desc(ISS, EX1, EX2, WR);
5943 #define ISS S0
5944 #define EX1 S1
5945 #define EX2 S2
5946 #define WR S3
5947
5948 // Integer ALU reg operation
5949 pipeline %{
5950
5951 attributes %{
5952 // ARM instructions are of fixed length
5953 fixed_size_instructions; // Fixed size instructions TODO does
5954 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5955 // ARM instructions come in 32-bit word units
5956 instruction_unit_size = 4; // An instruction is 4 bytes long
5957 instruction_fetch_unit_size = 64; // The processor fetches one line
5958 instruction_fetch_units = 1; // of 64 bytes
5959
5960 // List of nop instructions
5961 nops( MachNop );
5962 %}
5963
5964 // We don't use an actual pipeline model so don't care about resources
5965 // or description. we do use pipeline classes to introduce fixed
5966 // latencies
5967
5968 //----------RESOURCES----------------------------------------------------------
5969 // Resources are the functional units available to the machine
5970
5971 resources( INS0, INS1, INS01 = INS0 | INS1,
5972 ALU0, ALU1, ALU = ALU0 | ALU1,
5973 MAC,
5974 DIV,
5975 BRANCH,
5976 LDST,
5977 NEON_FP);
5978
5979 //----------PIPELINE DESCRIPTION-----------------------------------------------
5980 // Pipeline Description specifies the stages in the machine's pipeline
5981
5982 // Define the pipeline as a generic 6 stage pipeline
5983 pipe_desc(S0, S1, S2, S3, S4, S5);
5984
5985 //----------PIPELINE CLASSES---------------------------------------------------
5986 // Pipeline Classes describe the stages in which input and output are
5987 // referenced by the hardware pipeline.
5988
5989 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5990 %{
5991 single_instruction;
5992 src1 : S1(read);
5993 src2 : S2(read);
5994 dst : S5(write);
5995 INS01 : ISS;
5996 NEON_FP : S5;
5997 %}
5998
5999 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6000 %{
6001 single_instruction;
6002 src1 : S1(read);
6003 src2 : S2(read);
6004 dst : S5(write);
6005 INS01 : ISS;
6006 NEON_FP : S5;
6007 %}
6008
6009 pipe_class fp_uop_s(vRegF dst, vRegF src)
6010 %{
6011 single_instruction;
6012 src : S1(read);
6013 dst : S5(write);
6014 INS01 : ISS;
6015 NEON_FP : S5;
6016 %}
6017
6018 pipe_class fp_uop_d(vRegD dst, vRegD src)
6019 %{
6020 single_instruction;
6021 src : S1(read);
6022 dst : S5(write);
6023 INS01 : ISS;
6024 NEON_FP : S5;
6025 %}
6026
6027 pipe_class fp_d2f(vRegF dst, vRegD src)
6028 %{
6029 single_instruction;
6030 src : S1(read);
6031 dst : S5(write);
6032 INS01 : ISS;
6033 NEON_FP : S5;
6034 %}
6035
6036 pipe_class fp_f2d(vRegD dst, vRegF src)
6037 %{
6038 single_instruction;
6039 src : S1(read);
6040 dst : S5(write);
6041 INS01 : ISS;
6042 NEON_FP : S5;
6043 %}
6044
6045 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6046 %{
6047 single_instruction;
6048 src : S1(read);
6049 dst : S5(write);
6050 INS01 : ISS;
6051 NEON_FP : S5;
6052 %}
6053
6054 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6055 %{
6056 single_instruction;
6057 src : S1(read);
6058 dst : S5(write);
6059 INS01 : ISS;
6060 NEON_FP : S5;
6061 %}
6062
6063 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6064 %{
6065 single_instruction;
6066 src : S1(read);
6067 dst : S5(write);
6068 INS01 : ISS;
6069 NEON_FP : S5;
6070 %}
6071
6072 pipe_class fp_l2f(vRegF dst, iRegL src)
6073 %{
6074 single_instruction;
6075 src : S1(read);
6076 dst : S5(write);
6077 INS01 : ISS;
6078 NEON_FP : S5;
6079 %}
6080
6081 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6082 %{
6083 single_instruction;
6084 src : S1(read);
6085 dst : S5(write);
6086 INS01 : ISS;
6087 NEON_FP : S5;
6088 %}
6089
6090 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6091 %{
6092 single_instruction;
6093 src : S1(read);
6094 dst : S5(write);
6095 INS01 : ISS;
6096 NEON_FP : S5;
6097 %}
6098
6099 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6100 %{
6101 single_instruction;
6102 src : S1(read);
6103 dst : S5(write);
6104 INS01 : ISS;
6105 NEON_FP : S5;
6106 %}
6107
6108 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6109 %{
6110 single_instruction;
6111 src : S1(read);
6112 dst : S5(write);
6113 INS01 : ISS;
6114 NEON_FP : S5;
6115 %}
6116
6117 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6118 %{
6119 single_instruction;
6120 src1 : S1(read);
6121 src2 : S2(read);
6122 dst : S5(write);
6123 INS0 : ISS;
6124 NEON_FP : S5;
6125 %}
6126
6127 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6128 %{
6129 single_instruction;
6130 src1 : S1(read);
6131 src2 : S2(read);
6132 dst : S5(write);
6133 INS0 : ISS;
6134 NEON_FP : S5;
6135 %}
6136
6137 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6138 %{
6139 single_instruction;
6140 cr : S1(read);
6141 src1 : S1(read);
6142 src2 : S1(read);
6143 dst : S3(write);
6144 INS01 : ISS;
6145 NEON_FP : S3;
6146 %}
6147
6148 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6149 %{
6150 single_instruction;
6151 cr : S1(read);
6152 src1 : S1(read);
6153 src2 : S1(read);
6154 dst : S3(write);
6155 INS01 : ISS;
6156 NEON_FP : S3;
6157 %}
6158
6159 pipe_class fp_imm_s(vRegF dst)
6160 %{
6161 single_instruction;
6162 dst : S3(write);
6163 INS01 : ISS;
6164 NEON_FP : S3;
6165 %}
6166
6167 pipe_class fp_imm_d(vRegD dst)
6168 %{
6169 single_instruction;
6170 dst : S3(write);
6171 INS01 : ISS;
6172 NEON_FP : S3;
6173 %}
6174
6175 pipe_class fp_load_constant_s(vRegF dst)
6176 %{
6177 single_instruction;
6178 dst : S4(write);
6179 INS01 : ISS;
6180 NEON_FP : S4;
6181 %}
6182
6183 pipe_class fp_load_constant_d(vRegD dst)
6184 %{
6185 single_instruction;
6186 dst : S4(write);
6187 INS01 : ISS;
6188 NEON_FP : S4;
6189 %}
6190
6191 //------- Integer ALU operations --------------------------
6192
6193 // Integer ALU reg-reg operation
6194 // Operands needed in EX1, result generated in EX2
6195 // Eg. ADD x0, x1, x2
6196 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6197 %{
6198 single_instruction;
6199 dst : EX2(write);
6200 src1 : EX1(read);
6201 src2 : EX1(read);
6202 INS01 : ISS; // Dual issue as instruction 0 or 1
6203 ALU : EX2;
6204 %}
6205
6206 // Integer ALU reg-reg operation with constant shift
6207 // Shifted register must be available in LATE_ISS instead of EX1
6208 // Eg. ADD x0, x1, x2, LSL #2
6209 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6210 %{
6211 single_instruction;
6212 dst : EX2(write);
6213 src1 : EX1(read);
6214 src2 : ISS(read);
6215 INS01 : ISS;
6216 ALU : EX2;
6217 %}
6218
6219 // Integer ALU reg operation with constant shift
6220 // Eg. LSL x0, x1, #shift
6221 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6222 %{
6223 single_instruction;
6224 dst : EX2(write);
6225 src1 : ISS(read);
6226 INS01 : ISS;
6227 ALU : EX2;
6228 %}
6229
6230 // Integer ALU reg-reg operation with variable shift
6231 // Both operands must be available in LATE_ISS instead of EX1
6232 // Result is available in EX1 instead of EX2
6233 // Eg. LSLV x0, x1, x2
6234 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6235 %{
6236 single_instruction;
6237 dst : EX1(write);
6238 src1 : ISS(read);
6239 src2 : ISS(read);
6240 INS01 : ISS;
6241 ALU : EX1;
6242 %}
6243
6244 // Integer ALU reg-reg operation with extract
6245 // As for _vshift above, but result generated in EX2
6246 // Eg. EXTR x0, x1, x2, #N
6247 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6248 %{
6249 single_instruction;
6250 dst : EX2(write);
6251 src1 : ISS(read);
6252 src2 : ISS(read);
6253 INS1 : ISS; // Can only dual issue as Instruction 1
6254 ALU : EX1;
6255 %}
6256
6257 // Integer ALU reg operation
6258 // Eg. NEG x0, x1
6259 pipe_class ialu_reg(iRegI dst, iRegI src)
6260 %{
6261 single_instruction;
6262 dst : EX2(write);
6263 src : EX1(read);
6264 INS01 : ISS;
6265 ALU : EX2;
6266 %}
6267
6268 // Integer ALU reg mmediate operation
6269 // Eg. ADD x0, x1, #N
6270 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6271 %{
6272 single_instruction;
6273 dst : EX2(write);
6274 src1 : EX1(read);
6275 INS01 : ISS;
6276 ALU : EX2;
6277 %}
6278
6279 // Integer ALU immediate operation (no source operands)
6280 // Eg. MOV x0, #N
6281 pipe_class ialu_imm(iRegI dst)
6282 %{
6283 single_instruction;
6284 dst : EX1(write);
6285 INS01 : ISS;
6286 ALU : EX1;
6287 %}
6288
6289 //------- Compare operation -------------------------------
6290
6291 // Compare reg-reg
6292 // Eg. CMP x0, x1
6293 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6294 %{
6295 single_instruction;
6296 // fixed_latency(16);
6297 cr : EX2(write);
6298 op1 : EX1(read);
6299 op2 : EX1(read);
6300 INS01 : ISS;
6301 ALU : EX2;
6302 %}
6303
6304 // Compare reg-reg
6305 // Eg. CMP x0, #N
6306 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6307 %{
6308 single_instruction;
6309 // fixed_latency(16);
6310 cr : EX2(write);
6311 op1 : EX1(read);
6312 INS01 : ISS;
6313 ALU : EX2;
6314 %}
6315
6316 //------- Conditional instructions ------------------------
6317
6318 // Conditional no operands
6319 // Eg. CSINC x0, zr, zr, <cond>
6320 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6321 %{
6322 single_instruction;
6323 cr : EX1(read);
6324 dst : EX2(write);
6325 INS01 : ISS;
6326 ALU : EX2;
6327 %}
6328
6329 // Conditional 2 operand
6330 // EG. CSEL X0, X1, X2, <cond>
6331 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6332 %{
6333 single_instruction;
6334 cr : EX1(read);
6335 src1 : EX1(read);
6336 src2 : EX1(read);
6337 dst : EX2(write);
6338 INS01 : ISS;
6339 ALU : EX2;
6340 %}
6341
6342 // Conditional 2 operand
6343 // EG. CSEL X0, X1, X2, <cond>
6344 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6345 %{
6346 single_instruction;
6347 cr : EX1(read);
6348 src : EX1(read);
6349 dst : EX2(write);
6350 INS01 : ISS;
6351 ALU : EX2;
6352 %}
6353
6354 //------- Multiply pipeline operations --------------------
6355
6356 // Multiply reg-reg
6357 // Eg. MUL w0, w1, w2
6358 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6359 %{
6360 single_instruction;
6361 dst : WR(write);
6362 src1 : ISS(read);
6363 src2 : ISS(read);
6364 INS01 : ISS;
6365 MAC : WR;
6366 %}
6367
6368 // Multiply accumulate
6369 // Eg. MADD w0, w1, w2, w3
6370 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6371 %{
6372 single_instruction;
6373 dst : WR(write);
6374 src1 : ISS(read);
6375 src2 : ISS(read);
6376 src3 : ISS(read);
6377 INS01 : ISS;
6378 MAC : WR;
6379 %}
6380
6381 // Eg. MUL w0, w1, w2
6382 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6383 %{
6384 single_instruction;
6385 fixed_latency(3); // Maximum latency for 64 bit mul
6386 dst : WR(write);
6387 src1 : ISS(read);
6388 src2 : ISS(read);
6389 INS01 : ISS;
6390 MAC : WR;
6391 %}
6392
6393 // Multiply accumulate
6394 // Eg. MADD w0, w1, w2, w3
6395 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6396 %{
6397 single_instruction;
6398 fixed_latency(3); // Maximum latency for 64 bit mul
6399 dst : WR(write);
6400 src1 : ISS(read);
6401 src2 : ISS(read);
6402 src3 : ISS(read);
6403 INS01 : ISS;
6404 MAC : WR;
6405 %}
6406
6407 //------- Divide pipeline operations --------------------
6408
6409 // Eg. SDIV w0, w1, w2
6410 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6411 %{
6412 single_instruction;
6413 fixed_latency(8); // Maximum latency for 32 bit divide
6414 dst : WR(write);
6415 src1 : ISS(read);
6416 src2 : ISS(read);
6417 INS0 : ISS; // Can only dual issue as instruction 0
6418 DIV : WR;
6419 %}
6420
6421 // Eg. SDIV x0, x1, x2
6422 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6423 %{
6424 single_instruction;
6425 fixed_latency(16); // Maximum latency for 64 bit divide
6426 dst : WR(write);
6427 src1 : ISS(read);
6428 src2 : ISS(read);
6429 INS0 : ISS; // Can only dual issue as instruction 0
6430 DIV : WR;
6431 %}
6432
6433 //------- Load pipeline operations ------------------------
6434
6435 // Load - prefetch
6436 // Eg. PFRM <mem>
6437 pipe_class iload_prefetch(memory mem)
6438 %{
6439 single_instruction;
6440 mem : ISS(read);
6441 INS01 : ISS;
6442 LDST : WR;
6443 %}
6444
6445 // Load - reg, mem
6446 // Eg. LDR x0, <mem>
6447 pipe_class iload_reg_mem(iRegI dst, memory mem)
6448 %{
6449 single_instruction;
6450 dst : WR(write);
6451 mem : ISS(read);
6452 INS01 : ISS;
6453 LDST : WR;
6454 %}
6455
6456 // Load - reg, reg
6457 // Eg. LDR x0, [sp, x1]
6458 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6459 %{
6460 single_instruction;
6461 dst : WR(write);
6462 src : ISS(read);
6463 INS01 : ISS;
6464 LDST : WR;
6465 %}
6466
6467 //------- Store pipeline operations -----------------------
6468
6469 // Store - zr, mem
6470 // Eg. STR zr, <mem>
6471 pipe_class istore_mem(memory mem)
6472 %{
6473 single_instruction;
6474 mem : ISS(read);
6475 INS01 : ISS;
6476 LDST : WR;
6477 %}
6478
6479 // Store - reg, mem
6480 // Eg. STR x0, <mem>
6481 pipe_class istore_reg_mem(iRegI src, memory mem)
6482 %{
6483 single_instruction;
6484 mem : ISS(read);
6485 src : EX2(read);
6486 INS01 : ISS;
6487 LDST : WR;
6488 %}
6489
6490 // Store - reg, reg
6491 // Eg. STR x0, [sp, x1]
6492 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6493 %{
6494 single_instruction;
6495 dst : ISS(read);
6496 src : EX2(read);
6497 INS01 : ISS;
6498 LDST : WR;
6499 %}
6500
6501 //------- Store pipeline operations -----------------------
6502
6503 // Branch
6504 pipe_class pipe_branch()
6505 %{
6506 single_instruction;
6507 INS01 : ISS;
6508 BRANCH : EX1;
6509 %}
6510
6511 // Conditional branch
6512 pipe_class pipe_branch_cond(rFlagsReg cr)
6513 %{
6514 single_instruction;
6515 cr : EX1(read);
6516 INS01 : ISS;
6517 BRANCH : EX1;
6518 %}
6519
6520 // Compare & Branch
6521 // EG. CBZ/CBNZ
6522 pipe_class pipe_cmp_branch(iRegI op1)
6523 %{
6524 single_instruction;
6525 op1 : EX1(read);
6526 INS01 : ISS;
6527 BRANCH : EX1;
6528 %}
6529
6530 //------- Synchronisation operations ----------------------
6531
6532 // Any operation requiring serialization.
6533 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6534 pipe_class pipe_serial()
6535 %{
6536 single_instruction;
6537 force_serialization;
6538 fixed_latency(16);
6539 INS01 : ISS(2); // Cannot dual issue with any other instruction
6540 LDST : WR;
6541 %}
6542
6543 // Generic big/slow expanded idiom - also serialized
6544 pipe_class pipe_slow()
6545 %{
6546 instruction_count(10);
6547 multiple_bundles;
6548 force_serialization;
6549 fixed_latency(16);
6550 INS01 : ISS(2); // Cannot dual issue with any other instruction
6551 LDST : WR;
6552 %}
6553
6554 // Empty pipeline class
6555 pipe_class pipe_class_empty()
6556 %{
6557 single_instruction;
6558 fixed_latency(0);
6559 %}
6560
6561 // Default pipeline class.
6562 pipe_class pipe_class_default()
6563 %{
6564 single_instruction;
6565 fixed_latency(2);
6566 %}
6567
6568 // Pipeline class for compares.
6569 pipe_class pipe_class_compare()
6570 %{
6571 single_instruction;
6572 fixed_latency(16);
6573 %}
6574
6575 // Pipeline class for memory operations.
6576 pipe_class pipe_class_memory()
6577 %{
6578 single_instruction;
6579 fixed_latency(16);
6580 %}
6581
6582 // Pipeline class for call.
6583 pipe_class pipe_class_call()
6584 %{
6585 single_instruction;
6586 fixed_latency(100);
6587 %}
6588
6589 // Define the class for the Nop node.
6590 define %{
6591 MachNop = pipe_class_empty;
6592 %}
6593
6594 %}
6595 //----------INSTRUCTIONS-------------------------------------------------------
6596 //
6597 // match -- States which machine-independent subtree may be replaced
6598 // by this instruction.
6599 // ins_cost -- The estimated cost of this instruction is used by instruction
6600 // selection to identify a minimum cost tree of machine
6601 // instructions that matches a tree of machine-independent
6602 // instructions.
6603 // format -- A string providing the disassembly for this instruction.
6604 // The value of an instruction's operand may be inserted
6605 // by referring to it with a '$' prefix.
6606 // opcode -- Three instruction opcodes may be provided. These are referred
6607 // to within an encode class as $primary, $secondary, and $tertiary
6608 // rrspectively. The primary opcode is commonly used to
6609 // indicate the type of machine instruction, while secondary
6610 // and tertiary are often used for prefix options or addressing
6611 // modes.
6612 // ins_encode -- A list of encode classes with parameters. The encode class
6613 // name must have been defined in an 'enc_class' specification
6614 // in the encode section of the architecture description.
6615
6616 // ============================================================================
6617 // Memory (Load/Store) Instructions
6618
6619 // Load Instructions
6620
6621 // Load Byte (8 bit signed)
6622 instruct loadB(iRegINoSp dst, memory1 mem)
6623 %{
6624 match(Set dst (LoadB mem));
6625 predicate(!needs_acquiring_load(n));
6626
6627 ins_cost(4 * INSN_COST);
6628 format %{ "ldrsbw $dst, $mem\t# byte" %}
6629
6630 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6631
6632 ins_pipe(iload_reg_mem);
6633 %}
6634
6635 // Load Byte (8 bit signed) into long
6636 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6637 %{
6638 match(Set dst (ConvI2L (LoadB mem)));
6639 predicate(!needs_acquiring_load(n->in(1)));
6640
6641 ins_cost(4 * INSN_COST);
6642 format %{ "ldrsb $dst, $mem\t# byte" %}
6643
6644 ins_encode(aarch64_enc_ldrsb(dst, mem));
6645
6646 ins_pipe(iload_reg_mem);
6647 %}
6648
6649 // Load Byte (8 bit unsigned)
6650 instruct loadUB(iRegINoSp dst, memory1 mem)
6651 %{
6652 match(Set dst (LoadUB mem));
6653 predicate(!needs_acquiring_load(n));
6654
6655 ins_cost(4 * INSN_COST);
6656 format %{ "ldrbw $dst, $mem\t# byte" %}
6657
6658 ins_encode(aarch64_enc_ldrb(dst, mem));
6659
6660 ins_pipe(iload_reg_mem);
6661 %}
6662
6663 // Load Byte (8 bit unsigned) into long
6664 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6665 %{
6666 match(Set dst (ConvI2L (LoadUB mem)));
6667 predicate(!needs_acquiring_load(n->in(1)));
6668
6669 ins_cost(4 * INSN_COST);
6670 format %{ "ldrb $dst, $mem\t# byte" %}
6671
6672 ins_encode(aarch64_enc_ldrb(dst, mem));
6673
6674 ins_pipe(iload_reg_mem);
6675 %}
6676
6677 // Load Short (16 bit signed)
6678 instruct loadS(iRegINoSp dst, memory2 mem)
6679 %{
6680 match(Set dst (LoadS mem));
6681 predicate(!needs_acquiring_load(n));
6682
6683 ins_cost(4 * INSN_COST);
6684 format %{ "ldrshw $dst, $mem\t# short" %}
6685
6686 ins_encode(aarch64_enc_ldrshw(dst, mem));
6687
6688 ins_pipe(iload_reg_mem);
6689 %}
6690
6691 // Load Short (16 bit signed) into long
6692 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6693 %{
6694 match(Set dst (ConvI2L (LoadS mem)));
6695 predicate(!needs_acquiring_load(n->in(1)));
6696
6697 ins_cost(4 * INSN_COST);
6698 format %{ "ldrsh $dst, $mem\t# short" %}
6699
6700 ins_encode(aarch64_enc_ldrsh(dst, mem));
6701
6702 ins_pipe(iload_reg_mem);
6703 %}
6704
6705 // Load Char (16 bit unsigned)
6706 instruct loadUS(iRegINoSp dst, memory2 mem)
6707 %{
6708 match(Set dst (LoadUS mem));
6709 predicate(!needs_acquiring_load(n));
6710
6711 ins_cost(4 * INSN_COST);
6712 format %{ "ldrh $dst, $mem\t# short" %}
6713
6714 ins_encode(aarch64_enc_ldrh(dst, mem));
6715
6716 ins_pipe(iload_reg_mem);
6717 %}
6718
6719 // Load Short/Char (16 bit unsigned) into long
6720 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6721 %{
6722 match(Set dst (ConvI2L (LoadUS mem)));
6723 predicate(!needs_acquiring_load(n->in(1)));
6724
6725 ins_cost(4 * INSN_COST);
6726 format %{ "ldrh $dst, $mem\t# short" %}
6727
6728 ins_encode(aarch64_enc_ldrh(dst, mem));
6729
6730 ins_pipe(iload_reg_mem);
6731 %}
6732
6733 // Load Integer (32 bit signed)
6734 instruct loadI(iRegINoSp dst, memory4 mem)
6735 %{
6736 match(Set dst (LoadI mem));
6737 predicate(!needs_acquiring_load(n));
6738
6739 ins_cost(4 * INSN_COST);
6740 format %{ "ldrw $dst, $mem\t# int" %}
6741
6742 ins_encode(aarch64_enc_ldrw(dst, mem));
6743
6744 ins_pipe(iload_reg_mem);
6745 %}
6746
6747 // Load Integer (32 bit signed) into long
6748 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6749 %{
6750 match(Set dst (ConvI2L (LoadI mem)));
6751 predicate(!needs_acquiring_load(n->in(1)));
6752
6753 ins_cost(4 * INSN_COST);
6754 format %{ "ldrsw $dst, $mem\t# int" %}
6755
6756 ins_encode(aarch64_enc_ldrsw(dst, mem));
6757
6758 ins_pipe(iload_reg_mem);
6759 %}
6760
6761 // Load Integer (32 bit unsigned) into long
6762 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6763 %{
6764 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6765 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6766
6767 ins_cost(4 * INSN_COST);
6768 format %{ "ldrw $dst, $mem\t# int" %}
6769
6770 ins_encode(aarch64_enc_ldrw(dst, mem));
6771
6772 ins_pipe(iload_reg_mem);
6773 %}
6774
6775 // Load Long (64 bit signed)
6776 instruct loadL(iRegLNoSp dst, memory8 mem)
6777 %{
6778 match(Set dst (LoadL mem));
6779 predicate(!needs_acquiring_load(n));
6780
6781 ins_cost(4 * INSN_COST);
6782 format %{ "ldr $dst, $mem\t# int" %}
6783
6784 ins_encode(aarch64_enc_ldr(dst, mem));
6785
6786 ins_pipe(iload_reg_mem);
6787 %}
6788
6789 // Load Range
6790 instruct loadRange(iRegINoSp dst, memory4 mem)
6791 %{
6792 match(Set dst (LoadRange mem));
6793
6794 ins_cost(4 * INSN_COST);
6795 format %{ "ldrw $dst, $mem\t# range" %}
6796
6797 ins_encode(aarch64_enc_ldrw(dst, mem));
6798
6799 ins_pipe(iload_reg_mem);
6800 %}
6801
6802 // Load Pointer
6803 instruct loadP(iRegPNoSp dst, memory8 mem)
6804 %{
6805 match(Set dst (LoadP mem));
6806 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6807
6808 ins_cost(4 * INSN_COST);
6809 format %{ "ldr $dst, $mem\t# ptr" %}
6810
6811 ins_encode(aarch64_enc_ldr(dst, mem));
6812
6813 ins_pipe(iload_reg_mem);
6814 %}
6815
6816 // Load Compressed Pointer
6817 instruct loadN(iRegNNoSp dst, memory4 mem)
6818 %{
6819 match(Set dst (LoadN mem));
6820 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6821
6822 ins_cost(4 * INSN_COST);
6823 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6824
6825 ins_encode(aarch64_enc_ldrw(dst, mem));
6826
6827 ins_pipe(iload_reg_mem);
6828 %}
6829
6830 // Load Klass Pointer
6831 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6832 %{
6833 match(Set dst (LoadKlass mem));
6834 predicate(!needs_acquiring_load(n));
6835
6836 ins_cost(4 * INSN_COST);
6837 format %{ "ldr $dst, $mem\t# class" %}
6838
6839 ins_encode(aarch64_enc_ldr(dst, mem));
6840
6841 ins_pipe(iload_reg_mem);
6842 %}
6843
6844 // Load Narrow Klass Pointer
6845 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6846 %{
6847 match(Set dst (LoadNKlass mem));
6848 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6849
6850 ins_cost(4 * INSN_COST);
6851 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6852
6853 ins_encode(aarch64_enc_ldrw(dst, mem));
6854
6855 ins_pipe(iload_reg_mem);
6856 %}
6857
6858 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6859 %{
6860 match(Set dst (LoadNKlass mem));
6861 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6862
6863 ins_cost(4 * INSN_COST);
6864 format %{
6865 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6866 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6867 %}
6868 ins_encode %{
6869 // inlined aarch64_enc_ldrw
6870 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6871 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6872 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6873 %}
6874 ins_pipe(iload_reg_mem);
6875 %}
6876
6877 // Load Float
6878 instruct loadF(vRegF dst, memory4 mem)
6879 %{
6880 match(Set dst (LoadF mem));
6881 predicate(!needs_acquiring_load(n));
6882
6883 ins_cost(4 * INSN_COST);
6884 format %{ "ldrs $dst, $mem\t# float" %}
6885
6886 ins_encode( aarch64_enc_ldrs(dst, mem) );
6887
6888 ins_pipe(pipe_class_memory);
6889 %}
6890
6891 // Load Double
6892 instruct loadD(vRegD dst, memory8 mem)
6893 %{
6894 match(Set dst (LoadD mem));
6895 predicate(!needs_acquiring_load(n));
6896
6897 ins_cost(4 * INSN_COST);
6898 format %{ "ldrd $dst, $mem\t# double" %}
6899
6900 ins_encode( aarch64_enc_ldrd(dst, mem) );
6901
6902 ins_pipe(pipe_class_memory);
6903 %}
6904
6905
6906 // Load Int Constant
6907 instruct loadConI(iRegINoSp dst, immI src)
6908 %{
6909 match(Set dst src);
6910
6911 ins_cost(INSN_COST);
6912 format %{ "mov $dst, $src\t# int" %}
6913
6914 ins_encode( aarch64_enc_movw_imm(dst, src) );
6915
6916 ins_pipe(ialu_imm);
6917 %}
6918
6919 // Load Long Constant
6920 instruct loadConL(iRegLNoSp dst, immL src)
6921 %{
6922 match(Set dst src);
6923
6924 ins_cost(INSN_COST);
6925 format %{ "mov $dst, $src\t# long" %}
6926
6927 ins_encode( aarch64_enc_mov_imm(dst, src) );
6928
6929 ins_pipe(ialu_imm);
6930 %}
6931
6932 // Load Pointer Constant
6933
6934 instruct loadConP(iRegPNoSp dst, immP con)
6935 %{
6936 match(Set dst con);
6937
6938 ins_cost(INSN_COST * 4);
6939 format %{
6940 "mov $dst, $con\t# ptr\n\t"
6941 %}
6942
6943 ins_encode(aarch64_enc_mov_p(dst, con));
6944
6945 ins_pipe(ialu_imm);
6946 %}
6947
6948 // Load Null Pointer Constant
6949
6950 instruct loadConP0(iRegPNoSp dst, immP0 con)
6951 %{
6952 match(Set dst con);
6953
6954 ins_cost(INSN_COST);
6955 format %{ "mov $dst, $con\t# nullptr ptr" %}
6956
6957 ins_encode(aarch64_enc_mov_p0(dst, con));
6958
6959 ins_pipe(ialu_imm);
6960 %}
6961
6962 // Load Pointer Constant One
6963
6964 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6965 %{
6966 match(Set dst con);
6967
6968 ins_cost(INSN_COST);
6969 format %{ "mov $dst, $con\t# nullptr ptr" %}
6970
6971 ins_encode(aarch64_enc_mov_p1(dst, con));
6972
6973 ins_pipe(ialu_imm);
6974 %}
6975
6976 // Load Narrow Pointer Constant
6977
6978 instruct loadConN(iRegNNoSp dst, immN con)
6979 %{
6980 match(Set dst con);
6981
6982 ins_cost(INSN_COST * 4);
6983 format %{ "mov $dst, $con\t# compressed ptr" %}
6984
6985 ins_encode(aarch64_enc_mov_n(dst, con));
6986
6987 ins_pipe(ialu_imm);
6988 %}
6989
6990 // Load Narrow Null Pointer Constant
6991
6992 instruct loadConN0(iRegNNoSp dst, immN0 con)
6993 %{
6994 match(Set dst con);
6995
6996 ins_cost(INSN_COST);
6997 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6998
6999 ins_encode(aarch64_enc_mov_n0(dst, con));
7000
7001 ins_pipe(ialu_imm);
7002 %}
7003
7004 // Load Narrow Klass Constant
7005
7006 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7007 %{
7008 match(Set dst con);
7009
7010 ins_cost(INSN_COST);
7011 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7012
7013 ins_encode(aarch64_enc_mov_nk(dst, con));
7014
7015 ins_pipe(ialu_imm);
7016 %}
7017
7018 // Load Packed Float Constant
7019
7020 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7021 match(Set dst con);
7022 ins_cost(INSN_COST * 4);
7023 format %{ "fmovs $dst, $con"%}
7024 ins_encode %{
7025 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7026 %}
7027
7028 ins_pipe(fp_imm_s);
7029 %}
7030
7031 // Load Float Constant
7032
7033 instruct loadConF(vRegF dst, immF con) %{
7034 match(Set dst con);
7035
7036 ins_cost(INSN_COST * 4);
7037
7038 format %{
7039 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7040 %}
7041
7042 ins_encode %{
7043 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7044 %}
7045
7046 ins_pipe(fp_load_constant_s);
7047 %}
7048
7049 // Load Packed Double Constant
7050
7051 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7052 match(Set dst con);
7053 ins_cost(INSN_COST);
7054 format %{ "fmovd $dst, $con"%}
7055 ins_encode %{
7056 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7057 %}
7058
7059 ins_pipe(fp_imm_d);
7060 %}
7061
7062 // Load Double Constant
7063
7064 instruct loadConD(vRegD dst, immD con) %{
7065 match(Set dst con);
7066
7067 ins_cost(INSN_COST * 5);
7068 format %{
7069 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7070 %}
7071
7072 ins_encode %{
7073 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7074 %}
7075
7076 ins_pipe(fp_load_constant_d);
7077 %}
7078
7079 // Load Half Float Constant
7080 // The "ldr" instruction loads a 32-bit word from the constant pool into a
7081 // 32-bit register but only the bottom half will be populated and the top
7082 // 16 bits are zero.
7083 instruct loadConH(vRegF dst, immH con) %{
7084 match(Set dst con);
7085 format %{
7086 "ldrs $dst, [$constantaddress]\t# load from constant table: half float=$con\n\t"
7087 %}
7088 ins_encode %{
7089 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7090 %}
7091 ins_pipe(fp_load_constant_s);
7092 %}
7093
7094 // Store Instructions
7095
7096 // Store Byte
7097 instruct storeB(iRegIorL2I src, memory1 mem)
7098 %{
7099 match(Set mem (StoreB mem src));
7100 predicate(!needs_releasing_store(n));
7101
7102 ins_cost(INSN_COST);
7103 format %{ "strb $src, $mem\t# byte" %}
7104
7105 ins_encode(aarch64_enc_strb(src, mem));
7106
7107 ins_pipe(istore_reg_mem);
7108 %}
7109
7110
7111 instruct storeimmB0(immI0 zero, memory1 mem)
7112 %{
7113 match(Set mem (StoreB mem zero));
7114 predicate(!needs_releasing_store(n));
7115
7116 ins_cost(INSN_COST);
7117 format %{ "strb rscractch2, $mem\t# byte" %}
7118
7119 ins_encode(aarch64_enc_strb0(mem));
7120
7121 ins_pipe(istore_mem);
7122 %}
7123
7124 // Store Char/Short
7125 instruct storeC(iRegIorL2I src, memory2 mem)
7126 %{
7127 match(Set mem (StoreC mem src));
7128 predicate(!needs_releasing_store(n));
7129
7130 ins_cost(INSN_COST);
7131 format %{ "strh $src, $mem\t# short" %}
7132
7133 ins_encode(aarch64_enc_strh(src, mem));
7134
7135 ins_pipe(istore_reg_mem);
7136 %}
7137
7138 instruct storeimmC0(immI0 zero, memory2 mem)
7139 %{
7140 match(Set mem (StoreC mem zero));
7141 predicate(!needs_releasing_store(n));
7142
7143 ins_cost(INSN_COST);
7144 format %{ "strh zr, $mem\t# short" %}
7145
7146 ins_encode(aarch64_enc_strh0(mem));
7147
7148 ins_pipe(istore_mem);
7149 %}
7150
7151 // Store Integer
7152
7153 instruct storeI(iRegIorL2I src, memory4 mem)
7154 %{
7155 match(Set mem(StoreI mem src));
7156 predicate(!needs_releasing_store(n));
7157
7158 ins_cost(INSN_COST);
7159 format %{ "strw $src, $mem\t# int" %}
7160
7161 ins_encode(aarch64_enc_strw(src, mem));
7162
7163 ins_pipe(istore_reg_mem);
7164 %}
7165
7166 instruct storeimmI0(immI0 zero, memory4 mem)
7167 %{
7168 match(Set mem(StoreI mem zero));
7169 predicate(!needs_releasing_store(n));
7170
7171 ins_cost(INSN_COST);
7172 format %{ "strw zr, $mem\t# int" %}
7173
7174 ins_encode(aarch64_enc_strw0(mem));
7175
7176 ins_pipe(istore_mem);
7177 %}
7178
7179 // Store Long (64 bit signed)
7180 instruct storeL(iRegL src, memory8 mem)
7181 %{
7182 match(Set mem (StoreL mem src));
7183 predicate(!needs_releasing_store(n));
7184
7185 ins_cost(INSN_COST);
7186 format %{ "str $src, $mem\t# int" %}
7187
7188 ins_encode(aarch64_enc_str(src, mem));
7189
7190 ins_pipe(istore_reg_mem);
7191 %}
7192
7193 // Store Long (64 bit signed)
7194 instruct storeimmL0(immL0 zero, memory8 mem)
7195 %{
7196 match(Set mem (StoreL mem zero));
7197 predicate(!needs_releasing_store(n));
7198
7199 ins_cost(INSN_COST);
7200 format %{ "str zr, $mem\t# int" %}
7201
7202 ins_encode(aarch64_enc_str0(mem));
7203
7204 ins_pipe(istore_mem);
7205 %}
7206
7207 // Store Pointer
7208 instruct storeP(iRegP src, memory8 mem)
7209 %{
7210 match(Set mem (StoreP mem src));
7211 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7212
7213 ins_cost(INSN_COST);
7214 format %{ "str $src, $mem\t# ptr" %}
7215
7216 ins_encode(aarch64_enc_str(src, mem));
7217
7218 ins_pipe(istore_reg_mem);
7219 %}
7220
7221 // Store Pointer
7222 instruct storeimmP0(immP0 zero, memory8 mem)
7223 %{
7224 match(Set mem (StoreP mem zero));
7225 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7226
7227 ins_cost(INSN_COST);
7228 format %{ "str zr, $mem\t# ptr" %}
7229
7230 ins_encode(aarch64_enc_str0(mem));
7231
7232 ins_pipe(istore_mem);
7233 %}
7234
7235 // Store Compressed Pointer
7236 instruct storeN(iRegN src, memory4 mem)
7237 %{
7238 match(Set mem (StoreN mem src));
7239 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7240
7241 ins_cost(INSN_COST);
7242 format %{ "strw $src, $mem\t# compressed ptr" %}
7243
7244 ins_encode(aarch64_enc_strw(src, mem));
7245
7246 ins_pipe(istore_reg_mem);
7247 %}
7248
7249 instruct storeImmN0(immN0 zero, memory4 mem)
7250 %{
7251 match(Set mem (StoreN mem zero));
7252 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7253
7254 ins_cost(INSN_COST);
7255 format %{ "strw zr, $mem\t# compressed ptr" %}
7256
7257 ins_encode(aarch64_enc_strw0(mem));
7258
7259 ins_pipe(istore_mem);
7260 %}
7261
7262 // Store Float
7263 instruct storeF(vRegF src, memory4 mem)
7264 %{
7265 match(Set mem (StoreF mem src));
7266 predicate(!needs_releasing_store(n));
7267
7268 ins_cost(INSN_COST);
7269 format %{ "strs $src, $mem\t# float" %}
7270
7271 ins_encode( aarch64_enc_strs(src, mem) );
7272
7273 ins_pipe(pipe_class_memory);
7274 %}
7275
7276 // TODO
7277 // implement storeImmF0 and storeFImmPacked
7278
7279 // Store Double
7280 instruct storeD(vRegD src, memory8 mem)
7281 %{
7282 match(Set mem (StoreD mem src));
7283 predicate(!needs_releasing_store(n));
7284
7285 ins_cost(INSN_COST);
7286 format %{ "strd $src, $mem\t# double" %}
7287
7288 ins_encode( aarch64_enc_strd(src, mem) );
7289
7290 ins_pipe(pipe_class_memory);
7291 %}
7292
7293 // Store Compressed Klass Pointer
7294 instruct storeNKlass(iRegN src, memory4 mem)
7295 %{
7296 predicate(!needs_releasing_store(n));
7297 match(Set mem (StoreNKlass mem src));
7298
7299 ins_cost(INSN_COST);
7300 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7301
7302 ins_encode(aarch64_enc_strw(src, mem));
7303
7304 ins_pipe(istore_reg_mem);
7305 %}
7306
7307 // TODO
7308 // implement storeImmD0 and storeDImmPacked
7309
7310 // prefetch instructions
7311 // Must be safe to execute with invalid address (cannot fault).
7312
7313 instruct prefetchalloc( memory8 mem ) %{
7314 match(PrefetchAllocation mem);
7315
7316 ins_cost(INSN_COST);
7317 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7318
7319 ins_encode( aarch64_enc_prefetchw(mem) );
7320
7321 ins_pipe(iload_prefetch);
7322 %}
7323
7324 // ---------------- volatile loads and stores ----------------
7325
7326 // Load Byte (8 bit signed)
7327 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7328 %{
7329 match(Set dst (LoadB mem));
7330
7331 ins_cost(VOLATILE_REF_COST);
7332 format %{ "ldarsb $dst, $mem\t# byte" %}
7333
7334 ins_encode(aarch64_enc_ldarsb(dst, mem));
7335
7336 ins_pipe(pipe_serial);
7337 %}
7338
7339 // Load Byte (8 bit signed) into long
7340 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7341 %{
7342 match(Set dst (ConvI2L (LoadB mem)));
7343
7344 ins_cost(VOLATILE_REF_COST);
7345 format %{ "ldarsb $dst, $mem\t# byte" %}
7346
7347 ins_encode(aarch64_enc_ldarsb(dst, mem));
7348
7349 ins_pipe(pipe_serial);
7350 %}
7351
7352 // Load Byte (8 bit unsigned)
7353 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7354 %{
7355 match(Set dst (LoadUB mem));
7356
7357 ins_cost(VOLATILE_REF_COST);
7358 format %{ "ldarb $dst, $mem\t# byte" %}
7359
7360 ins_encode(aarch64_enc_ldarb(dst, mem));
7361
7362 ins_pipe(pipe_serial);
7363 %}
7364
7365 // Load Byte (8 bit unsigned) into long
7366 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7367 %{
7368 match(Set dst (ConvI2L (LoadUB mem)));
7369
7370 ins_cost(VOLATILE_REF_COST);
7371 format %{ "ldarb $dst, $mem\t# byte" %}
7372
7373 ins_encode(aarch64_enc_ldarb(dst, mem));
7374
7375 ins_pipe(pipe_serial);
7376 %}
7377
7378 // Load Short (16 bit signed)
7379 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7380 %{
7381 match(Set dst (LoadS mem));
7382
7383 ins_cost(VOLATILE_REF_COST);
7384 format %{ "ldarshw $dst, $mem\t# short" %}
7385
7386 ins_encode(aarch64_enc_ldarshw(dst, mem));
7387
7388 ins_pipe(pipe_serial);
7389 %}
7390
7391 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7392 %{
7393 match(Set dst (LoadUS mem));
7394
7395 ins_cost(VOLATILE_REF_COST);
7396 format %{ "ldarhw $dst, $mem\t# short" %}
7397
7398 ins_encode(aarch64_enc_ldarhw(dst, mem));
7399
7400 ins_pipe(pipe_serial);
7401 %}
7402
7403 // Load Short/Char (16 bit unsigned) into long
7404 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7405 %{
7406 match(Set dst (ConvI2L (LoadUS mem)));
7407
7408 ins_cost(VOLATILE_REF_COST);
7409 format %{ "ldarh $dst, $mem\t# short" %}
7410
7411 ins_encode(aarch64_enc_ldarh(dst, mem));
7412
7413 ins_pipe(pipe_serial);
7414 %}
7415
7416 // Load Short/Char (16 bit signed) into long
7417 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7418 %{
7419 match(Set dst (ConvI2L (LoadS mem)));
7420
7421 ins_cost(VOLATILE_REF_COST);
7422 format %{ "ldarh $dst, $mem\t# short" %}
7423
7424 ins_encode(aarch64_enc_ldarsh(dst, mem));
7425
7426 ins_pipe(pipe_serial);
7427 %}
7428
7429 // Load Integer (32 bit signed)
7430 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7431 %{
7432 match(Set dst (LoadI mem));
7433
7434 ins_cost(VOLATILE_REF_COST);
7435 format %{ "ldarw $dst, $mem\t# int" %}
7436
7437 ins_encode(aarch64_enc_ldarw(dst, mem));
7438
7439 ins_pipe(pipe_serial);
7440 %}
7441
7442 // Load Integer (32 bit unsigned) into long
7443 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7444 %{
7445 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7446
7447 ins_cost(VOLATILE_REF_COST);
7448 format %{ "ldarw $dst, $mem\t# int" %}
7449
7450 ins_encode(aarch64_enc_ldarw(dst, mem));
7451
7452 ins_pipe(pipe_serial);
7453 %}
7454
7455 // Load Long (64 bit signed)
7456 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7457 %{
7458 match(Set dst (LoadL mem));
7459
7460 ins_cost(VOLATILE_REF_COST);
7461 format %{ "ldar $dst, $mem\t# int" %}
7462
7463 ins_encode(aarch64_enc_ldar(dst, mem));
7464
7465 ins_pipe(pipe_serial);
7466 %}
7467
7468 // Load Pointer
7469 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7470 %{
7471 match(Set dst (LoadP mem));
7472 predicate(n->as_Load()->barrier_data() == 0);
7473
7474 ins_cost(VOLATILE_REF_COST);
7475 format %{ "ldar $dst, $mem\t# ptr" %}
7476
7477 ins_encode(aarch64_enc_ldar(dst, mem));
7478
7479 ins_pipe(pipe_serial);
7480 %}
7481
7482 // Load Compressed Pointer
7483 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7484 %{
7485 match(Set dst (LoadN mem));
7486 predicate(n->as_Load()->barrier_data() == 0);
7487
7488 ins_cost(VOLATILE_REF_COST);
7489 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7490
7491 ins_encode(aarch64_enc_ldarw(dst, mem));
7492
7493 ins_pipe(pipe_serial);
7494 %}
7495
7496 // Load Float
7497 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7498 %{
7499 match(Set dst (LoadF mem));
7500
7501 ins_cost(VOLATILE_REF_COST);
7502 format %{ "ldars $dst, $mem\t# float" %}
7503
7504 ins_encode( aarch64_enc_fldars(dst, mem) );
7505
7506 ins_pipe(pipe_serial);
7507 %}
7508
7509 // Load Double
7510 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7511 %{
7512 match(Set dst (LoadD mem));
7513
7514 ins_cost(VOLATILE_REF_COST);
7515 format %{ "ldard $dst, $mem\t# double" %}
7516
7517 ins_encode( aarch64_enc_fldard(dst, mem) );
7518
7519 ins_pipe(pipe_serial);
7520 %}
7521
7522 // Store Byte
7523 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7524 %{
7525 match(Set mem (StoreB mem src));
7526
7527 ins_cost(VOLATILE_REF_COST);
7528 format %{ "stlrb $src, $mem\t# byte" %}
7529
7530 ins_encode(aarch64_enc_stlrb(src, mem));
7531
7532 ins_pipe(pipe_class_memory);
7533 %}
7534
7535 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7536 %{
7537 match(Set mem (StoreB mem zero));
7538
7539 ins_cost(VOLATILE_REF_COST);
7540 format %{ "stlrb zr, $mem\t# byte" %}
7541
7542 ins_encode(aarch64_enc_stlrb0(mem));
7543
7544 ins_pipe(pipe_class_memory);
7545 %}
7546
7547 // Store Char/Short
7548 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7549 %{
7550 match(Set mem (StoreC mem src));
7551
7552 ins_cost(VOLATILE_REF_COST);
7553 format %{ "stlrh $src, $mem\t# short" %}
7554
7555 ins_encode(aarch64_enc_stlrh(src, mem));
7556
7557 ins_pipe(pipe_class_memory);
7558 %}
7559
7560 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7561 %{
7562 match(Set mem (StoreC mem zero));
7563
7564 ins_cost(VOLATILE_REF_COST);
7565 format %{ "stlrh zr, $mem\t# short" %}
7566
7567 ins_encode(aarch64_enc_stlrh0(mem));
7568
7569 ins_pipe(pipe_class_memory);
7570 %}
7571
7572 // Store Integer
7573
7574 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7575 %{
7576 match(Set mem(StoreI mem src));
7577
7578 ins_cost(VOLATILE_REF_COST);
7579 format %{ "stlrw $src, $mem\t# int" %}
7580
7581 ins_encode(aarch64_enc_stlrw(src, mem));
7582
7583 ins_pipe(pipe_class_memory);
7584 %}
7585
7586 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7587 %{
7588 match(Set mem(StoreI mem zero));
7589
7590 ins_cost(VOLATILE_REF_COST);
7591 format %{ "stlrw zr, $mem\t# int" %}
7592
7593 ins_encode(aarch64_enc_stlrw0(mem));
7594
7595 ins_pipe(pipe_class_memory);
7596 %}
7597
7598 // Store Long (64 bit signed)
7599 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7600 %{
7601 match(Set mem (StoreL mem src));
7602
7603 ins_cost(VOLATILE_REF_COST);
7604 format %{ "stlr $src, $mem\t# int" %}
7605
7606 ins_encode(aarch64_enc_stlr(src, mem));
7607
7608 ins_pipe(pipe_class_memory);
7609 %}
7610
7611 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7612 %{
7613 match(Set mem (StoreL mem zero));
7614
7615 ins_cost(VOLATILE_REF_COST);
7616 format %{ "stlr zr, $mem\t# int" %}
7617
7618 ins_encode(aarch64_enc_stlr0(mem));
7619
7620 ins_pipe(pipe_class_memory);
7621 %}
7622
7623 // Store Pointer
7624 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7625 %{
7626 match(Set mem (StoreP mem src));
7627 predicate(n->as_Store()->barrier_data() == 0);
7628
7629 ins_cost(VOLATILE_REF_COST);
7630 format %{ "stlr $src, $mem\t# ptr" %}
7631
7632 ins_encode(aarch64_enc_stlr(src, mem));
7633
7634 ins_pipe(pipe_class_memory);
7635 %}
7636
7637 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7638 %{
7639 match(Set mem (StoreP mem zero));
7640 predicate(n->as_Store()->barrier_data() == 0);
7641
7642 ins_cost(VOLATILE_REF_COST);
7643 format %{ "stlr zr, $mem\t# ptr" %}
7644
7645 ins_encode(aarch64_enc_stlr0(mem));
7646
7647 ins_pipe(pipe_class_memory);
7648 %}
7649
7650 // Store Compressed Pointer
7651 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7652 %{
7653 match(Set mem (StoreN mem src));
7654 predicate(n->as_Store()->barrier_data() == 0);
7655
7656 ins_cost(VOLATILE_REF_COST);
7657 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7658
7659 ins_encode(aarch64_enc_stlrw(src, mem));
7660
7661 ins_pipe(pipe_class_memory);
7662 %}
7663
7664 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7665 %{
7666 match(Set mem (StoreN mem zero));
7667 predicate(n->as_Store()->barrier_data() == 0);
7668
7669 ins_cost(VOLATILE_REF_COST);
7670 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7671
7672 ins_encode(aarch64_enc_stlrw0(mem));
7673
7674 ins_pipe(pipe_class_memory);
7675 %}
7676
7677 // Store Float
7678 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7679 %{
7680 match(Set mem (StoreF mem src));
7681
7682 ins_cost(VOLATILE_REF_COST);
7683 format %{ "stlrs $src, $mem\t# float" %}
7684
7685 ins_encode( aarch64_enc_fstlrs(src, mem) );
7686
7687 ins_pipe(pipe_class_memory);
7688 %}
7689
7690 // TODO
7691 // implement storeImmF0 and storeFImmPacked
7692
7693 // Store Double
7694 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7695 %{
7696 match(Set mem (StoreD mem src));
7697
7698 ins_cost(VOLATILE_REF_COST);
7699 format %{ "stlrd $src, $mem\t# double" %}
7700
7701 ins_encode( aarch64_enc_fstlrd(src, mem) );
7702
7703 ins_pipe(pipe_class_memory);
7704 %}
7705
7706 // ---------------- end of volatile loads and stores ----------------
7707
7708 instruct cacheWB(indirect addr)
7709 %{
7710 predicate(VM_Version::supports_data_cache_line_flush());
7711 match(CacheWB addr);
7712
7713 ins_cost(100);
7714 format %{"cache wb $addr" %}
7715 ins_encode %{
7716 assert($addr->index_position() < 0, "should be");
7717 assert($addr$$disp == 0, "should be");
7718 __ cache_wb(Address($addr$$base$$Register, 0));
7719 %}
7720 ins_pipe(pipe_slow); // XXX
7721 %}
7722
7723 instruct cacheWBPreSync()
7724 %{
7725 predicate(VM_Version::supports_data_cache_line_flush());
7726 match(CacheWBPreSync);
7727
7728 ins_cost(100);
7729 format %{"cache wb presync" %}
7730 ins_encode %{
7731 __ cache_wbsync(true);
7732 %}
7733 ins_pipe(pipe_slow); // XXX
7734 %}
7735
7736 instruct cacheWBPostSync()
7737 %{
7738 predicate(VM_Version::supports_data_cache_line_flush());
7739 match(CacheWBPostSync);
7740
7741 ins_cost(100);
7742 format %{"cache wb postsync" %}
7743 ins_encode %{
7744 __ cache_wbsync(false);
7745 %}
7746 ins_pipe(pipe_slow); // XXX
7747 %}
7748
7749 // ============================================================================
7750 // BSWAP Instructions
7751
7752 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7753 match(Set dst (ReverseBytesI src));
7754
7755 ins_cost(INSN_COST);
7756 format %{ "revw $dst, $src" %}
7757
7758 ins_encode %{
7759 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7760 %}
7761
7762 ins_pipe(ialu_reg);
7763 %}
7764
7765 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7766 match(Set dst (ReverseBytesL src));
7767
7768 ins_cost(INSN_COST);
7769 format %{ "rev $dst, $src" %}
7770
7771 ins_encode %{
7772 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7773 %}
7774
7775 ins_pipe(ialu_reg);
7776 %}
7777
7778 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7779 match(Set dst (ReverseBytesUS src));
7780
7781 ins_cost(INSN_COST);
7782 format %{ "rev16w $dst, $src" %}
7783
7784 ins_encode %{
7785 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7786 %}
7787
7788 ins_pipe(ialu_reg);
7789 %}
7790
7791 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7792 match(Set dst (ReverseBytesS src));
7793
7794 ins_cost(INSN_COST);
7795 format %{ "rev16w $dst, $src\n\t"
7796 "sbfmw $dst, $dst, #0, #15" %}
7797
7798 ins_encode %{
7799 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7800 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7801 %}
7802
7803 ins_pipe(ialu_reg);
7804 %}
7805
7806 // ============================================================================
7807 // Zero Count Instructions
7808
7809 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7810 match(Set dst (CountLeadingZerosI src));
7811
7812 ins_cost(INSN_COST);
7813 format %{ "clzw $dst, $src" %}
7814 ins_encode %{
7815 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7816 %}
7817
7818 ins_pipe(ialu_reg);
7819 %}
7820
7821 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7822 match(Set dst (CountLeadingZerosL src));
7823
7824 ins_cost(INSN_COST);
7825 format %{ "clz $dst, $src" %}
7826 ins_encode %{
7827 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7828 %}
7829
7830 ins_pipe(ialu_reg);
7831 %}
7832
7833 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7834 match(Set dst (CountTrailingZerosI src));
7835
7836 ins_cost(INSN_COST * 2);
7837 format %{ "rbitw $dst, $src\n\t"
7838 "clzw $dst, $dst" %}
7839 ins_encode %{
7840 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7841 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7842 %}
7843
7844 ins_pipe(ialu_reg);
7845 %}
7846
7847 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7848 match(Set dst (CountTrailingZerosL src));
7849
7850 ins_cost(INSN_COST * 2);
7851 format %{ "rbit $dst, $src\n\t"
7852 "clz $dst, $dst" %}
7853 ins_encode %{
7854 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7855 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7856 %}
7857
7858 ins_pipe(ialu_reg);
7859 %}
7860
7861 //---------- Population Count Instructions -------------------------------------
7862 //
7863
7864 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7865 match(Set dst (PopCountI src));
7866 effect(TEMP tmp);
7867 ins_cost(INSN_COST * 13);
7868
7869 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7870 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7871 "addv $tmp, $tmp\t# vector (8B)\n\t"
7872 "mov $dst, $tmp\t# vector (1D)" %}
7873 ins_encode %{
7874 __ fmovs($tmp$$FloatRegister, $src$$Register);
7875 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7876 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7877 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7878 %}
7879
7880 ins_pipe(pipe_class_default);
7881 %}
7882
7883 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7884 match(Set dst (PopCountI (LoadI mem)));
7885 effect(TEMP tmp);
7886 ins_cost(INSN_COST * 13);
7887
7888 format %{ "ldrs $tmp, $mem\n\t"
7889 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7890 "addv $tmp, $tmp\t# vector (8B)\n\t"
7891 "mov $dst, $tmp\t# vector (1D)" %}
7892 ins_encode %{
7893 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7894 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7895 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7896 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7897 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7898 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7899 %}
7900
7901 ins_pipe(pipe_class_default);
7902 %}
7903
7904 // Note: Long.bitCount(long) returns an int.
7905 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7906 match(Set dst (PopCountL src));
7907 effect(TEMP tmp);
7908 ins_cost(INSN_COST * 13);
7909
7910 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7911 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7912 "addv $tmp, $tmp\t# vector (8B)\n\t"
7913 "mov $dst, $tmp\t# vector (1D)" %}
7914 ins_encode %{
7915 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7916 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7917 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7918 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7919 %}
7920
7921 ins_pipe(pipe_class_default);
7922 %}
7923
7924 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7925 match(Set dst (PopCountL (LoadL mem)));
7926 effect(TEMP tmp);
7927 ins_cost(INSN_COST * 13);
7928
7929 format %{ "ldrd $tmp, $mem\n\t"
7930 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7931 "addv $tmp, $tmp\t# vector (8B)\n\t"
7932 "mov $dst, $tmp\t# vector (1D)" %}
7933 ins_encode %{
7934 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7935 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7936 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7937 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7938 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7939 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7940 %}
7941
7942 ins_pipe(pipe_class_default);
7943 %}
7944
7945 // ============================================================================
7946 // VerifyVectorAlignment Instruction
7947
7948 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7949 match(Set addr (VerifyVectorAlignment addr mask));
7950 effect(KILL cr);
7951 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7952 ins_encode %{
7953 Label Lskip;
7954 // check if masked bits of addr are zero
7955 __ tst($addr$$Register, $mask$$constant);
7956 __ br(Assembler::EQ, Lskip);
7957 __ stop("verify_vector_alignment found a misaligned vector memory access");
7958 __ bind(Lskip);
7959 %}
7960 ins_pipe(pipe_slow);
7961 %}
7962
7963 // ============================================================================
7964 // MemBar Instruction
7965
7966 instruct load_fence() %{
7967 match(LoadFence);
7968 ins_cost(VOLATILE_REF_COST);
7969
7970 format %{ "load_fence" %}
7971
7972 ins_encode %{
7973 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7974 %}
7975 ins_pipe(pipe_serial);
7976 %}
7977
7978 instruct unnecessary_membar_acquire() %{
7979 predicate(unnecessary_acquire(n));
7980 match(MemBarAcquire);
7981 ins_cost(0);
7982
7983 format %{ "membar_acquire (elided)" %}
7984
7985 ins_encode %{
7986 __ block_comment("membar_acquire (elided)");
7987 %}
7988
7989 ins_pipe(pipe_class_empty);
7990 %}
7991
7992 instruct membar_acquire() %{
7993 match(MemBarAcquire);
7994 ins_cost(VOLATILE_REF_COST);
7995
7996 format %{ "membar_acquire\n\t"
7997 "dmb ishld" %}
7998
7999 ins_encode %{
8000 __ block_comment("membar_acquire");
8001 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8002 %}
8003
8004 ins_pipe(pipe_serial);
8005 %}
8006
8007
8008 instruct membar_acquire_lock() %{
8009 match(MemBarAcquireLock);
8010 ins_cost(VOLATILE_REF_COST);
8011
8012 format %{ "membar_acquire_lock (elided)" %}
8013
8014 ins_encode %{
8015 __ block_comment("membar_acquire_lock (elided)");
8016 %}
8017
8018 ins_pipe(pipe_serial);
8019 %}
8020
8021 instruct store_fence() %{
8022 match(StoreFence);
8023 ins_cost(VOLATILE_REF_COST);
8024
8025 format %{ "store_fence" %}
8026
8027 ins_encode %{
8028 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8029 %}
8030 ins_pipe(pipe_serial);
8031 %}
8032
8033 instruct unnecessary_membar_release() %{
8034 predicate(unnecessary_release(n));
8035 match(MemBarRelease);
8036 ins_cost(0);
8037
8038 format %{ "membar_release (elided)" %}
8039
8040 ins_encode %{
8041 __ block_comment("membar_release (elided)");
8042 %}
8043 ins_pipe(pipe_serial);
8044 %}
8045
8046 instruct membar_release() %{
8047 match(MemBarRelease);
8048 ins_cost(VOLATILE_REF_COST);
8049
8050 format %{ "membar_release\n\t"
8051 "dmb ishst\n\tdmb ishld" %}
8052
8053 ins_encode %{
8054 __ block_comment("membar_release");
8055 // These will be merged if AlwaysMergeDMB is enabled.
8056 __ membar(Assembler::StoreStore);
8057 __ membar(Assembler::LoadStore);
8058 %}
8059 ins_pipe(pipe_serial);
8060 %}
8061
8062 instruct membar_storestore() %{
8063 match(MemBarStoreStore);
8064 match(StoreStoreFence);
8065 ins_cost(VOLATILE_REF_COST);
8066
8067 format %{ "MEMBAR-store-store" %}
8068
8069 ins_encode %{
8070 __ membar(Assembler::StoreStore);
8071 %}
8072 ins_pipe(pipe_serial);
8073 %}
8074
8075 instruct membar_release_lock() %{
8076 match(MemBarReleaseLock);
8077 ins_cost(VOLATILE_REF_COST);
8078
8079 format %{ "membar_release_lock (elided)" %}
8080
8081 ins_encode %{
8082 __ block_comment("membar_release_lock (elided)");
8083 %}
8084
8085 ins_pipe(pipe_serial);
8086 %}
8087
8088 instruct unnecessary_membar_volatile() %{
8089 predicate(unnecessary_volatile(n));
8090 match(MemBarVolatile);
8091 ins_cost(0);
8092
8093 format %{ "membar_volatile (elided)" %}
8094
8095 ins_encode %{
8096 __ block_comment("membar_volatile (elided)");
8097 %}
8098
8099 ins_pipe(pipe_serial);
8100 %}
8101
8102 instruct membar_volatile() %{
8103 match(MemBarVolatile);
8104 ins_cost(VOLATILE_REF_COST*100);
8105
8106 format %{ "membar_volatile\n\t"
8107 "dmb ish"%}
8108
8109 ins_encode %{
8110 __ block_comment("membar_volatile");
8111 __ membar(Assembler::StoreLoad);
8112 %}
8113
8114 ins_pipe(pipe_serial);
8115 %}
8116
8117 // ============================================================================
8118 // Cast/Convert Instructions
8119
8120 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8121 match(Set dst (CastX2P src));
8122
8123 ins_cost(INSN_COST);
8124 format %{ "mov $dst, $src\t# long -> ptr" %}
8125
8126 ins_encode %{
8127 if ($dst$$reg != $src$$reg) {
8128 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8129 }
8130 %}
8131
8132 ins_pipe(ialu_reg);
8133 %}
8134
8135 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8136 match(Set dst (CastP2X src));
8137
8138 ins_cost(INSN_COST);
8139 format %{ "mov $dst, $src\t# ptr -> long" %}
8140
8141 ins_encode %{
8142 if ($dst$$reg != $src$$reg) {
8143 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8144 }
8145 %}
8146
8147 ins_pipe(ialu_reg);
8148 %}
8149
8150 // Convert oop into int for vectors alignment masking
8151 instruct convP2I(iRegINoSp dst, iRegP src) %{
8152 match(Set dst (ConvL2I (CastP2X src)));
8153
8154 ins_cost(INSN_COST);
8155 format %{ "movw $dst, $src\t# ptr -> int" %}
8156 ins_encode %{
8157 __ movw($dst$$Register, $src$$Register);
8158 %}
8159
8160 ins_pipe(ialu_reg);
8161 %}
8162
8163 // Convert compressed oop into int for vectors alignment masking
8164 // in case of 32bit oops (heap < 4Gb).
8165 instruct convN2I(iRegINoSp dst, iRegN src)
8166 %{
8167 predicate(CompressedOops::shift() == 0);
8168 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8169
8170 ins_cost(INSN_COST);
8171 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8172 ins_encode %{
8173 __ movw($dst$$Register, $src$$Register);
8174 %}
8175
8176 ins_pipe(ialu_reg);
8177 %}
8178
8179
8180 // Convert oop pointer into compressed form
8181 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8182 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8183 match(Set dst (EncodeP src));
8184 effect(KILL cr);
8185 ins_cost(INSN_COST * 3);
8186 format %{ "encode_heap_oop $dst, $src" %}
8187 ins_encode %{
8188 Register s = $src$$Register;
8189 Register d = $dst$$Register;
8190 __ encode_heap_oop(d, s);
8191 %}
8192 ins_pipe(ialu_reg);
8193 %}
8194
8195 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8196 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8197 match(Set dst (EncodeP src));
8198 ins_cost(INSN_COST * 3);
8199 format %{ "encode_heap_oop_not_null $dst, $src" %}
8200 ins_encode %{
8201 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8202 %}
8203 ins_pipe(ialu_reg);
8204 %}
8205
8206 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8207 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8208 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8209 match(Set dst (DecodeN src));
8210 ins_cost(INSN_COST * 3);
8211 format %{ "decode_heap_oop $dst, $src" %}
8212 ins_encode %{
8213 Register s = $src$$Register;
8214 Register d = $dst$$Register;
8215 __ decode_heap_oop(d, s);
8216 %}
8217 ins_pipe(ialu_reg);
8218 %}
8219
8220 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8221 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8222 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8223 match(Set dst (DecodeN src));
8224 ins_cost(INSN_COST * 3);
8225 format %{ "decode_heap_oop_not_null $dst, $src" %}
8226 ins_encode %{
8227 Register s = $src$$Register;
8228 Register d = $dst$$Register;
8229 __ decode_heap_oop_not_null(d, s);
8230 %}
8231 ins_pipe(ialu_reg);
8232 %}
8233
8234 // n.b. AArch64 implementations of encode_klass_not_null and
8235 // decode_klass_not_null do not modify the flags register so, unlike
8236 // Intel, we don't kill CR as a side effect here
8237
8238 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8239 match(Set dst (EncodePKlass src));
8240
8241 ins_cost(INSN_COST * 3);
8242 format %{ "encode_klass_not_null $dst,$src" %}
8243
8244 ins_encode %{
8245 Register src_reg = as_Register($src$$reg);
8246 Register dst_reg = as_Register($dst$$reg);
8247 __ encode_klass_not_null(dst_reg, src_reg);
8248 %}
8249
8250 ins_pipe(ialu_reg);
8251 %}
8252
8253 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8254 match(Set dst (DecodeNKlass src));
8255
8256 ins_cost(INSN_COST * 3);
8257 format %{ "decode_klass_not_null $dst,$src" %}
8258
8259 ins_encode %{
8260 Register src_reg = as_Register($src$$reg);
8261 Register dst_reg = as_Register($dst$$reg);
8262 if (dst_reg != src_reg) {
8263 __ decode_klass_not_null(dst_reg, src_reg);
8264 } else {
8265 __ decode_klass_not_null(dst_reg);
8266 }
8267 %}
8268
8269 ins_pipe(ialu_reg);
8270 %}
8271
8272 instruct checkCastPP(iRegPNoSp dst)
8273 %{
8274 match(Set dst (CheckCastPP dst));
8275
8276 size(0);
8277 format %{ "# checkcastPP of $dst" %}
8278 ins_encode(/* empty encoding */);
8279 ins_pipe(pipe_class_empty);
8280 %}
8281
8282 instruct castPP(iRegPNoSp dst)
8283 %{
8284 match(Set dst (CastPP dst));
8285
8286 size(0);
8287 format %{ "# castPP of $dst" %}
8288 ins_encode(/* empty encoding */);
8289 ins_pipe(pipe_class_empty);
8290 %}
8291
8292 instruct castII(iRegI dst)
8293 %{
8294 predicate(VerifyConstraintCasts == 0);
8295 match(Set dst (CastII dst));
8296
8297 size(0);
8298 format %{ "# castII of $dst" %}
8299 ins_encode(/* empty encoding */);
8300 ins_cost(0);
8301 ins_pipe(pipe_class_empty);
8302 %}
8303
8304 instruct castII_checked(iRegI dst, rFlagsReg cr)
8305 %{
8306 predicate(VerifyConstraintCasts > 0);
8307 match(Set dst (CastII dst));
8308 effect(KILL cr);
8309
8310 format %{ "# castII_checked of $dst" %}
8311 ins_encode %{
8312 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8313 %}
8314 ins_pipe(pipe_slow);
8315 %}
8316
8317 instruct castLL(iRegL dst)
8318 %{
8319 predicate(VerifyConstraintCasts == 0);
8320 match(Set dst (CastLL dst));
8321
8322 size(0);
8323 format %{ "# castLL of $dst" %}
8324 ins_encode(/* empty encoding */);
8325 ins_cost(0);
8326 ins_pipe(pipe_class_empty);
8327 %}
8328
8329 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8330 %{
8331 predicate(VerifyConstraintCasts > 0);
8332 match(Set dst (CastLL dst));
8333 effect(KILL cr);
8334
8335 format %{ "# castLL_checked of $dst" %}
8336 ins_encode %{
8337 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8338 %}
8339 ins_pipe(pipe_slow);
8340 %}
8341
8342 instruct castHH(vRegF dst)
8343 %{
8344 match(Set dst (CastHH dst));
8345 size(0);
8346 format %{ "# castHH of $dst" %}
8347 ins_encode(/* empty encoding */);
8348 ins_cost(0);
8349 ins_pipe(pipe_class_empty);
8350 %}
8351
8352 instruct castFF(vRegF dst)
8353 %{
8354 match(Set dst (CastFF dst));
8355
8356 size(0);
8357 format %{ "# castFF of $dst" %}
8358 ins_encode(/* empty encoding */);
8359 ins_cost(0);
8360 ins_pipe(pipe_class_empty);
8361 %}
8362
8363 instruct castDD(vRegD dst)
8364 %{
8365 match(Set dst (CastDD dst));
8366
8367 size(0);
8368 format %{ "# castDD of $dst" %}
8369 ins_encode(/* empty encoding */);
8370 ins_cost(0);
8371 ins_pipe(pipe_class_empty);
8372 %}
8373
8374 instruct castVV(vReg dst)
8375 %{
8376 match(Set dst (CastVV dst));
8377
8378 size(0);
8379 format %{ "# castVV of $dst" %}
8380 ins_encode(/* empty encoding */);
8381 ins_cost(0);
8382 ins_pipe(pipe_class_empty);
8383 %}
8384
8385 instruct castVVMask(pRegGov dst)
8386 %{
8387 match(Set dst (CastVV dst));
8388
8389 size(0);
8390 format %{ "# castVV of $dst" %}
8391 ins_encode(/* empty encoding */);
8392 ins_cost(0);
8393 ins_pipe(pipe_class_empty);
8394 %}
8395
8396 // ============================================================================
8397 // Atomic operation instructions
8398 //
8399
8400 // standard CompareAndSwapX when we are using barriers
8401 // these have higher priority than the rules selected by a predicate
8402
8403 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8404 // can't match them
8405
8406 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8407
8408 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8409 ins_cost(2 * VOLATILE_REF_COST);
8410
8411 effect(KILL cr);
8412
8413 format %{
8414 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8415 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8416 %}
8417
8418 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8419 aarch64_enc_cset_eq(res));
8420
8421 ins_pipe(pipe_slow);
8422 %}
8423
8424 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8425
8426 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8427 ins_cost(2 * VOLATILE_REF_COST);
8428
8429 effect(KILL cr);
8430
8431 format %{
8432 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8433 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8434 %}
8435
8436 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8437 aarch64_enc_cset_eq(res));
8438
8439 ins_pipe(pipe_slow);
8440 %}
8441
8442 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8443
8444 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8445 ins_cost(2 * VOLATILE_REF_COST);
8446
8447 effect(KILL cr);
8448
8449 format %{
8450 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8451 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8452 %}
8453
8454 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8455 aarch64_enc_cset_eq(res));
8456
8457 ins_pipe(pipe_slow);
8458 %}
8459
8460 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8461
8462 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8463 ins_cost(2 * VOLATILE_REF_COST);
8464
8465 effect(KILL cr);
8466
8467 format %{
8468 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8469 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8470 %}
8471
8472 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8473 aarch64_enc_cset_eq(res));
8474
8475 ins_pipe(pipe_slow);
8476 %}
8477
8478 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8479
8480 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8481 predicate(n->as_LoadStore()->barrier_data() == 0);
8482 ins_cost(2 * VOLATILE_REF_COST);
8483
8484 effect(KILL cr);
8485
8486 format %{
8487 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8488 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8489 %}
8490
8491 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8492 aarch64_enc_cset_eq(res));
8493
8494 ins_pipe(pipe_slow);
8495 %}
8496
8497 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8498
8499 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8500 predicate(n->as_LoadStore()->barrier_data() == 0);
8501 ins_cost(2 * VOLATILE_REF_COST);
8502
8503 effect(KILL cr);
8504
8505 format %{
8506 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8507 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8508 %}
8509
8510 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8511 aarch64_enc_cset_eq(res));
8512
8513 ins_pipe(pipe_slow);
8514 %}
8515
8516 // alternative CompareAndSwapX when we are eliding barriers
8517
8518 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8519
8520 predicate(needs_acquiring_load_exclusive(n));
8521 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8522 ins_cost(VOLATILE_REF_COST);
8523
8524 effect(KILL cr);
8525
8526 format %{
8527 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8528 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8529 %}
8530
8531 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8532 aarch64_enc_cset_eq(res));
8533
8534 ins_pipe(pipe_slow);
8535 %}
8536
8537 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8538
8539 predicate(needs_acquiring_load_exclusive(n));
8540 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8541 ins_cost(VOLATILE_REF_COST);
8542
8543 effect(KILL cr);
8544
8545 format %{
8546 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8547 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8548 %}
8549
8550 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8551 aarch64_enc_cset_eq(res));
8552
8553 ins_pipe(pipe_slow);
8554 %}
8555
8556 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8557
8558 predicate(needs_acquiring_load_exclusive(n));
8559 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8560 ins_cost(VOLATILE_REF_COST);
8561
8562 effect(KILL cr);
8563
8564 format %{
8565 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8566 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8567 %}
8568
8569 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8570 aarch64_enc_cset_eq(res));
8571
8572 ins_pipe(pipe_slow);
8573 %}
8574
8575 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8576
8577 predicate(needs_acquiring_load_exclusive(n));
8578 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8579 ins_cost(VOLATILE_REF_COST);
8580
8581 effect(KILL cr);
8582
8583 format %{
8584 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8585 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8586 %}
8587
8588 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8589 aarch64_enc_cset_eq(res));
8590
8591 ins_pipe(pipe_slow);
8592 %}
8593
8594 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8595
8596 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8597 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8598 ins_cost(VOLATILE_REF_COST);
8599
8600 effect(KILL cr);
8601
8602 format %{
8603 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8604 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8605 %}
8606
8607 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8608 aarch64_enc_cset_eq(res));
8609
8610 ins_pipe(pipe_slow);
8611 %}
8612
8613 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8614
8615 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8616 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8617 ins_cost(VOLATILE_REF_COST);
8618
8619 effect(KILL cr);
8620
8621 format %{
8622 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8623 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8624 %}
8625
8626 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8627 aarch64_enc_cset_eq(res));
8628
8629 ins_pipe(pipe_slow);
8630 %}
8631
8632
8633 // ---------------------------------------------------------------------
8634
8635 // BEGIN This section of the file is automatically generated. Do not edit --------------
8636
8637 // Sundry CAS operations. Note that release is always true,
8638 // regardless of the memory ordering of the CAS. This is because we
8639 // need the volatile case to be sequentially consistent but there is
8640 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8641 // can't check the type of memory ordering here, so we always emit a
8642 // STLXR.
8643
8644 // This section is generated from cas.m4
8645
8646
8647 // This pattern is generated automatically from cas.m4.
8648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8649 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8650 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8651 ins_cost(2 * VOLATILE_REF_COST);
8652 effect(TEMP_DEF res, KILL cr);
8653 format %{
8654 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8655 %}
8656 ins_encode %{
8657 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8658 Assembler::byte, /*acquire*/ false, /*release*/ true,
8659 /*weak*/ false, $res$$Register);
8660 __ sxtbw($res$$Register, $res$$Register);
8661 %}
8662 ins_pipe(pipe_slow);
8663 %}
8664
8665 // This pattern is generated automatically from cas.m4.
8666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8667 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8668 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8669 ins_cost(2 * VOLATILE_REF_COST);
8670 effect(TEMP_DEF res, KILL cr);
8671 format %{
8672 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8673 %}
8674 ins_encode %{
8675 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8676 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8677 /*weak*/ false, $res$$Register);
8678 __ sxthw($res$$Register, $res$$Register);
8679 %}
8680 ins_pipe(pipe_slow);
8681 %}
8682
8683 // This pattern is generated automatically from cas.m4.
8684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8685 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8686 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8687 ins_cost(2 * VOLATILE_REF_COST);
8688 effect(TEMP_DEF res, KILL cr);
8689 format %{
8690 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8691 %}
8692 ins_encode %{
8693 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8694 Assembler::word, /*acquire*/ false, /*release*/ true,
8695 /*weak*/ false, $res$$Register);
8696 %}
8697 ins_pipe(pipe_slow);
8698 %}
8699
8700 // This pattern is generated automatically from cas.m4.
8701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8702 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8703 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8704 ins_cost(2 * VOLATILE_REF_COST);
8705 effect(TEMP_DEF res, KILL cr);
8706 format %{
8707 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8708 %}
8709 ins_encode %{
8710 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8711 Assembler::xword, /*acquire*/ false, /*release*/ true,
8712 /*weak*/ false, $res$$Register);
8713 %}
8714 ins_pipe(pipe_slow);
8715 %}
8716
8717 // This pattern is generated automatically from cas.m4.
8718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8719 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8720 predicate(n->as_LoadStore()->barrier_data() == 0);
8721 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8722 ins_cost(2 * VOLATILE_REF_COST);
8723 effect(TEMP_DEF res, KILL cr);
8724 format %{
8725 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8726 %}
8727 ins_encode %{
8728 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8729 Assembler::word, /*acquire*/ false, /*release*/ true,
8730 /*weak*/ false, $res$$Register);
8731 %}
8732 ins_pipe(pipe_slow);
8733 %}
8734
8735 // This pattern is generated automatically from cas.m4.
8736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8737 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8738 predicate(n->as_LoadStore()->barrier_data() == 0);
8739 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8740 ins_cost(2 * VOLATILE_REF_COST);
8741 effect(TEMP_DEF res, KILL cr);
8742 format %{
8743 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8744 %}
8745 ins_encode %{
8746 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8747 Assembler::xword, /*acquire*/ false, /*release*/ true,
8748 /*weak*/ false, $res$$Register);
8749 %}
8750 ins_pipe(pipe_slow);
8751 %}
8752
8753 // This pattern is generated automatically from cas.m4.
8754 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8755 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8756 predicate(needs_acquiring_load_exclusive(n));
8757 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8758 ins_cost(VOLATILE_REF_COST);
8759 effect(TEMP_DEF res, KILL cr);
8760 format %{
8761 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8762 %}
8763 ins_encode %{
8764 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8765 Assembler::byte, /*acquire*/ true, /*release*/ true,
8766 /*weak*/ false, $res$$Register);
8767 __ sxtbw($res$$Register, $res$$Register);
8768 %}
8769 ins_pipe(pipe_slow);
8770 %}
8771
8772 // This pattern is generated automatically from cas.m4.
8773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8774 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8775 predicate(needs_acquiring_load_exclusive(n));
8776 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8777 ins_cost(VOLATILE_REF_COST);
8778 effect(TEMP_DEF res, KILL cr);
8779 format %{
8780 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8781 %}
8782 ins_encode %{
8783 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8784 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8785 /*weak*/ false, $res$$Register);
8786 __ sxthw($res$$Register, $res$$Register);
8787 %}
8788 ins_pipe(pipe_slow);
8789 %}
8790
8791 // This pattern is generated automatically from cas.m4.
8792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8793 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8794 predicate(needs_acquiring_load_exclusive(n));
8795 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8796 ins_cost(VOLATILE_REF_COST);
8797 effect(TEMP_DEF res, KILL cr);
8798 format %{
8799 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8800 %}
8801 ins_encode %{
8802 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8803 Assembler::word, /*acquire*/ true, /*release*/ true,
8804 /*weak*/ false, $res$$Register);
8805 %}
8806 ins_pipe(pipe_slow);
8807 %}
8808
8809 // This pattern is generated automatically from cas.m4.
8810 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8811 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8812 predicate(needs_acquiring_load_exclusive(n));
8813 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8814 ins_cost(VOLATILE_REF_COST);
8815 effect(TEMP_DEF res, KILL cr);
8816 format %{
8817 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8818 %}
8819 ins_encode %{
8820 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8821 Assembler::xword, /*acquire*/ true, /*release*/ true,
8822 /*weak*/ false, $res$$Register);
8823 %}
8824 ins_pipe(pipe_slow);
8825 %}
8826
8827 // This pattern is generated automatically from cas.m4.
8828 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8829 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8830 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8831 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8832 ins_cost(VOLATILE_REF_COST);
8833 effect(TEMP_DEF res, KILL cr);
8834 format %{
8835 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8836 %}
8837 ins_encode %{
8838 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8839 Assembler::word, /*acquire*/ true, /*release*/ true,
8840 /*weak*/ false, $res$$Register);
8841 %}
8842 ins_pipe(pipe_slow);
8843 %}
8844
8845 // This pattern is generated automatically from cas.m4.
8846 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8847 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8848 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8849 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8850 ins_cost(VOLATILE_REF_COST);
8851 effect(TEMP_DEF res, KILL cr);
8852 format %{
8853 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8854 %}
8855 ins_encode %{
8856 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8857 Assembler::xword, /*acquire*/ true, /*release*/ true,
8858 /*weak*/ false, $res$$Register);
8859 %}
8860 ins_pipe(pipe_slow);
8861 %}
8862
8863 // This pattern is generated automatically from cas.m4.
8864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8865 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8866 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8867 ins_cost(2 * VOLATILE_REF_COST);
8868 effect(KILL cr);
8869 format %{
8870 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8871 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8872 %}
8873 ins_encode %{
8874 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8875 Assembler::byte, /*acquire*/ false, /*release*/ true,
8876 /*weak*/ true, noreg);
8877 __ csetw($res$$Register, Assembler::EQ);
8878 %}
8879 ins_pipe(pipe_slow);
8880 %}
8881
8882 // This pattern is generated automatically from cas.m4.
8883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8884 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8885 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8886 ins_cost(2 * VOLATILE_REF_COST);
8887 effect(KILL cr);
8888 format %{
8889 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8890 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8891 %}
8892 ins_encode %{
8893 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8894 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8895 /*weak*/ true, noreg);
8896 __ csetw($res$$Register, Assembler::EQ);
8897 %}
8898 ins_pipe(pipe_slow);
8899 %}
8900
8901 // This pattern is generated automatically from cas.m4.
8902 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8903 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8904 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8905 ins_cost(2 * VOLATILE_REF_COST);
8906 effect(KILL cr);
8907 format %{
8908 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8909 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8910 %}
8911 ins_encode %{
8912 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8913 Assembler::word, /*acquire*/ false, /*release*/ true,
8914 /*weak*/ true, noreg);
8915 __ csetw($res$$Register, Assembler::EQ);
8916 %}
8917 ins_pipe(pipe_slow);
8918 %}
8919
8920 // This pattern is generated automatically from cas.m4.
8921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8922 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8923 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8924 ins_cost(2 * VOLATILE_REF_COST);
8925 effect(KILL cr);
8926 format %{
8927 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8928 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8929 %}
8930 ins_encode %{
8931 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8932 Assembler::xword, /*acquire*/ false, /*release*/ true,
8933 /*weak*/ true, noreg);
8934 __ csetw($res$$Register, Assembler::EQ);
8935 %}
8936 ins_pipe(pipe_slow);
8937 %}
8938
8939 // This pattern is generated automatically from cas.m4.
8940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8941 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8942 predicate(n->as_LoadStore()->barrier_data() == 0);
8943 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8944 ins_cost(2 * VOLATILE_REF_COST);
8945 effect(KILL cr);
8946 format %{
8947 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8948 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8949 %}
8950 ins_encode %{
8951 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8952 Assembler::word, /*acquire*/ false, /*release*/ true,
8953 /*weak*/ true, noreg);
8954 __ csetw($res$$Register, Assembler::EQ);
8955 %}
8956 ins_pipe(pipe_slow);
8957 %}
8958
8959 // This pattern is generated automatically from cas.m4.
8960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8961 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8962 predicate(n->as_LoadStore()->barrier_data() == 0);
8963 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8964 ins_cost(2 * VOLATILE_REF_COST);
8965 effect(KILL cr);
8966 format %{
8967 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8968 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8969 %}
8970 ins_encode %{
8971 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8972 Assembler::xword, /*acquire*/ false, /*release*/ true,
8973 /*weak*/ true, noreg);
8974 __ csetw($res$$Register, Assembler::EQ);
8975 %}
8976 ins_pipe(pipe_slow);
8977 %}
8978
8979 // This pattern is generated automatically from cas.m4.
8980 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8981 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8982 predicate(needs_acquiring_load_exclusive(n));
8983 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8984 ins_cost(VOLATILE_REF_COST);
8985 effect(KILL cr);
8986 format %{
8987 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8988 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8989 %}
8990 ins_encode %{
8991 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8992 Assembler::byte, /*acquire*/ true, /*release*/ true,
8993 /*weak*/ true, noreg);
8994 __ csetw($res$$Register, Assembler::EQ);
8995 %}
8996 ins_pipe(pipe_slow);
8997 %}
8998
8999 // This pattern is generated automatically from cas.m4.
9000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9001 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9002 predicate(needs_acquiring_load_exclusive(n));
9003 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9004 ins_cost(VOLATILE_REF_COST);
9005 effect(KILL cr);
9006 format %{
9007 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9008 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9009 %}
9010 ins_encode %{
9011 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9012 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9013 /*weak*/ true, noreg);
9014 __ csetw($res$$Register, Assembler::EQ);
9015 %}
9016 ins_pipe(pipe_slow);
9017 %}
9018
9019 // This pattern is generated automatically from cas.m4.
9020 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9021 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9022 predicate(needs_acquiring_load_exclusive(n));
9023 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9024 ins_cost(VOLATILE_REF_COST);
9025 effect(KILL cr);
9026 format %{
9027 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9028 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9029 %}
9030 ins_encode %{
9031 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9032 Assembler::word, /*acquire*/ true, /*release*/ true,
9033 /*weak*/ true, noreg);
9034 __ csetw($res$$Register, Assembler::EQ);
9035 %}
9036 ins_pipe(pipe_slow);
9037 %}
9038
9039 // This pattern is generated automatically from cas.m4.
9040 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9041 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9042 predicate(needs_acquiring_load_exclusive(n));
9043 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9044 ins_cost(VOLATILE_REF_COST);
9045 effect(KILL cr);
9046 format %{
9047 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9048 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9049 %}
9050 ins_encode %{
9051 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9052 Assembler::xword, /*acquire*/ true, /*release*/ true,
9053 /*weak*/ true, noreg);
9054 __ csetw($res$$Register, Assembler::EQ);
9055 %}
9056 ins_pipe(pipe_slow);
9057 %}
9058
9059 // This pattern is generated automatically from cas.m4.
9060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9061 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9062 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9063 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9064 ins_cost(VOLATILE_REF_COST);
9065 effect(KILL cr);
9066 format %{
9067 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9068 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9069 %}
9070 ins_encode %{
9071 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9072 Assembler::word, /*acquire*/ true, /*release*/ true,
9073 /*weak*/ true, noreg);
9074 __ csetw($res$$Register, Assembler::EQ);
9075 %}
9076 ins_pipe(pipe_slow);
9077 %}
9078
9079 // This pattern is generated automatically from cas.m4.
9080 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9081 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9082 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9083 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9084 ins_cost(VOLATILE_REF_COST);
9085 effect(KILL cr);
9086 format %{
9087 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9088 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9089 %}
9090 ins_encode %{
9091 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9092 Assembler::xword, /*acquire*/ true, /*release*/ true,
9093 /*weak*/ true, noreg);
9094 __ csetw($res$$Register, Assembler::EQ);
9095 %}
9096 ins_pipe(pipe_slow);
9097 %}
9098
9099 // END This section of the file is automatically generated. Do not edit --------------
9100 // ---------------------------------------------------------------------
9101
9102 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9103 match(Set prev (GetAndSetI mem newv));
9104 ins_cost(2 * VOLATILE_REF_COST);
9105 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9106 ins_encode %{
9107 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9108 %}
9109 ins_pipe(pipe_serial);
9110 %}
9111
9112 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9113 match(Set prev (GetAndSetL mem newv));
9114 ins_cost(2 * VOLATILE_REF_COST);
9115 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9116 ins_encode %{
9117 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9118 %}
9119 ins_pipe(pipe_serial);
9120 %}
9121
9122 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9123 predicate(n->as_LoadStore()->barrier_data() == 0);
9124 match(Set prev (GetAndSetN mem newv));
9125 ins_cost(2 * VOLATILE_REF_COST);
9126 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9127 ins_encode %{
9128 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9129 %}
9130 ins_pipe(pipe_serial);
9131 %}
9132
9133 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9134 predicate(n->as_LoadStore()->barrier_data() == 0);
9135 match(Set prev (GetAndSetP mem newv));
9136 ins_cost(2 * VOLATILE_REF_COST);
9137 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9138 ins_encode %{
9139 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9140 %}
9141 ins_pipe(pipe_serial);
9142 %}
9143
9144 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9145 predicate(needs_acquiring_load_exclusive(n));
9146 match(Set prev (GetAndSetI mem newv));
9147 ins_cost(VOLATILE_REF_COST);
9148 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9149 ins_encode %{
9150 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9151 %}
9152 ins_pipe(pipe_serial);
9153 %}
9154
9155 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9156 predicate(needs_acquiring_load_exclusive(n));
9157 match(Set prev (GetAndSetL mem newv));
9158 ins_cost(VOLATILE_REF_COST);
9159 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9160 ins_encode %{
9161 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9162 %}
9163 ins_pipe(pipe_serial);
9164 %}
9165
9166 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9167 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9168 match(Set prev (GetAndSetN mem newv));
9169 ins_cost(VOLATILE_REF_COST);
9170 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9171 ins_encode %{
9172 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9173 %}
9174 ins_pipe(pipe_serial);
9175 %}
9176
9177 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9178 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9179 match(Set prev (GetAndSetP mem newv));
9180 ins_cost(VOLATILE_REF_COST);
9181 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9182 ins_encode %{
9183 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9184 %}
9185 ins_pipe(pipe_serial);
9186 %}
9187
9188
9189 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9190 match(Set newval (GetAndAddL mem incr));
9191 ins_cost(2 * VOLATILE_REF_COST + 1);
9192 format %{ "get_and_addL $newval, [$mem], $incr" %}
9193 ins_encode %{
9194 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9195 %}
9196 ins_pipe(pipe_serial);
9197 %}
9198
9199 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9200 predicate(n->as_LoadStore()->result_not_used());
9201 match(Set dummy (GetAndAddL mem incr));
9202 ins_cost(2 * VOLATILE_REF_COST);
9203 format %{ "get_and_addL [$mem], $incr" %}
9204 ins_encode %{
9205 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9206 %}
9207 ins_pipe(pipe_serial);
9208 %}
9209
9210 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9211 match(Set newval (GetAndAddL mem incr));
9212 ins_cost(2 * VOLATILE_REF_COST + 1);
9213 format %{ "get_and_addL $newval, [$mem], $incr" %}
9214 ins_encode %{
9215 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9216 %}
9217 ins_pipe(pipe_serial);
9218 %}
9219
9220 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9221 predicate(n->as_LoadStore()->result_not_used());
9222 match(Set dummy (GetAndAddL mem incr));
9223 ins_cost(2 * VOLATILE_REF_COST);
9224 format %{ "get_and_addL [$mem], $incr" %}
9225 ins_encode %{
9226 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9227 %}
9228 ins_pipe(pipe_serial);
9229 %}
9230
9231 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9232 match(Set newval (GetAndAddI mem incr));
9233 ins_cost(2 * VOLATILE_REF_COST + 1);
9234 format %{ "get_and_addI $newval, [$mem], $incr" %}
9235 ins_encode %{
9236 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9237 %}
9238 ins_pipe(pipe_serial);
9239 %}
9240
9241 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9242 predicate(n->as_LoadStore()->result_not_used());
9243 match(Set dummy (GetAndAddI mem incr));
9244 ins_cost(2 * VOLATILE_REF_COST);
9245 format %{ "get_and_addI [$mem], $incr" %}
9246 ins_encode %{
9247 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9248 %}
9249 ins_pipe(pipe_serial);
9250 %}
9251
9252 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9253 match(Set newval (GetAndAddI mem incr));
9254 ins_cost(2 * VOLATILE_REF_COST + 1);
9255 format %{ "get_and_addI $newval, [$mem], $incr" %}
9256 ins_encode %{
9257 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9258 %}
9259 ins_pipe(pipe_serial);
9260 %}
9261
9262 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9263 predicate(n->as_LoadStore()->result_not_used());
9264 match(Set dummy (GetAndAddI mem incr));
9265 ins_cost(2 * VOLATILE_REF_COST);
9266 format %{ "get_and_addI [$mem], $incr" %}
9267 ins_encode %{
9268 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9269 %}
9270 ins_pipe(pipe_serial);
9271 %}
9272
9273 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9274 predicate(needs_acquiring_load_exclusive(n));
9275 match(Set newval (GetAndAddL mem incr));
9276 ins_cost(VOLATILE_REF_COST + 1);
9277 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9278 ins_encode %{
9279 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9280 %}
9281 ins_pipe(pipe_serial);
9282 %}
9283
9284 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9285 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9286 match(Set dummy (GetAndAddL mem incr));
9287 ins_cost(VOLATILE_REF_COST);
9288 format %{ "get_and_addL_acq [$mem], $incr" %}
9289 ins_encode %{
9290 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9291 %}
9292 ins_pipe(pipe_serial);
9293 %}
9294
9295 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9296 predicate(needs_acquiring_load_exclusive(n));
9297 match(Set newval (GetAndAddL mem incr));
9298 ins_cost(VOLATILE_REF_COST + 1);
9299 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9300 ins_encode %{
9301 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9302 %}
9303 ins_pipe(pipe_serial);
9304 %}
9305
9306 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9307 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9308 match(Set dummy (GetAndAddL mem incr));
9309 ins_cost(VOLATILE_REF_COST);
9310 format %{ "get_and_addL_acq [$mem], $incr" %}
9311 ins_encode %{
9312 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9313 %}
9314 ins_pipe(pipe_serial);
9315 %}
9316
9317 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9318 predicate(needs_acquiring_load_exclusive(n));
9319 match(Set newval (GetAndAddI mem incr));
9320 ins_cost(VOLATILE_REF_COST + 1);
9321 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9322 ins_encode %{
9323 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9324 %}
9325 ins_pipe(pipe_serial);
9326 %}
9327
9328 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9329 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9330 match(Set dummy (GetAndAddI mem incr));
9331 ins_cost(VOLATILE_REF_COST);
9332 format %{ "get_and_addI_acq [$mem], $incr" %}
9333 ins_encode %{
9334 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9335 %}
9336 ins_pipe(pipe_serial);
9337 %}
9338
9339 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9340 predicate(needs_acquiring_load_exclusive(n));
9341 match(Set newval (GetAndAddI mem incr));
9342 ins_cost(VOLATILE_REF_COST + 1);
9343 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9344 ins_encode %{
9345 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9346 %}
9347 ins_pipe(pipe_serial);
9348 %}
9349
9350 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9351 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9352 match(Set dummy (GetAndAddI mem incr));
9353 ins_cost(VOLATILE_REF_COST);
9354 format %{ "get_and_addI_acq [$mem], $incr" %}
9355 ins_encode %{
9356 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9357 %}
9358 ins_pipe(pipe_serial);
9359 %}
9360
9361 // Manifest a CmpU result in an integer register.
9362 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9363 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9364 %{
9365 match(Set dst (CmpU3 src1 src2));
9366 effect(KILL flags);
9367
9368 ins_cost(INSN_COST * 3);
9369 format %{
9370 "cmpw $src1, $src2\n\t"
9371 "csetw $dst, ne\n\t"
9372 "cnegw $dst, lo\t# CmpU3(reg)"
9373 %}
9374 ins_encode %{
9375 __ cmpw($src1$$Register, $src2$$Register);
9376 __ csetw($dst$$Register, Assembler::NE);
9377 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9378 %}
9379
9380 ins_pipe(pipe_class_default);
9381 %}
9382
9383 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9384 %{
9385 match(Set dst (CmpU3 src1 src2));
9386 effect(KILL flags);
9387
9388 ins_cost(INSN_COST * 3);
9389 format %{
9390 "subsw zr, $src1, $src2\n\t"
9391 "csetw $dst, ne\n\t"
9392 "cnegw $dst, lo\t# CmpU3(imm)"
9393 %}
9394 ins_encode %{
9395 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9396 __ csetw($dst$$Register, Assembler::NE);
9397 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9398 %}
9399
9400 ins_pipe(pipe_class_default);
9401 %}
9402
9403 // Manifest a CmpUL result in an integer register.
9404 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9405 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9406 %{
9407 match(Set dst (CmpUL3 src1 src2));
9408 effect(KILL flags);
9409
9410 ins_cost(INSN_COST * 3);
9411 format %{
9412 "cmp $src1, $src2\n\t"
9413 "csetw $dst, ne\n\t"
9414 "cnegw $dst, lo\t# CmpUL3(reg)"
9415 %}
9416 ins_encode %{
9417 __ cmp($src1$$Register, $src2$$Register);
9418 __ csetw($dst$$Register, Assembler::NE);
9419 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9420 %}
9421
9422 ins_pipe(pipe_class_default);
9423 %}
9424
9425 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9426 %{
9427 match(Set dst (CmpUL3 src1 src2));
9428 effect(KILL flags);
9429
9430 ins_cost(INSN_COST * 3);
9431 format %{
9432 "subs zr, $src1, $src2\n\t"
9433 "csetw $dst, ne\n\t"
9434 "cnegw $dst, lo\t# CmpUL3(imm)"
9435 %}
9436 ins_encode %{
9437 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9438 __ csetw($dst$$Register, Assembler::NE);
9439 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9440 %}
9441
9442 ins_pipe(pipe_class_default);
9443 %}
9444
9445 // Manifest a CmpL result in an integer register.
9446 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9447 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9448 %{
9449 match(Set dst (CmpL3 src1 src2));
9450 effect(KILL flags);
9451
9452 ins_cost(INSN_COST * 3);
9453 format %{
9454 "cmp $src1, $src2\n\t"
9455 "csetw $dst, ne\n\t"
9456 "cnegw $dst, lt\t# CmpL3(reg)"
9457 %}
9458 ins_encode %{
9459 __ cmp($src1$$Register, $src2$$Register);
9460 __ csetw($dst$$Register, Assembler::NE);
9461 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9462 %}
9463
9464 ins_pipe(pipe_class_default);
9465 %}
9466
9467 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9468 %{
9469 match(Set dst (CmpL3 src1 src2));
9470 effect(KILL flags);
9471
9472 ins_cost(INSN_COST * 3);
9473 format %{
9474 "subs zr, $src1, $src2\n\t"
9475 "csetw $dst, ne\n\t"
9476 "cnegw $dst, lt\t# CmpL3(imm)"
9477 %}
9478 ins_encode %{
9479 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9480 __ csetw($dst$$Register, Assembler::NE);
9481 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9482 %}
9483
9484 ins_pipe(pipe_class_default);
9485 %}
9486
9487 // ============================================================================
9488 // Conditional Move Instructions
9489
9490 // n.b. we have identical rules for both a signed compare op (cmpOp)
9491 // and an unsigned compare op (cmpOpU). it would be nice if we could
9492 // define an op class which merged both inputs and use it to type the
9493 // argument to a single rule. unfortunatelyt his fails because the
9494 // opclass does not live up to the COND_INTER interface of its
9495 // component operands. When the generic code tries to negate the
9496 // operand it ends up running the generci Machoper::negate method
9497 // which throws a ShouldNotHappen. So, we have to provide two flavours
9498 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9499
9500 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9501 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9502
9503 ins_cost(INSN_COST * 2);
9504 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9505
9506 ins_encode %{
9507 __ cselw(as_Register($dst$$reg),
9508 as_Register($src2$$reg),
9509 as_Register($src1$$reg),
9510 (Assembler::Condition)$cmp$$cmpcode);
9511 %}
9512
9513 ins_pipe(icond_reg_reg);
9514 %}
9515
9516 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9517 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9518
9519 ins_cost(INSN_COST * 2);
9520 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9521
9522 ins_encode %{
9523 __ cselw(as_Register($dst$$reg),
9524 as_Register($src2$$reg),
9525 as_Register($src1$$reg),
9526 (Assembler::Condition)$cmp$$cmpcode);
9527 %}
9528
9529 ins_pipe(icond_reg_reg);
9530 %}
9531
9532 // special cases where one arg is zero
9533
9534 // n.b. this is selected in preference to the rule above because it
9535 // avoids loading constant 0 into a source register
9536
9537 // TODO
9538 // we ought only to be able to cull one of these variants as the ideal
9539 // transforms ought always to order the zero consistently (to left/right?)
9540
9541 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9542 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9543
9544 ins_cost(INSN_COST * 2);
9545 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9546
9547 ins_encode %{
9548 __ cselw(as_Register($dst$$reg),
9549 as_Register($src$$reg),
9550 zr,
9551 (Assembler::Condition)$cmp$$cmpcode);
9552 %}
9553
9554 ins_pipe(icond_reg);
9555 %}
9556
9557 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9558 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9559
9560 ins_cost(INSN_COST * 2);
9561 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9562
9563 ins_encode %{
9564 __ cselw(as_Register($dst$$reg),
9565 as_Register($src$$reg),
9566 zr,
9567 (Assembler::Condition)$cmp$$cmpcode);
9568 %}
9569
9570 ins_pipe(icond_reg);
9571 %}
9572
9573 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9574 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9575
9576 ins_cost(INSN_COST * 2);
9577 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9578
9579 ins_encode %{
9580 __ cselw(as_Register($dst$$reg),
9581 zr,
9582 as_Register($src$$reg),
9583 (Assembler::Condition)$cmp$$cmpcode);
9584 %}
9585
9586 ins_pipe(icond_reg);
9587 %}
9588
9589 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9590 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9591
9592 ins_cost(INSN_COST * 2);
9593 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9594
9595 ins_encode %{
9596 __ cselw(as_Register($dst$$reg),
9597 zr,
9598 as_Register($src$$reg),
9599 (Assembler::Condition)$cmp$$cmpcode);
9600 %}
9601
9602 ins_pipe(icond_reg);
9603 %}
9604
9605 // special case for creating a boolean 0 or 1
9606
9607 // n.b. this is selected in preference to the rule above because it
9608 // avoids loading constants 0 and 1 into a source register
9609
9610 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9611 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9612
9613 ins_cost(INSN_COST * 2);
9614 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9615
9616 ins_encode %{
9617 // equivalently
9618 // cset(as_Register($dst$$reg),
9619 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9620 __ csincw(as_Register($dst$$reg),
9621 zr,
9622 zr,
9623 (Assembler::Condition)$cmp$$cmpcode);
9624 %}
9625
9626 ins_pipe(icond_none);
9627 %}
9628
9629 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9630 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9631
9632 ins_cost(INSN_COST * 2);
9633 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9634
9635 ins_encode %{
9636 // equivalently
9637 // cset(as_Register($dst$$reg),
9638 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9639 __ csincw(as_Register($dst$$reg),
9640 zr,
9641 zr,
9642 (Assembler::Condition)$cmp$$cmpcode);
9643 %}
9644
9645 ins_pipe(icond_none);
9646 %}
9647
9648 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9649 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9650
9651 ins_cost(INSN_COST * 2);
9652 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9653
9654 ins_encode %{
9655 __ csel(as_Register($dst$$reg),
9656 as_Register($src2$$reg),
9657 as_Register($src1$$reg),
9658 (Assembler::Condition)$cmp$$cmpcode);
9659 %}
9660
9661 ins_pipe(icond_reg_reg);
9662 %}
9663
9664 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9665 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9666
9667 ins_cost(INSN_COST * 2);
9668 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9669
9670 ins_encode %{
9671 __ csel(as_Register($dst$$reg),
9672 as_Register($src2$$reg),
9673 as_Register($src1$$reg),
9674 (Assembler::Condition)$cmp$$cmpcode);
9675 %}
9676
9677 ins_pipe(icond_reg_reg);
9678 %}
9679
9680 // special cases where one arg is zero
9681
9682 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9683 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9684
9685 ins_cost(INSN_COST * 2);
9686 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9687
9688 ins_encode %{
9689 __ csel(as_Register($dst$$reg),
9690 zr,
9691 as_Register($src$$reg),
9692 (Assembler::Condition)$cmp$$cmpcode);
9693 %}
9694
9695 ins_pipe(icond_reg);
9696 %}
9697
9698 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9699 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9700
9701 ins_cost(INSN_COST * 2);
9702 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9703
9704 ins_encode %{
9705 __ csel(as_Register($dst$$reg),
9706 zr,
9707 as_Register($src$$reg),
9708 (Assembler::Condition)$cmp$$cmpcode);
9709 %}
9710
9711 ins_pipe(icond_reg);
9712 %}
9713
9714 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9715 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9716
9717 ins_cost(INSN_COST * 2);
9718 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9719
9720 ins_encode %{
9721 __ csel(as_Register($dst$$reg),
9722 as_Register($src$$reg),
9723 zr,
9724 (Assembler::Condition)$cmp$$cmpcode);
9725 %}
9726
9727 ins_pipe(icond_reg);
9728 %}
9729
9730 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9731 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9732
9733 ins_cost(INSN_COST * 2);
9734 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9735
9736 ins_encode %{
9737 __ csel(as_Register($dst$$reg),
9738 as_Register($src$$reg),
9739 zr,
9740 (Assembler::Condition)$cmp$$cmpcode);
9741 %}
9742
9743 ins_pipe(icond_reg);
9744 %}
9745
9746 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9747 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9748
9749 ins_cost(INSN_COST * 2);
9750 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9751
9752 ins_encode %{
9753 __ csel(as_Register($dst$$reg),
9754 as_Register($src2$$reg),
9755 as_Register($src1$$reg),
9756 (Assembler::Condition)$cmp$$cmpcode);
9757 %}
9758
9759 ins_pipe(icond_reg_reg);
9760 %}
9761
9762 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9763 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9764
9765 ins_cost(INSN_COST * 2);
9766 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9767
9768 ins_encode %{
9769 __ csel(as_Register($dst$$reg),
9770 as_Register($src2$$reg),
9771 as_Register($src1$$reg),
9772 (Assembler::Condition)$cmp$$cmpcode);
9773 %}
9774
9775 ins_pipe(icond_reg_reg);
9776 %}
9777
9778 // special cases where one arg is zero
9779
9780 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9781 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9782
9783 ins_cost(INSN_COST * 2);
9784 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9785
9786 ins_encode %{
9787 __ csel(as_Register($dst$$reg),
9788 zr,
9789 as_Register($src$$reg),
9790 (Assembler::Condition)$cmp$$cmpcode);
9791 %}
9792
9793 ins_pipe(icond_reg);
9794 %}
9795
9796 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9797 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9798
9799 ins_cost(INSN_COST * 2);
9800 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9801
9802 ins_encode %{
9803 __ csel(as_Register($dst$$reg),
9804 zr,
9805 as_Register($src$$reg),
9806 (Assembler::Condition)$cmp$$cmpcode);
9807 %}
9808
9809 ins_pipe(icond_reg);
9810 %}
9811
9812 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9813 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9814
9815 ins_cost(INSN_COST * 2);
9816 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9817
9818 ins_encode %{
9819 __ csel(as_Register($dst$$reg),
9820 as_Register($src$$reg),
9821 zr,
9822 (Assembler::Condition)$cmp$$cmpcode);
9823 %}
9824
9825 ins_pipe(icond_reg);
9826 %}
9827
9828 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9829 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9830
9831 ins_cost(INSN_COST * 2);
9832 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9833
9834 ins_encode %{
9835 __ csel(as_Register($dst$$reg),
9836 as_Register($src$$reg),
9837 zr,
9838 (Assembler::Condition)$cmp$$cmpcode);
9839 %}
9840
9841 ins_pipe(icond_reg);
9842 %}
9843
9844 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9845 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9846
9847 ins_cost(INSN_COST * 2);
9848 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9849
9850 ins_encode %{
9851 __ cselw(as_Register($dst$$reg),
9852 as_Register($src2$$reg),
9853 as_Register($src1$$reg),
9854 (Assembler::Condition)$cmp$$cmpcode);
9855 %}
9856
9857 ins_pipe(icond_reg_reg);
9858 %}
9859
9860 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9861 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9862
9863 ins_cost(INSN_COST * 2);
9864 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9865
9866 ins_encode %{
9867 __ cselw(as_Register($dst$$reg),
9868 as_Register($src2$$reg),
9869 as_Register($src1$$reg),
9870 (Assembler::Condition)$cmp$$cmpcode);
9871 %}
9872
9873 ins_pipe(icond_reg_reg);
9874 %}
9875
9876 // special cases where one arg is zero
9877
9878 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9879 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9880
9881 ins_cost(INSN_COST * 2);
9882 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9883
9884 ins_encode %{
9885 __ cselw(as_Register($dst$$reg),
9886 zr,
9887 as_Register($src$$reg),
9888 (Assembler::Condition)$cmp$$cmpcode);
9889 %}
9890
9891 ins_pipe(icond_reg);
9892 %}
9893
9894 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9895 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9896
9897 ins_cost(INSN_COST * 2);
9898 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9899
9900 ins_encode %{
9901 __ cselw(as_Register($dst$$reg),
9902 zr,
9903 as_Register($src$$reg),
9904 (Assembler::Condition)$cmp$$cmpcode);
9905 %}
9906
9907 ins_pipe(icond_reg);
9908 %}
9909
9910 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9911 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9912
9913 ins_cost(INSN_COST * 2);
9914 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9915
9916 ins_encode %{
9917 __ cselw(as_Register($dst$$reg),
9918 as_Register($src$$reg),
9919 zr,
9920 (Assembler::Condition)$cmp$$cmpcode);
9921 %}
9922
9923 ins_pipe(icond_reg);
9924 %}
9925
9926 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9927 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9928
9929 ins_cost(INSN_COST * 2);
9930 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9931
9932 ins_encode %{
9933 __ cselw(as_Register($dst$$reg),
9934 as_Register($src$$reg),
9935 zr,
9936 (Assembler::Condition)$cmp$$cmpcode);
9937 %}
9938
9939 ins_pipe(icond_reg);
9940 %}
9941
9942 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9943 %{
9944 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9945
9946 ins_cost(INSN_COST * 3);
9947
9948 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9949 ins_encode %{
9950 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9951 __ fcsels(as_FloatRegister($dst$$reg),
9952 as_FloatRegister($src2$$reg),
9953 as_FloatRegister($src1$$reg),
9954 cond);
9955 %}
9956
9957 ins_pipe(fp_cond_reg_reg_s);
9958 %}
9959
9960 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9961 %{
9962 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9963
9964 ins_cost(INSN_COST * 3);
9965
9966 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9967 ins_encode %{
9968 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9969 __ fcsels(as_FloatRegister($dst$$reg),
9970 as_FloatRegister($src2$$reg),
9971 as_FloatRegister($src1$$reg),
9972 cond);
9973 %}
9974
9975 ins_pipe(fp_cond_reg_reg_s);
9976 %}
9977
9978 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9979 %{
9980 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9981
9982 ins_cost(INSN_COST * 3);
9983
9984 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9985 ins_encode %{
9986 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9987 __ fcseld(as_FloatRegister($dst$$reg),
9988 as_FloatRegister($src2$$reg),
9989 as_FloatRegister($src1$$reg),
9990 cond);
9991 %}
9992
9993 ins_pipe(fp_cond_reg_reg_d);
9994 %}
9995
9996 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9997 %{
9998 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9999
10000 ins_cost(INSN_COST * 3);
10001
10002 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10003 ins_encode %{
10004 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10005 __ fcseld(as_FloatRegister($dst$$reg),
10006 as_FloatRegister($src2$$reg),
10007 as_FloatRegister($src1$$reg),
10008 cond);
10009 %}
10010
10011 ins_pipe(fp_cond_reg_reg_d);
10012 %}
10013
10014 // ============================================================================
10015 // Arithmetic Instructions
10016 //
10017
10018 // Integer Addition
10019
10020 // TODO
10021 // these currently employ operations which do not set CR and hence are
10022 // not flagged as killing CR but we would like to isolate the cases
10023 // where we want to set flags from those where we don't. need to work
10024 // out how to do that.
10025
10026 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10027 match(Set dst (AddI src1 src2));
10028
10029 ins_cost(INSN_COST);
10030 format %{ "addw $dst, $src1, $src2" %}
10031
10032 ins_encode %{
10033 __ addw(as_Register($dst$$reg),
10034 as_Register($src1$$reg),
10035 as_Register($src2$$reg));
10036 %}
10037
10038 ins_pipe(ialu_reg_reg);
10039 %}
10040
10041 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10042 match(Set dst (AddI src1 src2));
10043
10044 ins_cost(INSN_COST);
10045 format %{ "addw $dst, $src1, $src2" %}
10046
10047 // use opcode to indicate that this is an add not a sub
10048 opcode(0x0);
10049
10050 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10051
10052 ins_pipe(ialu_reg_imm);
10053 %}
10054
10055 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10056 match(Set dst (AddI (ConvL2I src1) src2));
10057
10058 ins_cost(INSN_COST);
10059 format %{ "addw $dst, $src1, $src2" %}
10060
10061 // use opcode to indicate that this is an add not a sub
10062 opcode(0x0);
10063
10064 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10065
10066 ins_pipe(ialu_reg_imm);
10067 %}
10068
10069 // Pointer Addition
10070 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10071 match(Set dst (AddP src1 src2));
10072
10073 ins_cost(INSN_COST);
10074 format %{ "add $dst, $src1, $src2\t# ptr" %}
10075
10076 ins_encode %{
10077 __ add(as_Register($dst$$reg),
10078 as_Register($src1$$reg),
10079 as_Register($src2$$reg));
10080 %}
10081
10082 ins_pipe(ialu_reg_reg);
10083 %}
10084
10085 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10086 match(Set dst (AddP src1 (ConvI2L src2)));
10087
10088 ins_cost(1.9 * INSN_COST);
10089 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10090
10091 ins_encode %{
10092 __ add(as_Register($dst$$reg),
10093 as_Register($src1$$reg),
10094 as_Register($src2$$reg), ext::sxtw);
10095 %}
10096
10097 ins_pipe(ialu_reg_reg);
10098 %}
10099
10100 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10101 match(Set dst (AddP src1 (LShiftL src2 scale)));
10102
10103 ins_cost(1.9 * INSN_COST);
10104 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10105
10106 ins_encode %{
10107 __ lea(as_Register($dst$$reg),
10108 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10109 Address::lsl($scale$$constant)));
10110 %}
10111
10112 ins_pipe(ialu_reg_reg_shift);
10113 %}
10114
10115 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10116 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10117
10118 ins_cost(1.9 * INSN_COST);
10119 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10120
10121 ins_encode %{
10122 __ lea(as_Register($dst$$reg),
10123 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10124 Address::sxtw($scale$$constant)));
10125 %}
10126
10127 ins_pipe(ialu_reg_reg_shift);
10128 %}
10129
10130 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10131 match(Set dst (LShiftL (ConvI2L src) scale));
10132
10133 ins_cost(INSN_COST);
10134 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10135
10136 ins_encode %{
10137 __ sbfiz(as_Register($dst$$reg),
10138 as_Register($src$$reg),
10139 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10140 %}
10141
10142 ins_pipe(ialu_reg_shift);
10143 %}
10144
10145 // Pointer Immediate Addition
10146 // n.b. this needs to be more expensive than using an indirect memory
10147 // operand
10148 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10149 match(Set dst (AddP src1 src2));
10150
10151 ins_cost(INSN_COST);
10152 format %{ "add $dst, $src1, $src2\t# ptr" %}
10153
10154 // use opcode to indicate that this is an add not a sub
10155 opcode(0x0);
10156
10157 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10158
10159 ins_pipe(ialu_reg_imm);
10160 %}
10161
10162 // Long Addition
10163 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10164
10165 match(Set dst (AddL src1 src2));
10166
10167 ins_cost(INSN_COST);
10168 format %{ "add $dst, $src1, $src2" %}
10169
10170 ins_encode %{
10171 __ add(as_Register($dst$$reg),
10172 as_Register($src1$$reg),
10173 as_Register($src2$$reg));
10174 %}
10175
10176 ins_pipe(ialu_reg_reg);
10177 %}
10178
10179 // No constant pool entries requiredLong Immediate Addition.
10180 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10181 match(Set dst (AddL src1 src2));
10182
10183 ins_cost(INSN_COST);
10184 format %{ "add $dst, $src1, $src2" %}
10185
10186 // use opcode to indicate that this is an add not a sub
10187 opcode(0x0);
10188
10189 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10190
10191 ins_pipe(ialu_reg_imm);
10192 %}
10193
10194 // Integer Subtraction
10195 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10196 match(Set dst (SubI src1 src2));
10197
10198 ins_cost(INSN_COST);
10199 format %{ "subw $dst, $src1, $src2" %}
10200
10201 ins_encode %{
10202 __ subw(as_Register($dst$$reg),
10203 as_Register($src1$$reg),
10204 as_Register($src2$$reg));
10205 %}
10206
10207 ins_pipe(ialu_reg_reg);
10208 %}
10209
10210 // Immediate Subtraction
10211 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10212 match(Set dst (SubI src1 src2));
10213
10214 ins_cost(INSN_COST);
10215 format %{ "subw $dst, $src1, $src2" %}
10216
10217 // use opcode to indicate that this is a sub not an add
10218 opcode(0x1);
10219
10220 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10221
10222 ins_pipe(ialu_reg_imm);
10223 %}
10224
10225 // Long Subtraction
10226 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10227
10228 match(Set dst (SubL src1 src2));
10229
10230 ins_cost(INSN_COST);
10231 format %{ "sub $dst, $src1, $src2" %}
10232
10233 ins_encode %{
10234 __ sub(as_Register($dst$$reg),
10235 as_Register($src1$$reg),
10236 as_Register($src2$$reg));
10237 %}
10238
10239 ins_pipe(ialu_reg_reg);
10240 %}
10241
10242 // No constant pool entries requiredLong Immediate Subtraction.
10243 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10244 match(Set dst (SubL src1 src2));
10245
10246 ins_cost(INSN_COST);
10247 format %{ "sub$dst, $src1, $src2" %}
10248
10249 // use opcode to indicate that this is a sub not an add
10250 opcode(0x1);
10251
10252 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10253
10254 ins_pipe(ialu_reg_imm);
10255 %}
10256
10257 // Integer Negation (special case for sub)
10258
10259 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10260 match(Set dst (SubI zero src));
10261
10262 ins_cost(INSN_COST);
10263 format %{ "negw $dst, $src\t# int" %}
10264
10265 ins_encode %{
10266 __ negw(as_Register($dst$$reg),
10267 as_Register($src$$reg));
10268 %}
10269
10270 ins_pipe(ialu_reg);
10271 %}
10272
10273 // Long Negation
10274
10275 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10276 match(Set dst (SubL zero src));
10277
10278 ins_cost(INSN_COST);
10279 format %{ "neg $dst, $src\t# long" %}
10280
10281 ins_encode %{
10282 __ neg(as_Register($dst$$reg),
10283 as_Register($src$$reg));
10284 %}
10285
10286 ins_pipe(ialu_reg);
10287 %}
10288
10289 // Integer Multiply
10290
10291 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10292 match(Set dst (MulI src1 src2));
10293
10294 ins_cost(INSN_COST * 3);
10295 format %{ "mulw $dst, $src1, $src2" %}
10296
10297 ins_encode %{
10298 __ mulw(as_Register($dst$$reg),
10299 as_Register($src1$$reg),
10300 as_Register($src2$$reg));
10301 %}
10302
10303 ins_pipe(imul_reg_reg);
10304 %}
10305
10306 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10307 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10308
10309 ins_cost(INSN_COST * 3);
10310 format %{ "smull $dst, $src1, $src2" %}
10311
10312 ins_encode %{
10313 __ smull(as_Register($dst$$reg),
10314 as_Register($src1$$reg),
10315 as_Register($src2$$reg));
10316 %}
10317
10318 ins_pipe(imul_reg_reg);
10319 %}
10320
10321 // Long Multiply
10322
10323 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10324 match(Set dst (MulL src1 src2));
10325
10326 ins_cost(INSN_COST * 5);
10327 format %{ "mul $dst, $src1, $src2" %}
10328
10329 ins_encode %{
10330 __ mul(as_Register($dst$$reg),
10331 as_Register($src1$$reg),
10332 as_Register($src2$$reg));
10333 %}
10334
10335 ins_pipe(lmul_reg_reg);
10336 %}
10337
10338 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10339 %{
10340 match(Set dst (MulHiL src1 src2));
10341
10342 ins_cost(INSN_COST * 7);
10343 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10344
10345 ins_encode %{
10346 __ smulh(as_Register($dst$$reg),
10347 as_Register($src1$$reg),
10348 as_Register($src2$$reg));
10349 %}
10350
10351 ins_pipe(lmul_reg_reg);
10352 %}
10353
10354 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10355 %{
10356 match(Set dst (UMulHiL src1 src2));
10357
10358 ins_cost(INSN_COST * 7);
10359 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10360
10361 ins_encode %{
10362 __ umulh(as_Register($dst$$reg),
10363 as_Register($src1$$reg),
10364 as_Register($src2$$reg));
10365 %}
10366
10367 ins_pipe(lmul_reg_reg);
10368 %}
10369
10370 // Combined Integer Multiply & Add/Sub
10371
10372 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10373 match(Set dst (AddI src3 (MulI src1 src2)));
10374
10375 ins_cost(INSN_COST * 3);
10376 format %{ "madd $dst, $src1, $src2, $src3" %}
10377
10378 ins_encode %{
10379 __ maddw(as_Register($dst$$reg),
10380 as_Register($src1$$reg),
10381 as_Register($src2$$reg),
10382 as_Register($src3$$reg));
10383 %}
10384
10385 ins_pipe(imac_reg_reg);
10386 %}
10387
10388 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10389 match(Set dst (SubI src3 (MulI src1 src2)));
10390
10391 ins_cost(INSN_COST * 3);
10392 format %{ "msub $dst, $src1, $src2, $src3" %}
10393
10394 ins_encode %{
10395 __ msubw(as_Register($dst$$reg),
10396 as_Register($src1$$reg),
10397 as_Register($src2$$reg),
10398 as_Register($src3$$reg));
10399 %}
10400
10401 ins_pipe(imac_reg_reg);
10402 %}
10403
10404 // Combined Integer Multiply & Neg
10405
10406 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10407 match(Set dst (MulI (SubI zero src1) src2));
10408
10409 ins_cost(INSN_COST * 3);
10410 format %{ "mneg $dst, $src1, $src2" %}
10411
10412 ins_encode %{
10413 __ mnegw(as_Register($dst$$reg),
10414 as_Register($src1$$reg),
10415 as_Register($src2$$reg));
10416 %}
10417
10418 ins_pipe(imac_reg_reg);
10419 %}
10420
10421 // Combined Long Multiply & Add/Sub
10422
10423 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10424 match(Set dst (AddL src3 (MulL src1 src2)));
10425
10426 ins_cost(INSN_COST * 5);
10427 format %{ "madd $dst, $src1, $src2, $src3" %}
10428
10429 ins_encode %{
10430 __ madd(as_Register($dst$$reg),
10431 as_Register($src1$$reg),
10432 as_Register($src2$$reg),
10433 as_Register($src3$$reg));
10434 %}
10435
10436 ins_pipe(lmac_reg_reg);
10437 %}
10438
10439 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10440 match(Set dst (SubL src3 (MulL src1 src2)));
10441
10442 ins_cost(INSN_COST * 5);
10443 format %{ "msub $dst, $src1, $src2, $src3" %}
10444
10445 ins_encode %{
10446 __ msub(as_Register($dst$$reg),
10447 as_Register($src1$$reg),
10448 as_Register($src2$$reg),
10449 as_Register($src3$$reg));
10450 %}
10451
10452 ins_pipe(lmac_reg_reg);
10453 %}
10454
10455 // Combined Long Multiply & Neg
10456
10457 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10458 match(Set dst (MulL (SubL zero src1) src2));
10459
10460 ins_cost(INSN_COST * 5);
10461 format %{ "mneg $dst, $src1, $src2" %}
10462
10463 ins_encode %{
10464 __ mneg(as_Register($dst$$reg),
10465 as_Register($src1$$reg),
10466 as_Register($src2$$reg));
10467 %}
10468
10469 ins_pipe(lmac_reg_reg);
10470 %}
10471
10472 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10473
10474 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10475 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10476
10477 ins_cost(INSN_COST * 3);
10478 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10479
10480 ins_encode %{
10481 __ smaddl(as_Register($dst$$reg),
10482 as_Register($src1$$reg),
10483 as_Register($src2$$reg),
10484 as_Register($src3$$reg));
10485 %}
10486
10487 ins_pipe(imac_reg_reg);
10488 %}
10489
10490 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10491 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10492
10493 ins_cost(INSN_COST * 3);
10494 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10495
10496 ins_encode %{
10497 __ smsubl(as_Register($dst$$reg),
10498 as_Register($src1$$reg),
10499 as_Register($src2$$reg),
10500 as_Register($src3$$reg));
10501 %}
10502
10503 ins_pipe(imac_reg_reg);
10504 %}
10505
10506 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10507 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10508
10509 ins_cost(INSN_COST * 3);
10510 format %{ "smnegl $dst, $src1, $src2" %}
10511
10512 ins_encode %{
10513 __ smnegl(as_Register($dst$$reg),
10514 as_Register($src1$$reg),
10515 as_Register($src2$$reg));
10516 %}
10517
10518 ins_pipe(imac_reg_reg);
10519 %}
10520
10521 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10522
10523 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10524 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10525
10526 ins_cost(INSN_COST * 5);
10527 format %{ "mulw rscratch1, $src1, $src2\n\t"
10528 "maddw $dst, $src3, $src4, rscratch1" %}
10529
10530 ins_encode %{
10531 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10532 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10533
10534 ins_pipe(imac_reg_reg);
10535 %}
10536
10537 // Integer Divide
10538
10539 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10540 match(Set dst (DivI src1 src2));
10541
10542 ins_cost(INSN_COST * 19);
10543 format %{ "sdivw $dst, $src1, $src2" %}
10544
10545 ins_encode(aarch64_enc_divw(dst, src1, src2));
10546 ins_pipe(idiv_reg_reg);
10547 %}
10548
10549 // Long Divide
10550
10551 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10552 match(Set dst (DivL src1 src2));
10553
10554 ins_cost(INSN_COST * 35);
10555 format %{ "sdiv $dst, $src1, $src2" %}
10556
10557 ins_encode(aarch64_enc_div(dst, src1, src2));
10558 ins_pipe(ldiv_reg_reg);
10559 %}
10560
10561 // Integer Remainder
10562
10563 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10564 match(Set dst (ModI src1 src2));
10565
10566 ins_cost(INSN_COST * 22);
10567 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10568 "msubw $dst, rscratch1, $src2, $src1" %}
10569
10570 ins_encode(aarch64_enc_modw(dst, src1, src2));
10571 ins_pipe(idiv_reg_reg);
10572 %}
10573
10574 // Long Remainder
10575
10576 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10577 match(Set dst (ModL src1 src2));
10578
10579 ins_cost(INSN_COST * 38);
10580 format %{ "sdiv rscratch1, $src1, $src2\n"
10581 "msub $dst, rscratch1, $src2, $src1" %}
10582
10583 ins_encode(aarch64_enc_mod(dst, src1, src2));
10584 ins_pipe(ldiv_reg_reg);
10585 %}
10586
10587 // Unsigned Integer Divide
10588
10589 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10590 match(Set dst (UDivI src1 src2));
10591
10592 ins_cost(INSN_COST * 19);
10593 format %{ "udivw $dst, $src1, $src2" %}
10594
10595 ins_encode %{
10596 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10597 %}
10598
10599 ins_pipe(idiv_reg_reg);
10600 %}
10601
10602 // Unsigned Long Divide
10603
10604 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10605 match(Set dst (UDivL src1 src2));
10606
10607 ins_cost(INSN_COST * 35);
10608 format %{ "udiv $dst, $src1, $src2" %}
10609
10610 ins_encode %{
10611 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10612 %}
10613
10614 ins_pipe(ldiv_reg_reg);
10615 %}
10616
10617 // Unsigned Integer Remainder
10618
10619 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10620 match(Set dst (UModI src1 src2));
10621
10622 ins_cost(INSN_COST * 22);
10623 format %{ "udivw rscratch1, $src1, $src2\n\t"
10624 "msubw $dst, rscratch1, $src2, $src1" %}
10625
10626 ins_encode %{
10627 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10628 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10629 %}
10630
10631 ins_pipe(idiv_reg_reg);
10632 %}
10633
10634 // Unsigned Long Remainder
10635
10636 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10637 match(Set dst (UModL src1 src2));
10638
10639 ins_cost(INSN_COST * 38);
10640 format %{ "udiv rscratch1, $src1, $src2\n"
10641 "msub $dst, rscratch1, $src2, $src1" %}
10642
10643 ins_encode %{
10644 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10645 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10646 %}
10647
10648 ins_pipe(ldiv_reg_reg);
10649 %}
10650
10651 // Integer Shifts
10652
10653 // Shift Left Register
10654 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10655 match(Set dst (LShiftI src1 src2));
10656
10657 ins_cost(INSN_COST * 2);
10658 format %{ "lslvw $dst, $src1, $src2" %}
10659
10660 ins_encode %{
10661 __ lslvw(as_Register($dst$$reg),
10662 as_Register($src1$$reg),
10663 as_Register($src2$$reg));
10664 %}
10665
10666 ins_pipe(ialu_reg_reg_vshift);
10667 %}
10668
10669 // Shift Left Immediate
10670 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10671 match(Set dst (LShiftI src1 src2));
10672
10673 ins_cost(INSN_COST);
10674 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10675
10676 ins_encode %{
10677 __ lslw(as_Register($dst$$reg),
10678 as_Register($src1$$reg),
10679 $src2$$constant & 0x1f);
10680 %}
10681
10682 ins_pipe(ialu_reg_shift);
10683 %}
10684
10685 // Shift Right Logical Register
10686 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10687 match(Set dst (URShiftI src1 src2));
10688
10689 ins_cost(INSN_COST * 2);
10690 format %{ "lsrvw $dst, $src1, $src2" %}
10691
10692 ins_encode %{
10693 __ lsrvw(as_Register($dst$$reg),
10694 as_Register($src1$$reg),
10695 as_Register($src2$$reg));
10696 %}
10697
10698 ins_pipe(ialu_reg_reg_vshift);
10699 %}
10700
10701 // Shift Right Logical Immediate
10702 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10703 match(Set dst (URShiftI src1 src2));
10704
10705 ins_cost(INSN_COST);
10706 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10707
10708 ins_encode %{
10709 __ lsrw(as_Register($dst$$reg),
10710 as_Register($src1$$reg),
10711 $src2$$constant & 0x1f);
10712 %}
10713
10714 ins_pipe(ialu_reg_shift);
10715 %}
10716
10717 // Shift Right Arithmetic Register
10718 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10719 match(Set dst (RShiftI src1 src2));
10720
10721 ins_cost(INSN_COST * 2);
10722 format %{ "asrvw $dst, $src1, $src2" %}
10723
10724 ins_encode %{
10725 __ asrvw(as_Register($dst$$reg),
10726 as_Register($src1$$reg),
10727 as_Register($src2$$reg));
10728 %}
10729
10730 ins_pipe(ialu_reg_reg_vshift);
10731 %}
10732
10733 // Shift Right Arithmetic Immediate
10734 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10735 match(Set dst (RShiftI src1 src2));
10736
10737 ins_cost(INSN_COST);
10738 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10739
10740 ins_encode %{
10741 __ asrw(as_Register($dst$$reg),
10742 as_Register($src1$$reg),
10743 $src2$$constant & 0x1f);
10744 %}
10745
10746 ins_pipe(ialu_reg_shift);
10747 %}
10748
10749 // Combined Int Mask and Right Shift (using UBFM)
10750 // TODO
10751
10752 // Long Shifts
10753
10754 // Shift Left Register
10755 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10756 match(Set dst (LShiftL src1 src2));
10757
10758 ins_cost(INSN_COST * 2);
10759 format %{ "lslv $dst, $src1, $src2" %}
10760
10761 ins_encode %{
10762 __ lslv(as_Register($dst$$reg),
10763 as_Register($src1$$reg),
10764 as_Register($src2$$reg));
10765 %}
10766
10767 ins_pipe(ialu_reg_reg_vshift);
10768 %}
10769
10770 // Shift Left Immediate
10771 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10772 match(Set dst (LShiftL src1 src2));
10773
10774 ins_cost(INSN_COST);
10775 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10776
10777 ins_encode %{
10778 __ lsl(as_Register($dst$$reg),
10779 as_Register($src1$$reg),
10780 $src2$$constant & 0x3f);
10781 %}
10782
10783 ins_pipe(ialu_reg_shift);
10784 %}
10785
10786 // Shift Right Logical Register
10787 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10788 match(Set dst (URShiftL src1 src2));
10789
10790 ins_cost(INSN_COST * 2);
10791 format %{ "lsrv $dst, $src1, $src2" %}
10792
10793 ins_encode %{
10794 __ lsrv(as_Register($dst$$reg),
10795 as_Register($src1$$reg),
10796 as_Register($src2$$reg));
10797 %}
10798
10799 ins_pipe(ialu_reg_reg_vshift);
10800 %}
10801
10802 // Shift Right Logical Immediate
10803 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10804 match(Set dst (URShiftL src1 src2));
10805
10806 ins_cost(INSN_COST);
10807 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10808
10809 ins_encode %{
10810 __ lsr(as_Register($dst$$reg),
10811 as_Register($src1$$reg),
10812 $src2$$constant & 0x3f);
10813 %}
10814
10815 ins_pipe(ialu_reg_shift);
10816 %}
10817
10818 // A special-case pattern for card table stores.
10819 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10820 match(Set dst (URShiftL (CastP2X src1) src2));
10821
10822 ins_cost(INSN_COST);
10823 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10824
10825 ins_encode %{
10826 __ lsr(as_Register($dst$$reg),
10827 as_Register($src1$$reg),
10828 $src2$$constant & 0x3f);
10829 %}
10830
10831 ins_pipe(ialu_reg_shift);
10832 %}
10833
10834 // Shift Right Arithmetic Register
10835 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10836 match(Set dst (RShiftL src1 src2));
10837
10838 ins_cost(INSN_COST * 2);
10839 format %{ "asrv $dst, $src1, $src2" %}
10840
10841 ins_encode %{
10842 __ asrv(as_Register($dst$$reg),
10843 as_Register($src1$$reg),
10844 as_Register($src2$$reg));
10845 %}
10846
10847 ins_pipe(ialu_reg_reg_vshift);
10848 %}
10849
10850 // Shift Right Arithmetic Immediate
10851 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10852 match(Set dst (RShiftL src1 src2));
10853
10854 ins_cost(INSN_COST);
10855 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10856
10857 ins_encode %{
10858 __ asr(as_Register($dst$$reg),
10859 as_Register($src1$$reg),
10860 $src2$$constant & 0x3f);
10861 %}
10862
10863 ins_pipe(ialu_reg_shift);
10864 %}
10865
10866 // BEGIN This section of the file is automatically generated. Do not edit --------------
10867 // This section is generated from aarch64_ad.m4
10868
10869 // This pattern is automatically generated from aarch64_ad.m4
10870 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10871 instruct regL_not_reg(iRegLNoSp dst,
10872 iRegL src1, immL_M1 m1,
10873 rFlagsReg cr) %{
10874 match(Set dst (XorL src1 m1));
10875 ins_cost(INSN_COST);
10876 format %{ "eon $dst, $src1, zr" %}
10877
10878 ins_encode %{
10879 __ eon(as_Register($dst$$reg),
10880 as_Register($src1$$reg),
10881 zr,
10882 Assembler::LSL, 0);
10883 %}
10884
10885 ins_pipe(ialu_reg);
10886 %}
10887
10888 // This pattern is automatically generated from aarch64_ad.m4
10889 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10890 instruct regI_not_reg(iRegINoSp dst,
10891 iRegIorL2I src1, immI_M1 m1,
10892 rFlagsReg cr) %{
10893 match(Set dst (XorI src1 m1));
10894 ins_cost(INSN_COST);
10895 format %{ "eonw $dst, $src1, zr" %}
10896
10897 ins_encode %{
10898 __ eonw(as_Register($dst$$reg),
10899 as_Register($src1$$reg),
10900 zr,
10901 Assembler::LSL, 0);
10902 %}
10903
10904 ins_pipe(ialu_reg);
10905 %}
10906
10907 // This pattern is automatically generated from aarch64_ad.m4
10908 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10909 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10910 immI0 zero, iRegIorL2I src1, immI src2) %{
10911 match(Set dst (SubI zero (URShiftI src1 src2)));
10912
10913 ins_cost(1.9 * INSN_COST);
10914 format %{ "negw $dst, $src1, LSR $src2" %}
10915
10916 ins_encode %{
10917 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10918 Assembler::LSR, $src2$$constant & 0x1f);
10919 %}
10920
10921 ins_pipe(ialu_reg_shift);
10922 %}
10923
10924 // This pattern is automatically generated from aarch64_ad.m4
10925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10926 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10927 immI0 zero, iRegIorL2I src1, immI src2) %{
10928 match(Set dst (SubI zero (RShiftI src1 src2)));
10929
10930 ins_cost(1.9 * INSN_COST);
10931 format %{ "negw $dst, $src1, ASR $src2" %}
10932
10933 ins_encode %{
10934 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10935 Assembler::ASR, $src2$$constant & 0x1f);
10936 %}
10937
10938 ins_pipe(ialu_reg_shift);
10939 %}
10940
10941 // This pattern is automatically generated from aarch64_ad.m4
10942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10943 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10944 immI0 zero, iRegIorL2I src1, immI src2) %{
10945 match(Set dst (SubI zero (LShiftI src1 src2)));
10946
10947 ins_cost(1.9 * INSN_COST);
10948 format %{ "negw $dst, $src1, LSL $src2" %}
10949
10950 ins_encode %{
10951 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10952 Assembler::LSL, $src2$$constant & 0x1f);
10953 %}
10954
10955 ins_pipe(ialu_reg_shift);
10956 %}
10957
10958 // This pattern is automatically generated from aarch64_ad.m4
10959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10960 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10961 immL0 zero, iRegL src1, immI src2) %{
10962 match(Set dst (SubL zero (URShiftL src1 src2)));
10963
10964 ins_cost(1.9 * INSN_COST);
10965 format %{ "neg $dst, $src1, LSR $src2" %}
10966
10967 ins_encode %{
10968 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10969 Assembler::LSR, $src2$$constant & 0x3f);
10970 %}
10971
10972 ins_pipe(ialu_reg_shift);
10973 %}
10974
10975 // This pattern is automatically generated from aarch64_ad.m4
10976 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10977 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10978 immL0 zero, iRegL src1, immI src2) %{
10979 match(Set dst (SubL zero (RShiftL src1 src2)));
10980
10981 ins_cost(1.9 * INSN_COST);
10982 format %{ "neg $dst, $src1, ASR $src2" %}
10983
10984 ins_encode %{
10985 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10986 Assembler::ASR, $src2$$constant & 0x3f);
10987 %}
10988
10989 ins_pipe(ialu_reg_shift);
10990 %}
10991
10992 // This pattern is automatically generated from aarch64_ad.m4
10993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10994 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10995 immL0 zero, iRegL src1, immI src2) %{
10996 match(Set dst (SubL zero (LShiftL src1 src2)));
10997
10998 ins_cost(1.9 * INSN_COST);
10999 format %{ "neg $dst, $src1, LSL $src2" %}
11000
11001 ins_encode %{
11002 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11003 Assembler::LSL, $src2$$constant & 0x3f);
11004 %}
11005
11006 ins_pipe(ialu_reg_shift);
11007 %}
11008
11009 // This pattern is automatically generated from aarch64_ad.m4
11010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11011 instruct AndI_reg_not_reg(iRegINoSp dst,
11012 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11013 match(Set dst (AndI src1 (XorI src2 m1)));
11014 ins_cost(INSN_COST);
11015 format %{ "bicw $dst, $src1, $src2" %}
11016
11017 ins_encode %{
11018 __ bicw(as_Register($dst$$reg),
11019 as_Register($src1$$reg),
11020 as_Register($src2$$reg),
11021 Assembler::LSL, 0);
11022 %}
11023
11024 ins_pipe(ialu_reg_reg);
11025 %}
11026
11027 // This pattern is automatically generated from aarch64_ad.m4
11028 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11029 instruct AndL_reg_not_reg(iRegLNoSp dst,
11030 iRegL src1, iRegL src2, immL_M1 m1) %{
11031 match(Set dst (AndL src1 (XorL src2 m1)));
11032 ins_cost(INSN_COST);
11033 format %{ "bic $dst, $src1, $src2" %}
11034
11035 ins_encode %{
11036 __ bic(as_Register($dst$$reg),
11037 as_Register($src1$$reg),
11038 as_Register($src2$$reg),
11039 Assembler::LSL, 0);
11040 %}
11041
11042 ins_pipe(ialu_reg_reg);
11043 %}
11044
11045 // This pattern is automatically generated from aarch64_ad.m4
11046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11047 instruct OrI_reg_not_reg(iRegINoSp dst,
11048 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11049 match(Set dst (OrI src1 (XorI src2 m1)));
11050 ins_cost(INSN_COST);
11051 format %{ "ornw $dst, $src1, $src2" %}
11052
11053 ins_encode %{
11054 __ ornw(as_Register($dst$$reg),
11055 as_Register($src1$$reg),
11056 as_Register($src2$$reg),
11057 Assembler::LSL, 0);
11058 %}
11059
11060 ins_pipe(ialu_reg_reg);
11061 %}
11062
11063 // This pattern is automatically generated from aarch64_ad.m4
11064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11065 instruct OrL_reg_not_reg(iRegLNoSp dst,
11066 iRegL src1, iRegL src2, immL_M1 m1) %{
11067 match(Set dst (OrL src1 (XorL src2 m1)));
11068 ins_cost(INSN_COST);
11069 format %{ "orn $dst, $src1, $src2" %}
11070
11071 ins_encode %{
11072 __ orn(as_Register($dst$$reg),
11073 as_Register($src1$$reg),
11074 as_Register($src2$$reg),
11075 Assembler::LSL, 0);
11076 %}
11077
11078 ins_pipe(ialu_reg_reg);
11079 %}
11080
11081 // This pattern is automatically generated from aarch64_ad.m4
11082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11083 instruct XorI_reg_not_reg(iRegINoSp dst,
11084 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11085 match(Set dst (XorI m1 (XorI src2 src1)));
11086 ins_cost(INSN_COST);
11087 format %{ "eonw $dst, $src1, $src2" %}
11088
11089 ins_encode %{
11090 __ eonw(as_Register($dst$$reg),
11091 as_Register($src1$$reg),
11092 as_Register($src2$$reg),
11093 Assembler::LSL, 0);
11094 %}
11095
11096 ins_pipe(ialu_reg_reg);
11097 %}
11098
11099 // This pattern is automatically generated from aarch64_ad.m4
11100 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11101 instruct XorL_reg_not_reg(iRegLNoSp dst,
11102 iRegL src1, iRegL src2, immL_M1 m1) %{
11103 match(Set dst (XorL m1 (XorL src2 src1)));
11104 ins_cost(INSN_COST);
11105 format %{ "eon $dst, $src1, $src2" %}
11106
11107 ins_encode %{
11108 __ eon(as_Register($dst$$reg),
11109 as_Register($src1$$reg),
11110 as_Register($src2$$reg),
11111 Assembler::LSL, 0);
11112 %}
11113
11114 ins_pipe(ialu_reg_reg);
11115 %}
11116
11117 // This pattern is automatically generated from aarch64_ad.m4
11118 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11119 // val & (-1 ^ (val >>> shift)) ==> bicw
11120 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11121 iRegIorL2I src1, iRegIorL2I src2,
11122 immI src3, immI_M1 src4) %{
11123 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11124 ins_cost(1.9 * INSN_COST);
11125 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11126
11127 ins_encode %{
11128 __ bicw(as_Register($dst$$reg),
11129 as_Register($src1$$reg),
11130 as_Register($src2$$reg),
11131 Assembler::LSR,
11132 $src3$$constant & 0x1f);
11133 %}
11134
11135 ins_pipe(ialu_reg_reg_shift);
11136 %}
11137
11138 // This pattern is automatically generated from aarch64_ad.m4
11139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11140 // val & (-1 ^ (val >>> shift)) ==> bic
11141 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11142 iRegL src1, iRegL src2,
11143 immI src3, immL_M1 src4) %{
11144 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11145 ins_cost(1.9 * INSN_COST);
11146 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11147
11148 ins_encode %{
11149 __ bic(as_Register($dst$$reg),
11150 as_Register($src1$$reg),
11151 as_Register($src2$$reg),
11152 Assembler::LSR,
11153 $src3$$constant & 0x3f);
11154 %}
11155
11156 ins_pipe(ialu_reg_reg_shift);
11157 %}
11158
11159 // This pattern is automatically generated from aarch64_ad.m4
11160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11161 // val & (-1 ^ (val >> shift)) ==> bicw
11162 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11163 iRegIorL2I src1, iRegIorL2I src2,
11164 immI src3, immI_M1 src4) %{
11165 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11166 ins_cost(1.9 * INSN_COST);
11167 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11168
11169 ins_encode %{
11170 __ bicw(as_Register($dst$$reg),
11171 as_Register($src1$$reg),
11172 as_Register($src2$$reg),
11173 Assembler::ASR,
11174 $src3$$constant & 0x1f);
11175 %}
11176
11177 ins_pipe(ialu_reg_reg_shift);
11178 %}
11179
11180 // This pattern is automatically generated from aarch64_ad.m4
11181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11182 // val & (-1 ^ (val >> shift)) ==> bic
11183 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11184 iRegL src1, iRegL src2,
11185 immI src3, immL_M1 src4) %{
11186 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11187 ins_cost(1.9 * INSN_COST);
11188 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11189
11190 ins_encode %{
11191 __ bic(as_Register($dst$$reg),
11192 as_Register($src1$$reg),
11193 as_Register($src2$$reg),
11194 Assembler::ASR,
11195 $src3$$constant & 0x3f);
11196 %}
11197
11198 ins_pipe(ialu_reg_reg_shift);
11199 %}
11200
11201 // This pattern is automatically generated from aarch64_ad.m4
11202 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11203 // val & (-1 ^ (val ror shift)) ==> bicw
11204 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11205 iRegIorL2I src1, iRegIorL2I src2,
11206 immI src3, immI_M1 src4) %{
11207 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11208 ins_cost(1.9 * INSN_COST);
11209 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11210
11211 ins_encode %{
11212 __ bicw(as_Register($dst$$reg),
11213 as_Register($src1$$reg),
11214 as_Register($src2$$reg),
11215 Assembler::ROR,
11216 $src3$$constant & 0x1f);
11217 %}
11218
11219 ins_pipe(ialu_reg_reg_shift);
11220 %}
11221
11222 // This pattern is automatically generated from aarch64_ad.m4
11223 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11224 // val & (-1 ^ (val ror shift)) ==> bic
11225 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11226 iRegL src1, iRegL src2,
11227 immI src3, immL_M1 src4) %{
11228 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11229 ins_cost(1.9 * INSN_COST);
11230 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11231
11232 ins_encode %{
11233 __ bic(as_Register($dst$$reg),
11234 as_Register($src1$$reg),
11235 as_Register($src2$$reg),
11236 Assembler::ROR,
11237 $src3$$constant & 0x3f);
11238 %}
11239
11240 ins_pipe(ialu_reg_reg_shift);
11241 %}
11242
11243 // This pattern is automatically generated from aarch64_ad.m4
11244 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11245 // val & (-1 ^ (val << shift)) ==> bicw
11246 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11247 iRegIorL2I src1, iRegIorL2I src2,
11248 immI src3, immI_M1 src4) %{
11249 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11250 ins_cost(1.9 * INSN_COST);
11251 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11252
11253 ins_encode %{
11254 __ bicw(as_Register($dst$$reg),
11255 as_Register($src1$$reg),
11256 as_Register($src2$$reg),
11257 Assembler::LSL,
11258 $src3$$constant & 0x1f);
11259 %}
11260
11261 ins_pipe(ialu_reg_reg_shift);
11262 %}
11263
11264 // This pattern is automatically generated from aarch64_ad.m4
11265 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11266 // val & (-1 ^ (val << shift)) ==> bic
11267 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11268 iRegL src1, iRegL src2,
11269 immI src3, immL_M1 src4) %{
11270 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11271 ins_cost(1.9 * INSN_COST);
11272 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11273
11274 ins_encode %{
11275 __ bic(as_Register($dst$$reg),
11276 as_Register($src1$$reg),
11277 as_Register($src2$$reg),
11278 Assembler::LSL,
11279 $src3$$constant & 0x3f);
11280 %}
11281
11282 ins_pipe(ialu_reg_reg_shift);
11283 %}
11284
11285 // This pattern is automatically generated from aarch64_ad.m4
11286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11287 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11288 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11289 iRegIorL2I src1, iRegIorL2I src2,
11290 immI src3, immI_M1 src4) %{
11291 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11292 ins_cost(1.9 * INSN_COST);
11293 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11294
11295 ins_encode %{
11296 __ eonw(as_Register($dst$$reg),
11297 as_Register($src1$$reg),
11298 as_Register($src2$$reg),
11299 Assembler::LSR,
11300 $src3$$constant & 0x1f);
11301 %}
11302
11303 ins_pipe(ialu_reg_reg_shift);
11304 %}
11305
11306 // This pattern is automatically generated from aarch64_ad.m4
11307 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11308 // val ^ (-1 ^ (val >>> shift)) ==> eon
11309 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11310 iRegL src1, iRegL src2,
11311 immI src3, immL_M1 src4) %{
11312 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11313 ins_cost(1.9 * INSN_COST);
11314 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11315
11316 ins_encode %{
11317 __ eon(as_Register($dst$$reg),
11318 as_Register($src1$$reg),
11319 as_Register($src2$$reg),
11320 Assembler::LSR,
11321 $src3$$constant & 0x3f);
11322 %}
11323
11324 ins_pipe(ialu_reg_reg_shift);
11325 %}
11326
11327 // This pattern is automatically generated from aarch64_ad.m4
11328 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11329 // val ^ (-1 ^ (val >> shift)) ==> eonw
11330 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11331 iRegIorL2I src1, iRegIorL2I src2,
11332 immI src3, immI_M1 src4) %{
11333 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11334 ins_cost(1.9 * INSN_COST);
11335 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11336
11337 ins_encode %{
11338 __ eonw(as_Register($dst$$reg),
11339 as_Register($src1$$reg),
11340 as_Register($src2$$reg),
11341 Assembler::ASR,
11342 $src3$$constant & 0x1f);
11343 %}
11344
11345 ins_pipe(ialu_reg_reg_shift);
11346 %}
11347
11348 // This pattern is automatically generated from aarch64_ad.m4
11349 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11350 // val ^ (-1 ^ (val >> shift)) ==> eon
11351 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11352 iRegL src1, iRegL src2,
11353 immI src3, immL_M1 src4) %{
11354 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11355 ins_cost(1.9 * INSN_COST);
11356 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11357
11358 ins_encode %{
11359 __ eon(as_Register($dst$$reg),
11360 as_Register($src1$$reg),
11361 as_Register($src2$$reg),
11362 Assembler::ASR,
11363 $src3$$constant & 0x3f);
11364 %}
11365
11366 ins_pipe(ialu_reg_reg_shift);
11367 %}
11368
11369 // This pattern is automatically generated from aarch64_ad.m4
11370 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11371 // val ^ (-1 ^ (val ror shift)) ==> eonw
11372 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11373 iRegIorL2I src1, iRegIorL2I src2,
11374 immI src3, immI_M1 src4) %{
11375 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11376 ins_cost(1.9 * INSN_COST);
11377 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11378
11379 ins_encode %{
11380 __ eonw(as_Register($dst$$reg),
11381 as_Register($src1$$reg),
11382 as_Register($src2$$reg),
11383 Assembler::ROR,
11384 $src3$$constant & 0x1f);
11385 %}
11386
11387 ins_pipe(ialu_reg_reg_shift);
11388 %}
11389
11390 // This pattern is automatically generated from aarch64_ad.m4
11391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11392 // val ^ (-1 ^ (val ror shift)) ==> eon
11393 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11394 iRegL src1, iRegL src2,
11395 immI src3, immL_M1 src4) %{
11396 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11397 ins_cost(1.9 * INSN_COST);
11398 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11399
11400 ins_encode %{
11401 __ eon(as_Register($dst$$reg),
11402 as_Register($src1$$reg),
11403 as_Register($src2$$reg),
11404 Assembler::ROR,
11405 $src3$$constant & 0x3f);
11406 %}
11407
11408 ins_pipe(ialu_reg_reg_shift);
11409 %}
11410
11411 // This pattern is automatically generated from aarch64_ad.m4
11412 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11413 // val ^ (-1 ^ (val << shift)) ==> eonw
11414 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11415 iRegIorL2I src1, iRegIorL2I src2,
11416 immI src3, immI_M1 src4) %{
11417 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11418 ins_cost(1.9 * INSN_COST);
11419 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11420
11421 ins_encode %{
11422 __ eonw(as_Register($dst$$reg),
11423 as_Register($src1$$reg),
11424 as_Register($src2$$reg),
11425 Assembler::LSL,
11426 $src3$$constant & 0x1f);
11427 %}
11428
11429 ins_pipe(ialu_reg_reg_shift);
11430 %}
11431
11432 // This pattern is automatically generated from aarch64_ad.m4
11433 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11434 // val ^ (-1 ^ (val << shift)) ==> eon
11435 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11436 iRegL src1, iRegL src2,
11437 immI src3, immL_M1 src4) %{
11438 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11439 ins_cost(1.9 * INSN_COST);
11440 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11441
11442 ins_encode %{
11443 __ eon(as_Register($dst$$reg),
11444 as_Register($src1$$reg),
11445 as_Register($src2$$reg),
11446 Assembler::LSL,
11447 $src3$$constant & 0x3f);
11448 %}
11449
11450 ins_pipe(ialu_reg_reg_shift);
11451 %}
11452
11453 // This pattern is automatically generated from aarch64_ad.m4
11454 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11455 // val | (-1 ^ (val >>> shift)) ==> ornw
11456 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11457 iRegIorL2I src1, iRegIorL2I src2,
11458 immI src3, immI_M1 src4) %{
11459 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11460 ins_cost(1.9 * INSN_COST);
11461 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11462
11463 ins_encode %{
11464 __ ornw(as_Register($dst$$reg),
11465 as_Register($src1$$reg),
11466 as_Register($src2$$reg),
11467 Assembler::LSR,
11468 $src3$$constant & 0x1f);
11469 %}
11470
11471 ins_pipe(ialu_reg_reg_shift);
11472 %}
11473
11474 // This pattern is automatically generated from aarch64_ad.m4
11475 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11476 // val | (-1 ^ (val >>> shift)) ==> orn
11477 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11478 iRegL src1, iRegL src2,
11479 immI src3, immL_M1 src4) %{
11480 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11481 ins_cost(1.9 * INSN_COST);
11482 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11483
11484 ins_encode %{
11485 __ orn(as_Register($dst$$reg),
11486 as_Register($src1$$reg),
11487 as_Register($src2$$reg),
11488 Assembler::LSR,
11489 $src3$$constant & 0x3f);
11490 %}
11491
11492 ins_pipe(ialu_reg_reg_shift);
11493 %}
11494
11495 // This pattern is automatically generated from aarch64_ad.m4
11496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11497 // val | (-1 ^ (val >> shift)) ==> ornw
11498 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11499 iRegIorL2I src1, iRegIorL2I src2,
11500 immI src3, immI_M1 src4) %{
11501 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11502 ins_cost(1.9 * INSN_COST);
11503 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11504
11505 ins_encode %{
11506 __ ornw(as_Register($dst$$reg),
11507 as_Register($src1$$reg),
11508 as_Register($src2$$reg),
11509 Assembler::ASR,
11510 $src3$$constant & 0x1f);
11511 %}
11512
11513 ins_pipe(ialu_reg_reg_shift);
11514 %}
11515
11516 // This pattern is automatically generated from aarch64_ad.m4
11517 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11518 // val | (-1 ^ (val >> shift)) ==> orn
11519 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11520 iRegL src1, iRegL src2,
11521 immI src3, immL_M1 src4) %{
11522 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11523 ins_cost(1.9 * INSN_COST);
11524 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11525
11526 ins_encode %{
11527 __ orn(as_Register($dst$$reg),
11528 as_Register($src1$$reg),
11529 as_Register($src2$$reg),
11530 Assembler::ASR,
11531 $src3$$constant & 0x3f);
11532 %}
11533
11534 ins_pipe(ialu_reg_reg_shift);
11535 %}
11536
11537 // This pattern is automatically generated from aarch64_ad.m4
11538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11539 // val | (-1 ^ (val ror shift)) ==> ornw
11540 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11541 iRegIorL2I src1, iRegIorL2I src2,
11542 immI src3, immI_M1 src4) %{
11543 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11544 ins_cost(1.9 * INSN_COST);
11545 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11546
11547 ins_encode %{
11548 __ ornw(as_Register($dst$$reg),
11549 as_Register($src1$$reg),
11550 as_Register($src2$$reg),
11551 Assembler::ROR,
11552 $src3$$constant & 0x1f);
11553 %}
11554
11555 ins_pipe(ialu_reg_reg_shift);
11556 %}
11557
11558 // This pattern is automatically generated from aarch64_ad.m4
11559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11560 // val | (-1 ^ (val ror shift)) ==> orn
11561 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11562 iRegL src1, iRegL src2,
11563 immI src3, immL_M1 src4) %{
11564 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11565 ins_cost(1.9 * INSN_COST);
11566 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11567
11568 ins_encode %{
11569 __ orn(as_Register($dst$$reg),
11570 as_Register($src1$$reg),
11571 as_Register($src2$$reg),
11572 Assembler::ROR,
11573 $src3$$constant & 0x3f);
11574 %}
11575
11576 ins_pipe(ialu_reg_reg_shift);
11577 %}
11578
11579 // This pattern is automatically generated from aarch64_ad.m4
11580 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11581 // val | (-1 ^ (val << shift)) ==> ornw
11582 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11583 iRegIorL2I src1, iRegIorL2I src2,
11584 immI src3, immI_M1 src4) %{
11585 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11586 ins_cost(1.9 * INSN_COST);
11587 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11588
11589 ins_encode %{
11590 __ ornw(as_Register($dst$$reg),
11591 as_Register($src1$$reg),
11592 as_Register($src2$$reg),
11593 Assembler::LSL,
11594 $src3$$constant & 0x1f);
11595 %}
11596
11597 ins_pipe(ialu_reg_reg_shift);
11598 %}
11599
11600 // This pattern is automatically generated from aarch64_ad.m4
11601 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11602 // val | (-1 ^ (val << shift)) ==> orn
11603 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11604 iRegL src1, iRegL src2,
11605 immI src3, immL_M1 src4) %{
11606 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11607 ins_cost(1.9 * INSN_COST);
11608 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11609
11610 ins_encode %{
11611 __ orn(as_Register($dst$$reg),
11612 as_Register($src1$$reg),
11613 as_Register($src2$$reg),
11614 Assembler::LSL,
11615 $src3$$constant & 0x3f);
11616 %}
11617
11618 ins_pipe(ialu_reg_reg_shift);
11619 %}
11620
11621 // This pattern is automatically generated from aarch64_ad.m4
11622 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11623 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11624 iRegIorL2I src1, iRegIorL2I src2,
11625 immI src3) %{
11626 match(Set dst (AndI src1 (URShiftI src2 src3)));
11627
11628 ins_cost(1.9 * INSN_COST);
11629 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11630
11631 ins_encode %{
11632 __ andw(as_Register($dst$$reg),
11633 as_Register($src1$$reg),
11634 as_Register($src2$$reg),
11635 Assembler::LSR,
11636 $src3$$constant & 0x1f);
11637 %}
11638
11639 ins_pipe(ialu_reg_reg_shift);
11640 %}
11641
11642 // This pattern is automatically generated from aarch64_ad.m4
11643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11644 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11645 iRegL src1, iRegL src2,
11646 immI src3) %{
11647 match(Set dst (AndL src1 (URShiftL src2 src3)));
11648
11649 ins_cost(1.9 * INSN_COST);
11650 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11651
11652 ins_encode %{
11653 __ andr(as_Register($dst$$reg),
11654 as_Register($src1$$reg),
11655 as_Register($src2$$reg),
11656 Assembler::LSR,
11657 $src3$$constant & 0x3f);
11658 %}
11659
11660 ins_pipe(ialu_reg_reg_shift);
11661 %}
11662
11663 // This pattern is automatically generated from aarch64_ad.m4
11664 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11665 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11666 iRegIorL2I src1, iRegIorL2I src2,
11667 immI src3) %{
11668 match(Set dst (AndI src1 (RShiftI src2 src3)));
11669
11670 ins_cost(1.9 * INSN_COST);
11671 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11672
11673 ins_encode %{
11674 __ andw(as_Register($dst$$reg),
11675 as_Register($src1$$reg),
11676 as_Register($src2$$reg),
11677 Assembler::ASR,
11678 $src3$$constant & 0x1f);
11679 %}
11680
11681 ins_pipe(ialu_reg_reg_shift);
11682 %}
11683
11684 // This pattern is automatically generated from aarch64_ad.m4
11685 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11686 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11687 iRegL src1, iRegL src2,
11688 immI src3) %{
11689 match(Set dst (AndL src1 (RShiftL src2 src3)));
11690
11691 ins_cost(1.9 * INSN_COST);
11692 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11693
11694 ins_encode %{
11695 __ andr(as_Register($dst$$reg),
11696 as_Register($src1$$reg),
11697 as_Register($src2$$reg),
11698 Assembler::ASR,
11699 $src3$$constant & 0x3f);
11700 %}
11701
11702 ins_pipe(ialu_reg_reg_shift);
11703 %}
11704
11705 // This pattern is automatically generated from aarch64_ad.m4
11706 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11707 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11708 iRegIorL2I src1, iRegIorL2I src2,
11709 immI src3) %{
11710 match(Set dst (AndI src1 (LShiftI src2 src3)));
11711
11712 ins_cost(1.9 * INSN_COST);
11713 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11714
11715 ins_encode %{
11716 __ andw(as_Register($dst$$reg),
11717 as_Register($src1$$reg),
11718 as_Register($src2$$reg),
11719 Assembler::LSL,
11720 $src3$$constant & 0x1f);
11721 %}
11722
11723 ins_pipe(ialu_reg_reg_shift);
11724 %}
11725
11726 // This pattern is automatically generated from aarch64_ad.m4
11727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11728 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11729 iRegL src1, iRegL src2,
11730 immI src3) %{
11731 match(Set dst (AndL src1 (LShiftL src2 src3)));
11732
11733 ins_cost(1.9 * INSN_COST);
11734 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11735
11736 ins_encode %{
11737 __ andr(as_Register($dst$$reg),
11738 as_Register($src1$$reg),
11739 as_Register($src2$$reg),
11740 Assembler::LSL,
11741 $src3$$constant & 0x3f);
11742 %}
11743
11744 ins_pipe(ialu_reg_reg_shift);
11745 %}
11746
11747 // This pattern is automatically generated from aarch64_ad.m4
11748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11749 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11750 iRegIorL2I src1, iRegIorL2I src2,
11751 immI src3) %{
11752 match(Set dst (AndI src1 (RotateRight src2 src3)));
11753
11754 ins_cost(1.9 * INSN_COST);
11755 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11756
11757 ins_encode %{
11758 __ andw(as_Register($dst$$reg),
11759 as_Register($src1$$reg),
11760 as_Register($src2$$reg),
11761 Assembler::ROR,
11762 $src3$$constant & 0x1f);
11763 %}
11764
11765 ins_pipe(ialu_reg_reg_shift);
11766 %}
11767
11768 // This pattern is automatically generated from aarch64_ad.m4
11769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11770 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11771 iRegL src1, iRegL src2,
11772 immI src3) %{
11773 match(Set dst (AndL src1 (RotateRight src2 src3)));
11774
11775 ins_cost(1.9 * INSN_COST);
11776 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11777
11778 ins_encode %{
11779 __ andr(as_Register($dst$$reg),
11780 as_Register($src1$$reg),
11781 as_Register($src2$$reg),
11782 Assembler::ROR,
11783 $src3$$constant & 0x3f);
11784 %}
11785
11786 ins_pipe(ialu_reg_reg_shift);
11787 %}
11788
11789 // This pattern is automatically generated from aarch64_ad.m4
11790 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11791 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11792 iRegIorL2I src1, iRegIorL2I src2,
11793 immI src3) %{
11794 match(Set dst (XorI src1 (URShiftI src2 src3)));
11795
11796 ins_cost(1.9 * INSN_COST);
11797 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11798
11799 ins_encode %{
11800 __ eorw(as_Register($dst$$reg),
11801 as_Register($src1$$reg),
11802 as_Register($src2$$reg),
11803 Assembler::LSR,
11804 $src3$$constant & 0x1f);
11805 %}
11806
11807 ins_pipe(ialu_reg_reg_shift);
11808 %}
11809
11810 // This pattern is automatically generated from aarch64_ad.m4
11811 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11812 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11813 iRegL src1, iRegL src2,
11814 immI src3) %{
11815 match(Set dst (XorL src1 (URShiftL src2 src3)));
11816
11817 ins_cost(1.9 * INSN_COST);
11818 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11819
11820 ins_encode %{
11821 __ eor(as_Register($dst$$reg),
11822 as_Register($src1$$reg),
11823 as_Register($src2$$reg),
11824 Assembler::LSR,
11825 $src3$$constant & 0x3f);
11826 %}
11827
11828 ins_pipe(ialu_reg_reg_shift);
11829 %}
11830
11831 // This pattern is automatically generated from aarch64_ad.m4
11832 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11833 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11834 iRegIorL2I src1, iRegIorL2I src2,
11835 immI src3) %{
11836 match(Set dst (XorI src1 (RShiftI src2 src3)));
11837
11838 ins_cost(1.9 * INSN_COST);
11839 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11840
11841 ins_encode %{
11842 __ eorw(as_Register($dst$$reg),
11843 as_Register($src1$$reg),
11844 as_Register($src2$$reg),
11845 Assembler::ASR,
11846 $src3$$constant & 0x1f);
11847 %}
11848
11849 ins_pipe(ialu_reg_reg_shift);
11850 %}
11851
11852 // This pattern is automatically generated from aarch64_ad.m4
11853 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11854 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11855 iRegL src1, iRegL src2,
11856 immI src3) %{
11857 match(Set dst (XorL src1 (RShiftL src2 src3)));
11858
11859 ins_cost(1.9 * INSN_COST);
11860 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11861
11862 ins_encode %{
11863 __ eor(as_Register($dst$$reg),
11864 as_Register($src1$$reg),
11865 as_Register($src2$$reg),
11866 Assembler::ASR,
11867 $src3$$constant & 0x3f);
11868 %}
11869
11870 ins_pipe(ialu_reg_reg_shift);
11871 %}
11872
11873 // This pattern is automatically generated from aarch64_ad.m4
11874 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11875 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11876 iRegIorL2I src1, iRegIorL2I src2,
11877 immI src3) %{
11878 match(Set dst (XorI src1 (LShiftI src2 src3)));
11879
11880 ins_cost(1.9 * INSN_COST);
11881 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11882
11883 ins_encode %{
11884 __ eorw(as_Register($dst$$reg),
11885 as_Register($src1$$reg),
11886 as_Register($src2$$reg),
11887 Assembler::LSL,
11888 $src3$$constant & 0x1f);
11889 %}
11890
11891 ins_pipe(ialu_reg_reg_shift);
11892 %}
11893
11894 // This pattern is automatically generated from aarch64_ad.m4
11895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11896 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11897 iRegL src1, iRegL src2,
11898 immI src3) %{
11899 match(Set dst (XorL src1 (LShiftL src2 src3)));
11900
11901 ins_cost(1.9 * INSN_COST);
11902 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11903
11904 ins_encode %{
11905 __ eor(as_Register($dst$$reg),
11906 as_Register($src1$$reg),
11907 as_Register($src2$$reg),
11908 Assembler::LSL,
11909 $src3$$constant & 0x3f);
11910 %}
11911
11912 ins_pipe(ialu_reg_reg_shift);
11913 %}
11914
11915 // This pattern is automatically generated from aarch64_ad.m4
11916 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11917 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11918 iRegIorL2I src1, iRegIorL2I src2,
11919 immI src3) %{
11920 match(Set dst (XorI src1 (RotateRight src2 src3)));
11921
11922 ins_cost(1.9 * INSN_COST);
11923 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11924
11925 ins_encode %{
11926 __ eorw(as_Register($dst$$reg),
11927 as_Register($src1$$reg),
11928 as_Register($src2$$reg),
11929 Assembler::ROR,
11930 $src3$$constant & 0x1f);
11931 %}
11932
11933 ins_pipe(ialu_reg_reg_shift);
11934 %}
11935
11936 // This pattern is automatically generated from aarch64_ad.m4
11937 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11938 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11939 iRegL src1, iRegL src2,
11940 immI src3) %{
11941 match(Set dst (XorL src1 (RotateRight src2 src3)));
11942
11943 ins_cost(1.9 * INSN_COST);
11944 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11945
11946 ins_encode %{
11947 __ eor(as_Register($dst$$reg),
11948 as_Register($src1$$reg),
11949 as_Register($src2$$reg),
11950 Assembler::ROR,
11951 $src3$$constant & 0x3f);
11952 %}
11953
11954 ins_pipe(ialu_reg_reg_shift);
11955 %}
11956
11957 // This pattern is automatically generated from aarch64_ad.m4
11958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11959 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11960 iRegIorL2I src1, iRegIorL2I src2,
11961 immI src3) %{
11962 match(Set dst (OrI src1 (URShiftI src2 src3)));
11963
11964 ins_cost(1.9 * INSN_COST);
11965 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11966
11967 ins_encode %{
11968 __ orrw(as_Register($dst$$reg),
11969 as_Register($src1$$reg),
11970 as_Register($src2$$reg),
11971 Assembler::LSR,
11972 $src3$$constant & 0x1f);
11973 %}
11974
11975 ins_pipe(ialu_reg_reg_shift);
11976 %}
11977
11978 // This pattern is automatically generated from aarch64_ad.m4
11979 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11980 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11981 iRegL src1, iRegL src2,
11982 immI src3) %{
11983 match(Set dst (OrL src1 (URShiftL src2 src3)));
11984
11985 ins_cost(1.9 * INSN_COST);
11986 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11987
11988 ins_encode %{
11989 __ orr(as_Register($dst$$reg),
11990 as_Register($src1$$reg),
11991 as_Register($src2$$reg),
11992 Assembler::LSR,
11993 $src3$$constant & 0x3f);
11994 %}
11995
11996 ins_pipe(ialu_reg_reg_shift);
11997 %}
11998
11999 // This pattern is automatically generated from aarch64_ad.m4
12000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12001 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12002 iRegIorL2I src1, iRegIorL2I src2,
12003 immI src3) %{
12004 match(Set dst (OrI src1 (RShiftI src2 src3)));
12005
12006 ins_cost(1.9 * INSN_COST);
12007 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12008
12009 ins_encode %{
12010 __ orrw(as_Register($dst$$reg),
12011 as_Register($src1$$reg),
12012 as_Register($src2$$reg),
12013 Assembler::ASR,
12014 $src3$$constant & 0x1f);
12015 %}
12016
12017 ins_pipe(ialu_reg_reg_shift);
12018 %}
12019
12020 // This pattern is automatically generated from aarch64_ad.m4
12021 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12022 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12023 iRegL src1, iRegL src2,
12024 immI src3) %{
12025 match(Set dst (OrL src1 (RShiftL src2 src3)));
12026
12027 ins_cost(1.9 * INSN_COST);
12028 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12029
12030 ins_encode %{
12031 __ orr(as_Register($dst$$reg),
12032 as_Register($src1$$reg),
12033 as_Register($src2$$reg),
12034 Assembler::ASR,
12035 $src3$$constant & 0x3f);
12036 %}
12037
12038 ins_pipe(ialu_reg_reg_shift);
12039 %}
12040
12041 // This pattern is automatically generated from aarch64_ad.m4
12042 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12043 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12044 iRegIorL2I src1, iRegIorL2I src2,
12045 immI src3) %{
12046 match(Set dst (OrI src1 (LShiftI src2 src3)));
12047
12048 ins_cost(1.9 * INSN_COST);
12049 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12050
12051 ins_encode %{
12052 __ orrw(as_Register($dst$$reg),
12053 as_Register($src1$$reg),
12054 as_Register($src2$$reg),
12055 Assembler::LSL,
12056 $src3$$constant & 0x1f);
12057 %}
12058
12059 ins_pipe(ialu_reg_reg_shift);
12060 %}
12061
12062 // This pattern is automatically generated from aarch64_ad.m4
12063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12064 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12065 iRegL src1, iRegL src2,
12066 immI src3) %{
12067 match(Set dst (OrL src1 (LShiftL src2 src3)));
12068
12069 ins_cost(1.9 * INSN_COST);
12070 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12071
12072 ins_encode %{
12073 __ orr(as_Register($dst$$reg),
12074 as_Register($src1$$reg),
12075 as_Register($src2$$reg),
12076 Assembler::LSL,
12077 $src3$$constant & 0x3f);
12078 %}
12079
12080 ins_pipe(ialu_reg_reg_shift);
12081 %}
12082
12083 // This pattern is automatically generated from aarch64_ad.m4
12084 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12085 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12086 iRegIorL2I src1, iRegIorL2I src2,
12087 immI src3) %{
12088 match(Set dst (OrI src1 (RotateRight src2 src3)));
12089
12090 ins_cost(1.9 * INSN_COST);
12091 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12092
12093 ins_encode %{
12094 __ orrw(as_Register($dst$$reg),
12095 as_Register($src1$$reg),
12096 as_Register($src2$$reg),
12097 Assembler::ROR,
12098 $src3$$constant & 0x1f);
12099 %}
12100
12101 ins_pipe(ialu_reg_reg_shift);
12102 %}
12103
12104 // This pattern is automatically generated from aarch64_ad.m4
12105 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12106 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12107 iRegL src1, iRegL src2,
12108 immI src3) %{
12109 match(Set dst (OrL src1 (RotateRight src2 src3)));
12110
12111 ins_cost(1.9 * INSN_COST);
12112 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12113
12114 ins_encode %{
12115 __ orr(as_Register($dst$$reg),
12116 as_Register($src1$$reg),
12117 as_Register($src2$$reg),
12118 Assembler::ROR,
12119 $src3$$constant & 0x3f);
12120 %}
12121
12122 ins_pipe(ialu_reg_reg_shift);
12123 %}
12124
12125 // This pattern is automatically generated from aarch64_ad.m4
12126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12127 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12128 iRegIorL2I src1, iRegIorL2I src2,
12129 immI src3) %{
12130 match(Set dst (AddI src1 (URShiftI src2 src3)));
12131
12132 ins_cost(1.9 * INSN_COST);
12133 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12134
12135 ins_encode %{
12136 __ addw(as_Register($dst$$reg),
12137 as_Register($src1$$reg),
12138 as_Register($src2$$reg),
12139 Assembler::LSR,
12140 $src3$$constant & 0x1f);
12141 %}
12142
12143 ins_pipe(ialu_reg_reg_shift);
12144 %}
12145
12146 // This pattern is automatically generated from aarch64_ad.m4
12147 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12148 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12149 iRegL src1, iRegL src2,
12150 immI src3) %{
12151 match(Set dst (AddL src1 (URShiftL src2 src3)));
12152
12153 ins_cost(1.9 * INSN_COST);
12154 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12155
12156 ins_encode %{
12157 __ add(as_Register($dst$$reg),
12158 as_Register($src1$$reg),
12159 as_Register($src2$$reg),
12160 Assembler::LSR,
12161 $src3$$constant & 0x3f);
12162 %}
12163
12164 ins_pipe(ialu_reg_reg_shift);
12165 %}
12166
12167 // This pattern is automatically generated from aarch64_ad.m4
12168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12169 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12170 iRegIorL2I src1, iRegIorL2I src2,
12171 immI src3) %{
12172 match(Set dst (AddI src1 (RShiftI src2 src3)));
12173
12174 ins_cost(1.9 * INSN_COST);
12175 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12176
12177 ins_encode %{
12178 __ addw(as_Register($dst$$reg),
12179 as_Register($src1$$reg),
12180 as_Register($src2$$reg),
12181 Assembler::ASR,
12182 $src3$$constant & 0x1f);
12183 %}
12184
12185 ins_pipe(ialu_reg_reg_shift);
12186 %}
12187
12188 // This pattern is automatically generated from aarch64_ad.m4
12189 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12190 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12191 iRegL src1, iRegL src2,
12192 immI src3) %{
12193 match(Set dst (AddL src1 (RShiftL src2 src3)));
12194
12195 ins_cost(1.9 * INSN_COST);
12196 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12197
12198 ins_encode %{
12199 __ add(as_Register($dst$$reg),
12200 as_Register($src1$$reg),
12201 as_Register($src2$$reg),
12202 Assembler::ASR,
12203 $src3$$constant & 0x3f);
12204 %}
12205
12206 ins_pipe(ialu_reg_reg_shift);
12207 %}
12208
12209 // This pattern is automatically generated from aarch64_ad.m4
12210 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12211 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12212 iRegIorL2I src1, iRegIorL2I src2,
12213 immI src3) %{
12214 match(Set dst (AddI src1 (LShiftI src2 src3)));
12215
12216 ins_cost(1.9 * INSN_COST);
12217 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12218
12219 ins_encode %{
12220 __ addw(as_Register($dst$$reg),
12221 as_Register($src1$$reg),
12222 as_Register($src2$$reg),
12223 Assembler::LSL,
12224 $src3$$constant & 0x1f);
12225 %}
12226
12227 ins_pipe(ialu_reg_reg_shift);
12228 %}
12229
12230 // This pattern is automatically generated from aarch64_ad.m4
12231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12232 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12233 iRegL src1, iRegL src2,
12234 immI src3) %{
12235 match(Set dst (AddL src1 (LShiftL src2 src3)));
12236
12237 ins_cost(1.9 * INSN_COST);
12238 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12239
12240 ins_encode %{
12241 __ add(as_Register($dst$$reg),
12242 as_Register($src1$$reg),
12243 as_Register($src2$$reg),
12244 Assembler::LSL,
12245 $src3$$constant & 0x3f);
12246 %}
12247
12248 ins_pipe(ialu_reg_reg_shift);
12249 %}
12250
12251 // This pattern is automatically generated from aarch64_ad.m4
12252 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12253 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12254 iRegIorL2I src1, iRegIorL2I src2,
12255 immI src3) %{
12256 match(Set dst (SubI src1 (URShiftI src2 src3)));
12257
12258 ins_cost(1.9 * INSN_COST);
12259 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12260
12261 ins_encode %{
12262 __ subw(as_Register($dst$$reg),
12263 as_Register($src1$$reg),
12264 as_Register($src2$$reg),
12265 Assembler::LSR,
12266 $src3$$constant & 0x1f);
12267 %}
12268
12269 ins_pipe(ialu_reg_reg_shift);
12270 %}
12271
12272 // This pattern is automatically generated from aarch64_ad.m4
12273 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12274 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12275 iRegL src1, iRegL src2,
12276 immI src3) %{
12277 match(Set dst (SubL src1 (URShiftL src2 src3)));
12278
12279 ins_cost(1.9 * INSN_COST);
12280 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12281
12282 ins_encode %{
12283 __ sub(as_Register($dst$$reg),
12284 as_Register($src1$$reg),
12285 as_Register($src2$$reg),
12286 Assembler::LSR,
12287 $src3$$constant & 0x3f);
12288 %}
12289
12290 ins_pipe(ialu_reg_reg_shift);
12291 %}
12292
12293 // This pattern is automatically generated from aarch64_ad.m4
12294 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12295 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12296 iRegIorL2I src1, iRegIorL2I src2,
12297 immI src3) %{
12298 match(Set dst (SubI src1 (RShiftI src2 src3)));
12299
12300 ins_cost(1.9 * INSN_COST);
12301 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12302
12303 ins_encode %{
12304 __ subw(as_Register($dst$$reg),
12305 as_Register($src1$$reg),
12306 as_Register($src2$$reg),
12307 Assembler::ASR,
12308 $src3$$constant & 0x1f);
12309 %}
12310
12311 ins_pipe(ialu_reg_reg_shift);
12312 %}
12313
12314 // This pattern is automatically generated from aarch64_ad.m4
12315 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12316 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12317 iRegL src1, iRegL src2,
12318 immI src3) %{
12319 match(Set dst (SubL src1 (RShiftL src2 src3)));
12320
12321 ins_cost(1.9 * INSN_COST);
12322 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12323
12324 ins_encode %{
12325 __ sub(as_Register($dst$$reg),
12326 as_Register($src1$$reg),
12327 as_Register($src2$$reg),
12328 Assembler::ASR,
12329 $src3$$constant & 0x3f);
12330 %}
12331
12332 ins_pipe(ialu_reg_reg_shift);
12333 %}
12334
12335 // This pattern is automatically generated from aarch64_ad.m4
12336 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12337 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12338 iRegIorL2I src1, iRegIorL2I src2,
12339 immI src3) %{
12340 match(Set dst (SubI src1 (LShiftI src2 src3)));
12341
12342 ins_cost(1.9 * INSN_COST);
12343 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12344
12345 ins_encode %{
12346 __ subw(as_Register($dst$$reg),
12347 as_Register($src1$$reg),
12348 as_Register($src2$$reg),
12349 Assembler::LSL,
12350 $src3$$constant & 0x1f);
12351 %}
12352
12353 ins_pipe(ialu_reg_reg_shift);
12354 %}
12355
12356 // This pattern is automatically generated from aarch64_ad.m4
12357 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12358 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12359 iRegL src1, iRegL src2,
12360 immI src3) %{
12361 match(Set dst (SubL src1 (LShiftL src2 src3)));
12362
12363 ins_cost(1.9 * INSN_COST);
12364 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12365
12366 ins_encode %{
12367 __ sub(as_Register($dst$$reg),
12368 as_Register($src1$$reg),
12369 as_Register($src2$$reg),
12370 Assembler::LSL,
12371 $src3$$constant & 0x3f);
12372 %}
12373
12374 ins_pipe(ialu_reg_reg_shift);
12375 %}
12376
12377 // This pattern is automatically generated from aarch64_ad.m4
12378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12379
12380 // Shift Left followed by Shift Right.
12381 // This idiom is used by the compiler for the i2b bytecode etc.
12382 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12383 %{
12384 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12385 ins_cost(INSN_COST * 2);
12386 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12387 ins_encode %{
12388 int lshift = $lshift_count$$constant & 63;
12389 int rshift = $rshift_count$$constant & 63;
12390 int s = 63 - lshift;
12391 int r = (rshift - lshift) & 63;
12392 __ sbfm(as_Register($dst$$reg),
12393 as_Register($src$$reg),
12394 r, s);
12395 %}
12396
12397 ins_pipe(ialu_reg_shift);
12398 %}
12399
12400 // This pattern is automatically generated from aarch64_ad.m4
12401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12402
12403 // Shift Left followed by Shift Right.
12404 // This idiom is used by the compiler for the i2b bytecode etc.
12405 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12406 %{
12407 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12408 ins_cost(INSN_COST * 2);
12409 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12410 ins_encode %{
12411 int lshift = $lshift_count$$constant & 31;
12412 int rshift = $rshift_count$$constant & 31;
12413 int s = 31 - lshift;
12414 int r = (rshift - lshift) & 31;
12415 __ sbfmw(as_Register($dst$$reg),
12416 as_Register($src$$reg),
12417 r, s);
12418 %}
12419
12420 ins_pipe(ialu_reg_shift);
12421 %}
12422
12423 // This pattern is automatically generated from aarch64_ad.m4
12424 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12425
12426 // Shift Left followed by Shift Right.
12427 // This idiom is used by the compiler for the i2b bytecode etc.
12428 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12429 %{
12430 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12431 ins_cost(INSN_COST * 2);
12432 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12433 ins_encode %{
12434 int lshift = $lshift_count$$constant & 63;
12435 int rshift = $rshift_count$$constant & 63;
12436 int s = 63 - lshift;
12437 int r = (rshift - lshift) & 63;
12438 __ ubfm(as_Register($dst$$reg),
12439 as_Register($src$$reg),
12440 r, s);
12441 %}
12442
12443 ins_pipe(ialu_reg_shift);
12444 %}
12445
12446 // This pattern is automatically generated from aarch64_ad.m4
12447 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12448
12449 // Shift Left followed by Shift Right.
12450 // This idiom is used by the compiler for the i2b bytecode etc.
12451 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12452 %{
12453 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12454 ins_cost(INSN_COST * 2);
12455 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12456 ins_encode %{
12457 int lshift = $lshift_count$$constant & 31;
12458 int rshift = $rshift_count$$constant & 31;
12459 int s = 31 - lshift;
12460 int r = (rshift - lshift) & 31;
12461 __ ubfmw(as_Register($dst$$reg),
12462 as_Register($src$$reg),
12463 r, s);
12464 %}
12465
12466 ins_pipe(ialu_reg_shift);
12467 %}
12468
12469 // Bitfield extract with shift & mask
12470
12471 // This pattern is automatically generated from aarch64_ad.m4
12472 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12473 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12474 %{
12475 match(Set dst (AndI (URShiftI src rshift) mask));
12476 // Make sure we are not going to exceed what ubfxw can do.
12477 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12478
12479 ins_cost(INSN_COST);
12480 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12481 ins_encode %{
12482 int rshift = $rshift$$constant & 31;
12483 intptr_t mask = $mask$$constant;
12484 int width = exact_log2(mask+1);
12485 __ ubfxw(as_Register($dst$$reg),
12486 as_Register($src$$reg), rshift, width);
12487 %}
12488 ins_pipe(ialu_reg_shift);
12489 %}
12490
12491 // This pattern is automatically generated from aarch64_ad.m4
12492 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12493 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12494 %{
12495 match(Set dst (AndL (URShiftL src rshift) mask));
12496 // Make sure we are not going to exceed what ubfx can do.
12497 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12498
12499 ins_cost(INSN_COST);
12500 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12501 ins_encode %{
12502 int rshift = $rshift$$constant & 63;
12503 intptr_t mask = $mask$$constant;
12504 int width = exact_log2_long(mask+1);
12505 __ ubfx(as_Register($dst$$reg),
12506 as_Register($src$$reg), rshift, width);
12507 %}
12508 ins_pipe(ialu_reg_shift);
12509 %}
12510
12511
12512 // This pattern is automatically generated from aarch64_ad.m4
12513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12514
12515 // We can use ubfx when extending an And with a mask when we know mask
12516 // is positive. We know that because immI_bitmask guarantees it.
12517 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12518 %{
12519 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12520 // Make sure we are not going to exceed what ubfxw can do.
12521 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12522
12523 ins_cost(INSN_COST * 2);
12524 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12525 ins_encode %{
12526 int rshift = $rshift$$constant & 31;
12527 intptr_t mask = $mask$$constant;
12528 int width = exact_log2(mask+1);
12529 __ ubfx(as_Register($dst$$reg),
12530 as_Register($src$$reg), rshift, width);
12531 %}
12532 ins_pipe(ialu_reg_shift);
12533 %}
12534
12535
12536 // This pattern is automatically generated from aarch64_ad.m4
12537 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12538
12539 // We can use ubfiz when masking by a positive number and then left shifting the result.
12540 // We know that the mask is positive because immI_bitmask guarantees it.
12541 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12542 %{
12543 match(Set dst (LShiftI (AndI src mask) lshift));
12544 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12545
12546 ins_cost(INSN_COST);
12547 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12548 ins_encode %{
12549 int lshift = $lshift$$constant & 31;
12550 intptr_t mask = $mask$$constant;
12551 int width = exact_log2(mask+1);
12552 __ ubfizw(as_Register($dst$$reg),
12553 as_Register($src$$reg), lshift, width);
12554 %}
12555 ins_pipe(ialu_reg_shift);
12556 %}
12557
12558 // This pattern is automatically generated from aarch64_ad.m4
12559 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12560
12561 // We can use ubfiz when masking by a positive number and then left shifting the result.
12562 // We know that the mask is positive because immL_bitmask guarantees it.
12563 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12564 %{
12565 match(Set dst (LShiftL (AndL src mask) lshift));
12566 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12567
12568 ins_cost(INSN_COST);
12569 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12570 ins_encode %{
12571 int lshift = $lshift$$constant & 63;
12572 intptr_t mask = $mask$$constant;
12573 int width = exact_log2_long(mask+1);
12574 __ ubfiz(as_Register($dst$$reg),
12575 as_Register($src$$reg), lshift, width);
12576 %}
12577 ins_pipe(ialu_reg_shift);
12578 %}
12579
12580 // This pattern is automatically generated from aarch64_ad.m4
12581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12582
12583 // We can use ubfiz when masking by a positive number and then left shifting the result.
12584 // We know that the mask is positive because immI_bitmask guarantees it.
12585 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12586 %{
12587 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12588 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12589
12590 ins_cost(INSN_COST);
12591 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12592 ins_encode %{
12593 int lshift = $lshift$$constant & 31;
12594 intptr_t mask = $mask$$constant;
12595 int width = exact_log2(mask+1);
12596 __ ubfizw(as_Register($dst$$reg),
12597 as_Register($src$$reg), lshift, width);
12598 %}
12599 ins_pipe(ialu_reg_shift);
12600 %}
12601
12602 // This pattern is automatically generated from aarch64_ad.m4
12603 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12604
12605 // We can use ubfiz when masking by a positive number and then left shifting the result.
12606 // We know that the mask is positive because immL_bitmask guarantees it.
12607 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12608 %{
12609 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12610 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12611
12612 ins_cost(INSN_COST);
12613 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12614 ins_encode %{
12615 int lshift = $lshift$$constant & 63;
12616 intptr_t mask = $mask$$constant;
12617 int width = exact_log2_long(mask+1);
12618 __ ubfiz(as_Register($dst$$reg),
12619 as_Register($src$$reg), lshift, width);
12620 %}
12621 ins_pipe(ialu_reg_shift);
12622 %}
12623
12624
12625 // This pattern is automatically generated from aarch64_ad.m4
12626 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12627
12628 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12629 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12630 %{
12631 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12632 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12633
12634 ins_cost(INSN_COST);
12635 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12636 ins_encode %{
12637 int lshift = $lshift$$constant & 63;
12638 intptr_t mask = $mask$$constant;
12639 int width = exact_log2(mask+1);
12640 __ ubfiz(as_Register($dst$$reg),
12641 as_Register($src$$reg), lshift, width);
12642 %}
12643 ins_pipe(ialu_reg_shift);
12644 %}
12645
12646 // This pattern is automatically generated from aarch64_ad.m4
12647 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12648
12649 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12650 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12651 %{
12652 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12653 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12654
12655 ins_cost(INSN_COST);
12656 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12657 ins_encode %{
12658 int lshift = $lshift$$constant & 31;
12659 intptr_t mask = $mask$$constant;
12660 int width = exact_log2(mask+1);
12661 __ ubfiz(as_Register($dst$$reg),
12662 as_Register($src$$reg), lshift, width);
12663 %}
12664 ins_pipe(ialu_reg_shift);
12665 %}
12666
12667 // This pattern is automatically generated from aarch64_ad.m4
12668 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12669
12670 // Can skip int2long conversions after AND with small bitmask
12671 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12672 %{
12673 match(Set dst (ConvI2L (AndI src msk)));
12674 ins_cost(INSN_COST);
12675 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12676 ins_encode %{
12677 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12678 %}
12679 ins_pipe(ialu_reg_shift);
12680 %}
12681
12682
12683 // Rotations
12684
12685 // This pattern is automatically generated from aarch64_ad.m4
12686 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12687 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12688 %{
12689 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12690 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12691
12692 ins_cost(INSN_COST);
12693 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12694
12695 ins_encode %{
12696 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12697 $rshift$$constant & 63);
12698 %}
12699 ins_pipe(ialu_reg_reg_extr);
12700 %}
12701
12702
12703 // This pattern is automatically generated from aarch64_ad.m4
12704 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12705 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12706 %{
12707 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12708 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12709
12710 ins_cost(INSN_COST);
12711 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12712
12713 ins_encode %{
12714 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12715 $rshift$$constant & 31);
12716 %}
12717 ins_pipe(ialu_reg_reg_extr);
12718 %}
12719
12720
12721 // This pattern is automatically generated from aarch64_ad.m4
12722 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12723 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12724 %{
12725 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12726 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12727
12728 ins_cost(INSN_COST);
12729 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12730
12731 ins_encode %{
12732 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12733 $rshift$$constant & 63);
12734 %}
12735 ins_pipe(ialu_reg_reg_extr);
12736 %}
12737
12738
12739 // This pattern is automatically generated from aarch64_ad.m4
12740 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12741 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12742 %{
12743 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12744 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12745
12746 ins_cost(INSN_COST);
12747 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12748
12749 ins_encode %{
12750 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12751 $rshift$$constant & 31);
12752 %}
12753 ins_pipe(ialu_reg_reg_extr);
12754 %}
12755
12756 // This pattern is automatically generated from aarch64_ad.m4
12757 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12758 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12759 %{
12760 match(Set dst (RotateRight src shift));
12761
12762 ins_cost(INSN_COST);
12763 format %{ "ror $dst, $src, $shift" %}
12764
12765 ins_encode %{
12766 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12767 $shift$$constant & 0x1f);
12768 %}
12769 ins_pipe(ialu_reg_reg_vshift);
12770 %}
12771
12772 // This pattern is automatically generated from aarch64_ad.m4
12773 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12774 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12775 %{
12776 match(Set dst (RotateRight src shift));
12777
12778 ins_cost(INSN_COST);
12779 format %{ "ror $dst, $src, $shift" %}
12780
12781 ins_encode %{
12782 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12783 $shift$$constant & 0x3f);
12784 %}
12785 ins_pipe(ialu_reg_reg_vshift);
12786 %}
12787
12788 // This pattern is automatically generated from aarch64_ad.m4
12789 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12790 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12791 %{
12792 match(Set dst (RotateRight src shift));
12793
12794 ins_cost(INSN_COST);
12795 format %{ "ror $dst, $src, $shift" %}
12796
12797 ins_encode %{
12798 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12799 %}
12800 ins_pipe(ialu_reg_reg_vshift);
12801 %}
12802
12803 // This pattern is automatically generated from aarch64_ad.m4
12804 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12805 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12806 %{
12807 match(Set dst (RotateRight src shift));
12808
12809 ins_cost(INSN_COST);
12810 format %{ "ror $dst, $src, $shift" %}
12811
12812 ins_encode %{
12813 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12814 %}
12815 ins_pipe(ialu_reg_reg_vshift);
12816 %}
12817
12818 // This pattern is automatically generated from aarch64_ad.m4
12819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12820 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12821 %{
12822 match(Set dst (RotateLeft src shift));
12823
12824 ins_cost(INSN_COST);
12825 format %{ "rol $dst, $src, $shift" %}
12826
12827 ins_encode %{
12828 __ subw(rscratch1, zr, as_Register($shift$$reg));
12829 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12830 %}
12831 ins_pipe(ialu_reg_reg_vshift);
12832 %}
12833
12834 // This pattern is automatically generated from aarch64_ad.m4
12835 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12836 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12837 %{
12838 match(Set dst (RotateLeft src shift));
12839
12840 ins_cost(INSN_COST);
12841 format %{ "rol $dst, $src, $shift" %}
12842
12843 ins_encode %{
12844 __ subw(rscratch1, zr, as_Register($shift$$reg));
12845 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12846 %}
12847 ins_pipe(ialu_reg_reg_vshift);
12848 %}
12849
12850
12851 // Add/subtract (extended)
12852
12853 // This pattern is automatically generated from aarch64_ad.m4
12854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12855 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12856 %{
12857 match(Set dst (AddL src1 (ConvI2L src2)));
12858 ins_cost(INSN_COST);
12859 format %{ "add $dst, $src1, $src2, sxtw" %}
12860
12861 ins_encode %{
12862 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12863 as_Register($src2$$reg), ext::sxtw);
12864 %}
12865 ins_pipe(ialu_reg_reg);
12866 %}
12867
12868 // This pattern is automatically generated from aarch64_ad.m4
12869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12870 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12871 %{
12872 match(Set dst (SubL src1 (ConvI2L src2)));
12873 ins_cost(INSN_COST);
12874 format %{ "sub $dst, $src1, $src2, sxtw" %}
12875
12876 ins_encode %{
12877 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12878 as_Register($src2$$reg), ext::sxtw);
12879 %}
12880 ins_pipe(ialu_reg_reg);
12881 %}
12882
12883 // This pattern is automatically generated from aarch64_ad.m4
12884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12885 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12886 %{
12887 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12888 ins_cost(INSN_COST);
12889 format %{ "add $dst, $src1, $src2, sxth" %}
12890
12891 ins_encode %{
12892 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12893 as_Register($src2$$reg), ext::sxth);
12894 %}
12895 ins_pipe(ialu_reg_reg);
12896 %}
12897
12898 // This pattern is automatically generated from aarch64_ad.m4
12899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12900 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12901 %{
12902 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12903 ins_cost(INSN_COST);
12904 format %{ "add $dst, $src1, $src2, sxtb" %}
12905
12906 ins_encode %{
12907 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12908 as_Register($src2$$reg), ext::sxtb);
12909 %}
12910 ins_pipe(ialu_reg_reg);
12911 %}
12912
12913 // This pattern is automatically generated from aarch64_ad.m4
12914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12915 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12916 %{
12917 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12918 ins_cost(INSN_COST);
12919 format %{ "add $dst, $src1, $src2, uxtb" %}
12920
12921 ins_encode %{
12922 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12923 as_Register($src2$$reg), ext::uxtb);
12924 %}
12925 ins_pipe(ialu_reg_reg);
12926 %}
12927
12928 // This pattern is automatically generated from aarch64_ad.m4
12929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12930 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12931 %{
12932 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12933 ins_cost(INSN_COST);
12934 format %{ "add $dst, $src1, $src2, sxth" %}
12935
12936 ins_encode %{
12937 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12938 as_Register($src2$$reg), ext::sxth);
12939 %}
12940 ins_pipe(ialu_reg_reg);
12941 %}
12942
12943 // This pattern is automatically generated from aarch64_ad.m4
12944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12945 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12946 %{
12947 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12948 ins_cost(INSN_COST);
12949 format %{ "add $dst, $src1, $src2, sxtw" %}
12950
12951 ins_encode %{
12952 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12953 as_Register($src2$$reg), ext::sxtw);
12954 %}
12955 ins_pipe(ialu_reg_reg);
12956 %}
12957
12958 // This pattern is automatically generated from aarch64_ad.m4
12959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12960 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12961 %{
12962 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12963 ins_cost(INSN_COST);
12964 format %{ "add $dst, $src1, $src2, sxtb" %}
12965
12966 ins_encode %{
12967 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12968 as_Register($src2$$reg), ext::sxtb);
12969 %}
12970 ins_pipe(ialu_reg_reg);
12971 %}
12972
12973 // This pattern is automatically generated from aarch64_ad.m4
12974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12975 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12976 %{
12977 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12978 ins_cost(INSN_COST);
12979 format %{ "add $dst, $src1, $src2, uxtb" %}
12980
12981 ins_encode %{
12982 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12983 as_Register($src2$$reg), ext::uxtb);
12984 %}
12985 ins_pipe(ialu_reg_reg);
12986 %}
12987
12988 // This pattern is automatically generated from aarch64_ad.m4
12989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12990 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12991 %{
12992 match(Set dst (AddI src1 (AndI src2 mask)));
12993 ins_cost(INSN_COST);
12994 format %{ "addw $dst, $src1, $src2, uxtb" %}
12995
12996 ins_encode %{
12997 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12998 as_Register($src2$$reg), ext::uxtb);
12999 %}
13000 ins_pipe(ialu_reg_reg);
13001 %}
13002
13003 // This pattern is automatically generated from aarch64_ad.m4
13004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13005 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13006 %{
13007 match(Set dst (AddI src1 (AndI src2 mask)));
13008 ins_cost(INSN_COST);
13009 format %{ "addw $dst, $src1, $src2, uxth" %}
13010
13011 ins_encode %{
13012 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13013 as_Register($src2$$reg), ext::uxth);
13014 %}
13015 ins_pipe(ialu_reg_reg);
13016 %}
13017
13018 // This pattern is automatically generated from aarch64_ad.m4
13019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13020 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13021 %{
13022 match(Set dst (AddL src1 (AndL src2 mask)));
13023 ins_cost(INSN_COST);
13024 format %{ "add $dst, $src1, $src2, uxtb" %}
13025
13026 ins_encode %{
13027 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13028 as_Register($src2$$reg), ext::uxtb);
13029 %}
13030 ins_pipe(ialu_reg_reg);
13031 %}
13032
13033 // This pattern is automatically generated from aarch64_ad.m4
13034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13035 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13036 %{
13037 match(Set dst (AddL src1 (AndL src2 mask)));
13038 ins_cost(INSN_COST);
13039 format %{ "add $dst, $src1, $src2, uxth" %}
13040
13041 ins_encode %{
13042 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13043 as_Register($src2$$reg), ext::uxth);
13044 %}
13045 ins_pipe(ialu_reg_reg);
13046 %}
13047
13048 // This pattern is automatically generated from aarch64_ad.m4
13049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13050 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13051 %{
13052 match(Set dst (AddL src1 (AndL src2 mask)));
13053 ins_cost(INSN_COST);
13054 format %{ "add $dst, $src1, $src2, uxtw" %}
13055
13056 ins_encode %{
13057 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13058 as_Register($src2$$reg), ext::uxtw);
13059 %}
13060 ins_pipe(ialu_reg_reg);
13061 %}
13062
13063 // This pattern is automatically generated from aarch64_ad.m4
13064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13065 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13066 %{
13067 match(Set dst (SubI src1 (AndI src2 mask)));
13068 ins_cost(INSN_COST);
13069 format %{ "subw $dst, $src1, $src2, uxtb" %}
13070
13071 ins_encode %{
13072 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13073 as_Register($src2$$reg), ext::uxtb);
13074 %}
13075 ins_pipe(ialu_reg_reg);
13076 %}
13077
13078 // This pattern is automatically generated from aarch64_ad.m4
13079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13080 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13081 %{
13082 match(Set dst (SubI src1 (AndI src2 mask)));
13083 ins_cost(INSN_COST);
13084 format %{ "subw $dst, $src1, $src2, uxth" %}
13085
13086 ins_encode %{
13087 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13088 as_Register($src2$$reg), ext::uxth);
13089 %}
13090 ins_pipe(ialu_reg_reg);
13091 %}
13092
13093 // This pattern is automatically generated from aarch64_ad.m4
13094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13095 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13096 %{
13097 match(Set dst (SubL src1 (AndL src2 mask)));
13098 ins_cost(INSN_COST);
13099 format %{ "sub $dst, $src1, $src2, uxtb" %}
13100
13101 ins_encode %{
13102 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13103 as_Register($src2$$reg), ext::uxtb);
13104 %}
13105 ins_pipe(ialu_reg_reg);
13106 %}
13107
13108 // This pattern is automatically generated from aarch64_ad.m4
13109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13110 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13111 %{
13112 match(Set dst (SubL src1 (AndL src2 mask)));
13113 ins_cost(INSN_COST);
13114 format %{ "sub $dst, $src1, $src2, uxth" %}
13115
13116 ins_encode %{
13117 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13118 as_Register($src2$$reg), ext::uxth);
13119 %}
13120 ins_pipe(ialu_reg_reg);
13121 %}
13122
13123 // This pattern is automatically generated from aarch64_ad.m4
13124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13125 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13126 %{
13127 match(Set dst (SubL src1 (AndL src2 mask)));
13128 ins_cost(INSN_COST);
13129 format %{ "sub $dst, $src1, $src2, uxtw" %}
13130
13131 ins_encode %{
13132 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13133 as_Register($src2$$reg), ext::uxtw);
13134 %}
13135 ins_pipe(ialu_reg_reg);
13136 %}
13137
13138
13139 // This pattern is automatically generated from aarch64_ad.m4
13140 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13141 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13142 %{
13143 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13144 ins_cost(1.9 * INSN_COST);
13145 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13146
13147 ins_encode %{
13148 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13149 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13150 %}
13151 ins_pipe(ialu_reg_reg_shift);
13152 %}
13153
13154 // This pattern is automatically generated from aarch64_ad.m4
13155 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13156 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13157 %{
13158 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13159 ins_cost(1.9 * INSN_COST);
13160 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13161
13162 ins_encode %{
13163 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13164 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13165 %}
13166 ins_pipe(ialu_reg_reg_shift);
13167 %}
13168
13169 // This pattern is automatically generated from aarch64_ad.m4
13170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13171 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13172 %{
13173 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13174 ins_cost(1.9 * INSN_COST);
13175 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13176
13177 ins_encode %{
13178 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13179 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13180 %}
13181 ins_pipe(ialu_reg_reg_shift);
13182 %}
13183
13184 // This pattern is automatically generated from aarch64_ad.m4
13185 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13186 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13187 %{
13188 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13189 ins_cost(1.9 * INSN_COST);
13190 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13191
13192 ins_encode %{
13193 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13194 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13195 %}
13196 ins_pipe(ialu_reg_reg_shift);
13197 %}
13198
13199 // This pattern is automatically generated from aarch64_ad.m4
13200 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13201 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13202 %{
13203 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13204 ins_cost(1.9 * INSN_COST);
13205 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13206
13207 ins_encode %{
13208 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13209 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13210 %}
13211 ins_pipe(ialu_reg_reg_shift);
13212 %}
13213
13214 // This pattern is automatically generated from aarch64_ad.m4
13215 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13216 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13217 %{
13218 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13219 ins_cost(1.9 * INSN_COST);
13220 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13221
13222 ins_encode %{
13223 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13224 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13225 %}
13226 ins_pipe(ialu_reg_reg_shift);
13227 %}
13228
13229 // This pattern is automatically generated from aarch64_ad.m4
13230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13231 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13232 %{
13233 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13234 ins_cost(1.9 * INSN_COST);
13235 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13236
13237 ins_encode %{
13238 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13239 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13240 %}
13241 ins_pipe(ialu_reg_reg_shift);
13242 %}
13243
13244 // This pattern is automatically generated from aarch64_ad.m4
13245 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13246 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13247 %{
13248 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13249 ins_cost(1.9 * INSN_COST);
13250 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13251
13252 ins_encode %{
13253 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13254 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13255 %}
13256 ins_pipe(ialu_reg_reg_shift);
13257 %}
13258
13259 // This pattern is automatically generated from aarch64_ad.m4
13260 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13261 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13262 %{
13263 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13264 ins_cost(1.9 * INSN_COST);
13265 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13266
13267 ins_encode %{
13268 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13269 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13270 %}
13271 ins_pipe(ialu_reg_reg_shift);
13272 %}
13273
13274 // This pattern is automatically generated from aarch64_ad.m4
13275 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13276 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13277 %{
13278 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13279 ins_cost(1.9 * INSN_COST);
13280 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13281
13282 ins_encode %{
13283 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13284 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13285 %}
13286 ins_pipe(ialu_reg_reg_shift);
13287 %}
13288
13289 // This pattern is automatically generated from aarch64_ad.m4
13290 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13291 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13292 %{
13293 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13294 ins_cost(1.9 * INSN_COST);
13295 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13296
13297 ins_encode %{
13298 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13299 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13300 %}
13301 ins_pipe(ialu_reg_reg_shift);
13302 %}
13303
13304 // This pattern is automatically generated from aarch64_ad.m4
13305 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13306 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13307 %{
13308 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13309 ins_cost(1.9 * INSN_COST);
13310 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13311
13312 ins_encode %{
13313 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13314 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13315 %}
13316 ins_pipe(ialu_reg_reg_shift);
13317 %}
13318
13319 // This pattern is automatically generated from aarch64_ad.m4
13320 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13321 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13322 %{
13323 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13324 ins_cost(1.9 * INSN_COST);
13325 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13326
13327 ins_encode %{
13328 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13329 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13330 %}
13331 ins_pipe(ialu_reg_reg_shift);
13332 %}
13333
13334 // This pattern is automatically generated from aarch64_ad.m4
13335 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13336 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13337 %{
13338 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13339 ins_cost(1.9 * INSN_COST);
13340 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13341
13342 ins_encode %{
13343 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13344 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13345 %}
13346 ins_pipe(ialu_reg_reg_shift);
13347 %}
13348
13349 // This pattern is automatically generated from aarch64_ad.m4
13350 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13351 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13352 %{
13353 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13354 ins_cost(1.9 * INSN_COST);
13355 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13356
13357 ins_encode %{
13358 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13359 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13360 %}
13361 ins_pipe(ialu_reg_reg_shift);
13362 %}
13363
13364 // This pattern is automatically generated from aarch64_ad.m4
13365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13366 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13367 %{
13368 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13369 ins_cost(1.9 * INSN_COST);
13370 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13371
13372 ins_encode %{
13373 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13374 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13375 %}
13376 ins_pipe(ialu_reg_reg_shift);
13377 %}
13378
13379 // This pattern is automatically generated from aarch64_ad.m4
13380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13381 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13382 %{
13383 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13384 ins_cost(1.9 * INSN_COST);
13385 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13386
13387 ins_encode %{
13388 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13389 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13390 %}
13391 ins_pipe(ialu_reg_reg_shift);
13392 %}
13393
13394 // This pattern is automatically generated from aarch64_ad.m4
13395 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13396 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13397 %{
13398 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13399 ins_cost(1.9 * INSN_COST);
13400 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13401
13402 ins_encode %{
13403 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13404 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13405 %}
13406 ins_pipe(ialu_reg_reg_shift);
13407 %}
13408
13409 // This pattern is automatically generated from aarch64_ad.m4
13410 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13411 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13412 %{
13413 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13414 ins_cost(1.9 * INSN_COST);
13415 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13416
13417 ins_encode %{
13418 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13419 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13420 %}
13421 ins_pipe(ialu_reg_reg_shift);
13422 %}
13423
13424 // This pattern is automatically generated from aarch64_ad.m4
13425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13426 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13427 %{
13428 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13429 ins_cost(1.9 * INSN_COST);
13430 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13431
13432 ins_encode %{
13433 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13434 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13435 %}
13436 ins_pipe(ialu_reg_reg_shift);
13437 %}
13438
13439 // This pattern is automatically generated from aarch64_ad.m4
13440 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13441 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13442 %{
13443 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13444 ins_cost(1.9 * INSN_COST);
13445 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13446
13447 ins_encode %{
13448 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13449 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13450 %}
13451 ins_pipe(ialu_reg_reg_shift);
13452 %}
13453
13454 // This pattern is automatically generated from aarch64_ad.m4
13455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13456 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13457 %{
13458 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13459 ins_cost(1.9 * INSN_COST);
13460 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13461
13462 ins_encode %{
13463 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13464 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13465 %}
13466 ins_pipe(ialu_reg_reg_shift);
13467 %}
13468
13469 // This pattern is automatically generated from aarch64_ad.m4
13470 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13471 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13472 %{
13473 effect(DEF dst, USE src1, USE src2, USE cr);
13474 ins_cost(INSN_COST * 2);
13475 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13476
13477 ins_encode %{
13478 __ cselw($dst$$Register,
13479 $src1$$Register,
13480 $src2$$Register,
13481 Assembler::LT);
13482 %}
13483 ins_pipe(icond_reg_reg);
13484 %}
13485
13486 // This pattern is automatically generated from aarch64_ad.m4
13487 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13488 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13489 %{
13490 effect(DEF dst, USE src1, USE src2, USE cr);
13491 ins_cost(INSN_COST * 2);
13492 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13493
13494 ins_encode %{
13495 __ cselw($dst$$Register,
13496 $src1$$Register,
13497 $src2$$Register,
13498 Assembler::GT);
13499 %}
13500 ins_pipe(icond_reg_reg);
13501 %}
13502
13503 // This pattern is automatically generated from aarch64_ad.m4
13504 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13505 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13506 %{
13507 effect(DEF dst, USE src1, USE cr);
13508 ins_cost(INSN_COST * 2);
13509 format %{ "cselw $dst, $src1, zr lt\t" %}
13510
13511 ins_encode %{
13512 __ cselw($dst$$Register,
13513 $src1$$Register,
13514 zr,
13515 Assembler::LT);
13516 %}
13517 ins_pipe(icond_reg);
13518 %}
13519
13520 // This pattern is automatically generated from aarch64_ad.m4
13521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13522 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13523 %{
13524 effect(DEF dst, USE src1, USE cr);
13525 ins_cost(INSN_COST * 2);
13526 format %{ "cselw $dst, $src1, zr gt\t" %}
13527
13528 ins_encode %{
13529 __ cselw($dst$$Register,
13530 $src1$$Register,
13531 zr,
13532 Assembler::GT);
13533 %}
13534 ins_pipe(icond_reg);
13535 %}
13536
13537 // This pattern is automatically generated from aarch64_ad.m4
13538 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13539 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13540 %{
13541 effect(DEF dst, USE src1, USE cr);
13542 ins_cost(INSN_COST * 2);
13543 format %{ "csincw $dst, $src1, zr le\t" %}
13544
13545 ins_encode %{
13546 __ csincw($dst$$Register,
13547 $src1$$Register,
13548 zr,
13549 Assembler::LE);
13550 %}
13551 ins_pipe(icond_reg);
13552 %}
13553
13554 // This pattern is automatically generated from aarch64_ad.m4
13555 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13556 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13557 %{
13558 effect(DEF dst, USE src1, USE cr);
13559 ins_cost(INSN_COST * 2);
13560 format %{ "csincw $dst, $src1, zr gt\t" %}
13561
13562 ins_encode %{
13563 __ csincw($dst$$Register,
13564 $src1$$Register,
13565 zr,
13566 Assembler::GT);
13567 %}
13568 ins_pipe(icond_reg);
13569 %}
13570
13571 // This pattern is automatically generated from aarch64_ad.m4
13572 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13573 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13574 %{
13575 effect(DEF dst, USE src1, USE cr);
13576 ins_cost(INSN_COST * 2);
13577 format %{ "csinvw $dst, $src1, zr lt\t" %}
13578
13579 ins_encode %{
13580 __ csinvw($dst$$Register,
13581 $src1$$Register,
13582 zr,
13583 Assembler::LT);
13584 %}
13585 ins_pipe(icond_reg);
13586 %}
13587
13588 // This pattern is automatically generated from aarch64_ad.m4
13589 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13590 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13591 %{
13592 effect(DEF dst, USE src1, USE cr);
13593 ins_cost(INSN_COST * 2);
13594 format %{ "csinvw $dst, $src1, zr ge\t" %}
13595
13596 ins_encode %{
13597 __ csinvw($dst$$Register,
13598 $src1$$Register,
13599 zr,
13600 Assembler::GE);
13601 %}
13602 ins_pipe(icond_reg);
13603 %}
13604
13605 // This pattern is automatically generated from aarch64_ad.m4
13606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13607 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13608 %{
13609 match(Set dst (MinI src imm));
13610 ins_cost(INSN_COST * 3);
13611 expand %{
13612 rFlagsReg cr;
13613 compI_reg_imm0(cr, src);
13614 cmovI_reg_imm0_lt(dst, src, cr);
13615 %}
13616 %}
13617
13618 // This pattern is automatically generated from aarch64_ad.m4
13619 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13620 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13621 %{
13622 match(Set dst (MinI imm src));
13623 ins_cost(INSN_COST * 3);
13624 expand %{
13625 rFlagsReg cr;
13626 compI_reg_imm0(cr, src);
13627 cmovI_reg_imm0_lt(dst, src, cr);
13628 %}
13629 %}
13630
13631 // This pattern is automatically generated from aarch64_ad.m4
13632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13633 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13634 %{
13635 match(Set dst (MinI src imm));
13636 ins_cost(INSN_COST * 3);
13637 expand %{
13638 rFlagsReg cr;
13639 compI_reg_imm0(cr, src);
13640 cmovI_reg_imm1_le(dst, src, cr);
13641 %}
13642 %}
13643
13644 // This pattern is automatically generated from aarch64_ad.m4
13645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13646 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13647 %{
13648 match(Set dst (MinI imm src));
13649 ins_cost(INSN_COST * 3);
13650 expand %{
13651 rFlagsReg cr;
13652 compI_reg_imm0(cr, src);
13653 cmovI_reg_imm1_le(dst, src, cr);
13654 %}
13655 %}
13656
13657 // This pattern is automatically generated from aarch64_ad.m4
13658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13659 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13660 %{
13661 match(Set dst (MinI src imm));
13662 ins_cost(INSN_COST * 3);
13663 expand %{
13664 rFlagsReg cr;
13665 compI_reg_imm0(cr, src);
13666 cmovI_reg_immM1_lt(dst, src, cr);
13667 %}
13668 %}
13669
13670 // This pattern is automatically generated from aarch64_ad.m4
13671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13672 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13673 %{
13674 match(Set dst (MinI imm src));
13675 ins_cost(INSN_COST * 3);
13676 expand %{
13677 rFlagsReg cr;
13678 compI_reg_imm0(cr, src);
13679 cmovI_reg_immM1_lt(dst, src, cr);
13680 %}
13681 %}
13682
13683 // This pattern is automatically generated from aarch64_ad.m4
13684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13685 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13686 %{
13687 match(Set dst (MaxI src imm));
13688 ins_cost(INSN_COST * 3);
13689 expand %{
13690 rFlagsReg cr;
13691 compI_reg_imm0(cr, src);
13692 cmovI_reg_imm0_gt(dst, src, cr);
13693 %}
13694 %}
13695
13696 // This pattern is automatically generated from aarch64_ad.m4
13697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13698 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13699 %{
13700 match(Set dst (MaxI imm src));
13701 ins_cost(INSN_COST * 3);
13702 expand %{
13703 rFlagsReg cr;
13704 compI_reg_imm0(cr, src);
13705 cmovI_reg_imm0_gt(dst, src, cr);
13706 %}
13707 %}
13708
13709 // This pattern is automatically generated from aarch64_ad.m4
13710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13711 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13712 %{
13713 match(Set dst (MaxI src imm));
13714 ins_cost(INSN_COST * 3);
13715 expand %{
13716 rFlagsReg cr;
13717 compI_reg_imm0(cr, src);
13718 cmovI_reg_imm1_gt(dst, src, cr);
13719 %}
13720 %}
13721
13722 // This pattern is automatically generated from aarch64_ad.m4
13723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13724 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13725 %{
13726 match(Set dst (MaxI imm src));
13727 ins_cost(INSN_COST * 3);
13728 expand %{
13729 rFlagsReg cr;
13730 compI_reg_imm0(cr, src);
13731 cmovI_reg_imm1_gt(dst, src, cr);
13732 %}
13733 %}
13734
13735 // This pattern is automatically generated from aarch64_ad.m4
13736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13737 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13738 %{
13739 match(Set dst (MaxI src imm));
13740 ins_cost(INSN_COST * 3);
13741 expand %{
13742 rFlagsReg cr;
13743 compI_reg_imm0(cr, src);
13744 cmovI_reg_immM1_ge(dst, src, cr);
13745 %}
13746 %}
13747
13748 // This pattern is automatically generated from aarch64_ad.m4
13749 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13750 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13751 %{
13752 match(Set dst (MaxI imm src));
13753 ins_cost(INSN_COST * 3);
13754 expand %{
13755 rFlagsReg cr;
13756 compI_reg_imm0(cr, src);
13757 cmovI_reg_immM1_ge(dst, src, cr);
13758 %}
13759 %}
13760
13761 // This pattern is automatically generated from aarch64_ad.m4
13762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13763 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13764 %{
13765 match(Set dst (ReverseI src));
13766 ins_cost(INSN_COST);
13767 format %{ "rbitw $dst, $src" %}
13768 ins_encode %{
13769 __ rbitw($dst$$Register, $src$$Register);
13770 %}
13771 ins_pipe(ialu_reg);
13772 %}
13773
13774 // This pattern is automatically generated from aarch64_ad.m4
13775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13776 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13777 %{
13778 match(Set dst (ReverseL src));
13779 ins_cost(INSN_COST);
13780 format %{ "rbit $dst, $src" %}
13781 ins_encode %{
13782 __ rbit($dst$$Register, $src$$Register);
13783 %}
13784 ins_pipe(ialu_reg);
13785 %}
13786
13787
13788 // END This section of the file is automatically generated. Do not edit --------------
13789
13790
13791 // ============================================================================
13792 // Floating Point Arithmetic Instructions
13793
13794 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13795 match(Set dst (AddHF src1 src2));
13796 format %{ "faddh $dst, $src1, $src2" %}
13797 ins_encode %{
13798 __ faddh($dst$$FloatRegister,
13799 $src1$$FloatRegister,
13800 $src2$$FloatRegister);
13801 %}
13802 ins_pipe(fp_dop_reg_reg_s);
13803 %}
13804
13805 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13806 match(Set dst (AddF src1 src2));
13807
13808 ins_cost(INSN_COST * 5);
13809 format %{ "fadds $dst, $src1, $src2" %}
13810
13811 ins_encode %{
13812 __ fadds(as_FloatRegister($dst$$reg),
13813 as_FloatRegister($src1$$reg),
13814 as_FloatRegister($src2$$reg));
13815 %}
13816
13817 ins_pipe(fp_dop_reg_reg_s);
13818 %}
13819
13820 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13821 match(Set dst (AddD src1 src2));
13822
13823 ins_cost(INSN_COST * 5);
13824 format %{ "faddd $dst, $src1, $src2" %}
13825
13826 ins_encode %{
13827 __ faddd(as_FloatRegister($dst$$reg),
13828 as_FloatRegister($src1$$reg),
13829 as_FloatRegister($src2$$reg));
13830 %}
13831
13832 ins_pipe(fp_dop_reg_reg_d);
13833 %}
13834
13835 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13836 match(Set dst (SubHF src1 src2));
13837 format %{ "fsubh $dst, $src1, $src2" %}
13838 ins_encode %{
13839 __ fsubh($dst$$FloatRegister,
13840 $src1$$FloatRegister,
13841 $src2$$FloatRegister);
13842 %}
13843 ins_pipe(fp_dop_reg_reg_s);
13844 %}
13845
13846 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13847 match(Set dst (SubF src1 src2));
13848
13849 ins_cost(INSN_COST * 5);
13850 format %{ "fsubs $dst, $src1, $src2" %}
13851
13852 ins_encode %{
13853 __ fsubs(as_FloatRegister($dst$$reg),
13854 as_FloatRegister($src1$$reg),
13855 as_FloatRegister($src2$$reg));
13856 %}
13857
13858 ins_pipe(fp_dop_reg_reg_s);
13859 %}
13860
13861 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13862 match(Set dst (SubD src1 src2));
13863
13864 ins_cost(INSN_COST * 5);
13865 format %{ "fsubd $dst, $src1, $src2" %}
13866
13867 ins_encode %{
13868 __ fsubd(as_FloatRegister($dst$$reg),
13869 as_FloatRegister($src1$$reg),
13870 as_FloatRegister($src2$$reg));
13871 %}
13872
13873 ins_pipe(fp_dop_reg_reg_d);
13874 %}
13875
13876 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13877 match(Set dst (MulHF src1 src2));
13878 format %{ "fmulh $dst, $src1, $src2" %}
13879 ins_encode %{
13880 __ fmulh($dst$$FloatRegister,
13881 $src1$$FloatRegister,
13882 $src2$$FloatRegister);
13883 %}
13884 ins_pipe(fp_dop_reg_reg_s);
13885 %}
13886
13887 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13888 match(Set dst (MulF src1 src2));
13889
13890 ins_cost(INSN_COST * 6);
13891 format %{ "fmuls $dst, $src1, $src2" %}
13892
13893 ins_encode %{
13894 __ fmuls(as_FloatRegister($dst$$reg),
13895 as_FloatRegister($src1$$reg),
13896 as_FloatRegister($src2$$reg));
13897 %}
13898
13899 ins_pipe(fp_dop_reg_reg_s);
13900 %}
13901
13902 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13903 match(Set dst (MulD src1 src2));
13904
13905 ins_cost(INSN_COST * 6);
13906 format %{ "fmuld $dst, $src1, $src2" %}
13907
13908 ins_encode %{
13909 __ fmuld(as_FloatRegister($dst$$reg),
13910 as_FloatRegister($src1$$reg),
13911 as_FloatRegister($src2$$reg));
13912 %}
13913
13914 ins_pipe(fp_dop_reg_reg_d);
13915 %}
13916
13917 // src1 * src2 + src3 (half-precision float)
13918 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13919 match(Set dst (FmaHF src3 (Binary src1 src2)));
13920 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13921 ins_encode %{
13922 assert(UseFMA, "Needs FMA instructions support.");
13923 __ fmaddh($dst$$FloatRegister,
13924 $src1$$FloatRegister,
13925 $src2$$FloatRegister,
13926 $src3$$FloatRegister);
13927 %}
13928 ins_pipe(pipe_class_default);
13929 %}
13930
13931 // src1 * src2 + src3
13932 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13933 match(Set dst (FmaF src3 (Binary src1 src2)));
13934
13935 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13936
13937 ins_encode %{
13938 assert(UseFMA, "Needs FMA instructions support.");
13939 __ fmadds(as_FloatRegister($dst$$reg),
13940 as_FloatRegister($src1$$reg),
13941 as_FloatRegister($src2$$reg),
13942 as_FloatRegister($src3$$reg));
13943 %}
13944
13945 ins_pipe(pipe_class_default);
13946 %}
13947
13948 // src1 * src2 + src3
13949 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13950 match(Set dst (FmaD src3 (Binary src1 src2)));
13951
13952 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13953
13954 ins_encode %{
13955 assert(UseFMA, "Needs FMA instructions support.");
13956 __ fmaddd(as_FloatRegister($dst$$reg),
13957 as_FloatRegister($src1$$reg),
13958 as_FloatRegister($src2$$reg),
13959 as_FloatRegister($src3$$reg));
13960 %}
13961
13962 ins_pipe(pipe_class_default);
13963 %}
13964
13965 // src1 * (-src2) + src3
13966 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13967 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13968 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13969
13970 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13971
13972 ins_encode %{
13973 assert(UseFMA, "Needs FMA instructions support.");
13974 __ fmsubs(as_FloatRegister($dst$$reg),
13975 as_FloatRegister($src1$$reg),
13976 as_FloatRegister($src2$$reg),
13977 as_FloatRegister($src3$$reg));
13978 %}
13979
13980 ins_pipe(pipe_class_default);
13981 %}
13982
13983 // src1 * (-src2) + src3
13984 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13985 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13986 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13987
13988 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13989
13990 ins_encode %{
13991 assert(UseFMA, "Needs FMA instructions support.");
13992 __ fmsubd(as_FloatRegister($dst$$reg),
13993 as_FloatRegister($src1$$reg),
13994 as_FloatRegister($src2$$reg),
13995 as_FloatRegister($src3$$reg));
13996 %}
13997
13998 ins_pipe(pipe_class_default);
13999 %}
14000
14001 // src1 * (-src2) - src3
14002 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14003 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14004 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14005
14006 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14007
14008 ins_encode %{
14009 assert(UseFMA, "Needs FMA instructions support.");
14010 __ fnmadds(as_FloatRegister($dst$$reg),
14011 as_FloatRegister($src1$$reg),
14012 as_FloatRegister($src2$$reg),
14013 as_FloatRegister($src3$$reg));
14014 %}
14015
14016 ins_pipe(pipe_class_default);
14017 %}
14018
14019 // src1 * (-src2) - src3
14020 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14021 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14022 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14023
14024 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14025
14026 ins_encode %{
14027 assert(UseFMA, "Needs FMA instructions support.");
14028 __ fnmaddd(as_FloatRegister($dst$$reg),
14029 as_FloatRegister($src1$$reg),
14030 as_FloatRegister($src2$$reg),
14031 as_FloatRegister($src3$$reg));
14032 %}
14033
14034 ins_pipe(pipe_class_default);
14035 %}
14036
14037 // src1 * src2 - src3
14038 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14039 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14040
14041 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14042
14043 ins_encode %{
14044 assert(UseFMA, "Needs FMA instructions support.");
14045 __ fnmsubs(as_FloatRegister($dst$$reg),
14046 as_FloatRegister($src1$$reg),
14047 as_FloatRegister($src2$$reg),
14048 as_FloatRegister($src3$$reg));
14049 %}
14050
14051 ins_pipe(pipe_class_default);
14052 %}
14053
14054 // src1 * src2 - src3
14055 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14056 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14057
14058 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14059
14060 ins_encode %{
14061 assert(UseFMA, "Needs FMA instructions support.");
14062 // n.b. insn name should be fnmsubd
14063 __ fnmsub(as_FloatRegister($dst$$reg),
14064 as_FloatRegister($src1$$reg),
14065 as_FloatRegister($src2$$reg),
14066 as_FloatRegister($src3$$reg));
14067 %}
14068
14069 ins_pipe(pipe_class_default);
14070 %}
14071
14072 // Math.max(HH)H (half-precision float)
14073 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14074 match(Set dst (MaxHF src1 src2));
14075 format %{ "fmaxh $dst, $src1, $src2" %}
14076 ins_encode %{
14077 __ fmaxh($dst$$FloatRegister,
14078 $src1$$FloatRegister,
14079 $src2$$FloatRegister);
14080 %}
14081 ins_pipe(fp_dop_reg_reg_s);
14082 %}
14083
14084 // Math.min(HH)H (half-precision float)
14085 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14086 match(Set dst (MinHF src1 src2));
14087 format %{ "fminh $dst, $src1, $src2" %}
14088 ins_encode %{
14089 __ fminh($dst$$FloatRegister,
14090 $src1$$FloatRegister,
14091 $src2$$FloatRegister);
14092 %}
14093 ins_pipe(fp_dop_reg_reg_s);
14094 %}
14095
14096 // Math.max(FF)F
14097 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14098 match(Set dst (MaxF src1 src2));
14099
14100 format %{ "fmaxs $dst, $src1, $src2" %}
14101 ins_encode %{
14102 __ fmaxs(as_FloatRegister($dst$$reg),
14103 as_FloatRegister($src1$$reg),
14104 as_FloatRegister($src2$$reg));
14105 %}
14106
14107 ins_pipe(fp_dop_reg_reg_s);
14108 %}
14109
14110 // Math.min(FF)F
14111 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14112 match(Set dst (MinF src1 src2));
14113
14114 format %{ "fmins $dst, $src1, $src2" %}
14115 ins_encode %{
14116 __ fmins(as_FloatRegister($dst$$reg),
14117 as_FloatRegister($src1$$reg),
14118 as_FloatRegister($src2$$reg));
14119 %}
14120
14121 ins_pipe(fp_dop_reg_reg_s);
14122 %}
14123
14124 // Math.max(DD)D
14125 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14126 match(Set dst (MaxD src1 src2));
14127
14128 format %{ "fmaxd $dst, $src1, $src2" %}
14129 ins_encode %{
14130 __ fmaxd(as_FloatRegister($dst$$reg),
14131 as_FloatRegister($src1$$reg),
14132 as_FloatRegister($src2$$reg));
14133 %}
14134
14135 ins_pipe(fp_dop_reg_reg_d);
14136 %}
14137
14138 // Math.min(DD)D
14139 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14140 match(Set dst (MinD src1 src2));
14141
14142 format %{ "fmind $dst, $src1, $src2" %}
14143 ins_encode %{
14144 __ fmind(as_FloatRegister($dst$$reg),
14145 as_FloatRegister($src1$$reg),
14146 as_FloatRegister($src2$$reg));
14147 %}
14148
14149 ins_pipe(fp_dop_reg_reg_d);
14150 %}
14151
14152 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14153 match(Set dst (DivHF src1 src2));
14154 format %{ "fdivh $dst, $src1, $src2" %}
14155 ins_encode %{
14156 __ fdivh($dst$$FloatRegister,
14157 $src1$$FloatRegister,
14158 $src2$$FloatRegister);
14159 %}
14160 ins_pipe(fp_div_s);
14161 %}
14162
14163 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14164 match(Set dst (DivF src1 src2));
14165
14166 ins_cost(INSN_COST * 18);
14167 format %{ "fdivs $dst, $src1, $src2" %}
14168
14169 ins_encode %{
14170 __ fdivs(as_FloatRegister($dst$$reg),
14171 as_FloatRegister($src1$$reg),
14172 as_FloatRegister($src2$$reg));
14173 %}
14174
14175 ins_pipe(fp_div_s);
14176 %}
14177
14178 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14179 match(Set dst (DivD src1 src2));
14180
14181 ins_cost(INSN_COST * 32);
14182 format %{ "fdivd $dst, $src1, $src2" %}
14183
14184 ins_encode %{
14185 __ fdivd(as_FloatRegister($dst$$reg),
14186 as_FloatRegister($src1$$reg),
14187 as_FloatRegister($src2$$reg));
14188 %}
14189
14190 ins_pipe(fp_div_d);
14191 %}
14192
14193 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14194 match(Set dst (NegF src));
14195
14196 ins_cost(INSN_COST * 3);
14197 format %{ "fneg $dst, $src" %}
14198
14199 ins_encode %{
14200 __ fnegs(as_FloatRegister($dst$$reg),
14201 as_FloatRegister($src$$reg));
14202 %}
14203
14204 ins_pipe(fp_uop_s);
14205 %}
14206
14207 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14208 match(Set dst (NegD src));
14209
14210 ins_cost(INSN_COST * 3);
14211 format %{ "fnegd $dst, $src" %}
14212
14213 ins_encode %{
14214 __ fnegd(as_FloatRegister($dst$$reg),
14215 as_FloatRegister($src$$reg));
14216 %}
14217
14218 ins_pipe(fp_uop_d);
14219 %}
14220
14221 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14222 %{
14223 match(Set dst (AbsI src));
14224
14225 effect(KILL cr);
14226 ins_cost(INSN_COST * 2);
14227 format %{ "cmpw $src, zr\n\t"
14228 "cnegw $dst, $src, Assembler::LT\t# int abs"
14229 %}
14230
14231 ins_encode %{
14232 __ cmpw(as_Register($src$$reg), zr);
14233 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14234 %}
14235 ins_pipe(pipe_class_default);
14236 %}
14237
14238 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14239 %{
14240 match(Set dst (AbsL src));
14241
14242 effect(KILL cr);
14243 ins_cost(INSN_COST * 2);
14244 format %{ "cmp $src, zr\n\t"
14245 "cneg $dst, $src, Assembler::LT\t# long abs"
14246 %}
14247
14248 ins_encode %{
14249 __ cmp(as_Register($src$$reg), zr);
14250 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14251 %}
14252 ins_pipe(pipe_class_default);
14253 %}
14254
14255 instruct absF_reg(vRegF dst, vRegF src) %{
14256 match(Set dst (AbsF src));
14257
14258 ins_cost(INSN_COST * 3);
14259 format %{ "fabss $dst, $src" %}
14260 ins_encode %{
14261 __ fabss(as_FloatRegister($dst$$reg),
14262 as_FloatRegister($src$$reg));
14263 %}
14264
14265 ins_pipe(fp_uop_s);
14266 %}
14267
14268 instruct absD_reg(vRegD dst, vRegD src) %{
14269 match(Set dst (AbsD src));
14270
14271 ins_cost(INSN_COST * 3);
14272 format %{ "fabsd $dst, $src" %}
14273 ins_encode %{
14274 __ fabsd(as_FloatRegister($dst$$reg),
14275 as_FloatRegister($src$$reg));
14276 %}
14277
14278 ins_pipe(fp_uop_d);
14279 %}
14280
14281 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14282 match(Set dst (AbsF (SubF src1 src2)));
14283
14284 ins_cost(INSN_COST * 3);
14285 format %{ "fabds $dst, $src1, $src2" %}
14286 ins_encode %{
14287 __ fabds(as_FloatRegister($dst$$reg),
14288 as_FloatRegister($src1$$reg),
14289 as_FloatRegister($src2$$reg));
14290 %}
14291
14292 ins_pipe(fp_uop_s);
14293 %}
14294
14295 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14296 match(Set dst (AbsD (SubD src1 src2)));
14297
14298 ins_cost(INSN_COST * 3);
14299 format %{ "fabdd $dst, $src1, $src2" %}
14300 ins_encode %{
14301 __ fabdd(as_FloatRegister($dst$$reg),
14302 as_FloatRegister($src1$$reg),
14303 as_FloatRegister($src2$$reg));
14304 %}
14305
14306 ins_pipe(fp_uop_d);
14307 %}
14308
14309 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14310 match(Set dst (SqrtD src));
14311
14312 ins_cost(INSN_COST * 50);
14313 format %{ "fsqrtd $dst, $src" %}
14314 ins_encode %{
14315 __ fsqrtd(as_FloatRegister($dst$$reg),
14316 as_FloatRegister($src$$reg));
14317 %}
14318
14319 ins_pipe(fp_div_s);
14320 %}
14321
14322 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14323 match(Set dst (SqrtF src));
14324
14325 ins_cost(INSN_COST * 50);
14326 format %{ "fsqrts $dst, $src" %}
14327 ins_encode %{
14328 __ fsqrts(as_FloatRegister($dst$$reg),
14329 as_FloatRegister($src$$reg));
14330 %}
14331
14332 ins_pipe(fp_div_d);
14333 %}
14334
14335 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14336 match(Set dst (SqrtHF src));
14337 format %{ "fsqrth $dst, $src" %}
14338 ins_encode %{
14339 __ fsqrth($dst$$FloatRegister,
14340 $src$$FloatRegister);
14341 %}
14342 ins_pipe(fp_div_s);
14343 %}
14344
14345 // Math.rint, floor, ceil
14346 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14347 match(Set dst (RoundDoubleMode src rmode));
14348 format %{ "frint $dst, $src, $rmode" %}
14349 ins_encode %{
14350 switch ($rmode$$constant) {
14351 case RoundDoubleModeNode::rmode_rint:
14352 __ frintnd(as_FloatRegister($dst$$reg),
14353 as_FloatRegister($src$$reg));
14354 break;
14355 case RoundDoubleModeNode::rmode_floor:
14356 __ frintmd(as_FloatRegister($dst$$reg),
14357 as_FloatRegister($src$$reg));
14358 break;
14359 case RoundDoubleModeNode::rmode_ceil:
14360 __ frintpd(as_FloatRegister($dst$$reg),
14361 as_FloatRegister($src$$reg));
14362 break;
14363 }
14364 %}
14365 ins_pipe(fp_uop_d);
14366 %}
14367
14368 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14369 match(Set dst (CopySignD src1 (Binary src2 zero)));
14370 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14371 format %{ "CopySignD $dst $src1 $src2" %}
14372 ins_encode %{
14373 FloatRegister dst = as_FloatRegister($dst$$reg),
14374 src1 = as_FloatRegister($src1$$reg),
14375 src2 = as_FloatRegister($src2$$reg),
14376 zero = as_FloatRegister($zero$$reg);
14377 __ fnegd(dst, zero);
14378 __ bsl(dst, __ T8B, src2, src1);
14379 %}
14380 ins_pipe(fp_uop_d);
14381 %}
14382
14383 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14384 match(Set dst (CopySignF src1 src2));
14385 effect(TEMP_DEF dst, USE src1, USE src2);
14386 format %{ "CopySignF $dst $src1 $src2" %}
14387 ins_encode %{
14388 FloatRegister dst = as_FloatRegister($dst$$reg),
14389 src1 = as_FloatRegister($src1$$reg),
14390 src2 = as_FloatRegister($src2$$reg);
14391 __ movi(dst, __ T2S, 0x80, 24);
14392 __ bsl(dst, __ T8B, src2, src1);
14393 %}
14394 ins_pipe(fp_uop_d);
14395 %}
14396
14397 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14398 match(Set dst (SignumD src (Binary zero one)));
14399 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14400 format %{ "signumD $dst, $src" %}
14401 ins_encode %{
14402 FloatRegister src = as_FloatRegister($src$$reg),
14403 dst = as_FloatRegister($dst$$reg),
14404 zero = as_FloatRegister($zero$$reg),
14405 one = as_FloatRegister($one$$reg);
14406 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14407 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14408 // Bit selection instruction gets bit from "one" for each enabled bit in
14409 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14410 // NaN the whole "src" will be copied because "dst" is zero. For all other
14411 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14412 // from "src", and all other bits are copied from 1.0.
14413 __ bsl(dst, __ T8B, one, src);
14414 %}
14415 ins_pipe(fp_uop_d);
14416 %}
14417
14418 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14419 match(Set dst (SignumF src (Binary zero one)));
14420 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14421 format %{ "signumF $dst, $src" %}
14422 ins_encode %{
14423 FloatRegister src = as_FloatRegister($src$$reg),
14424 dst = as_FloatRegister($dst$$reg),
14425 zero = as_FloatRegister($zero$$reg),
14426 one = as_FloatRegister($one$$reg);
14427 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14428 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14429 // Bit selection instruction gets bit from "one" for each enabled bit in
14430 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14431 // NaN the whole "src" will be copied because "dst" is zero. For all other
14432 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14433 // from "src", and all other bits are copied from 1.0.
14434 __ bsl(dst, __ T8B, one, src);
14435 %}
14436 ins_pipe(fp_uop_d);
14437 %}
14438
14439 instruct onspinwait() %{
14440 match(OnSpinWait);
14441 ins_cost(INSN_COST);
14442
14443 format %{ "onspinwait" %}
14444
14445 ins_encode %{
14446 __ spin_wait();
14447 %}
14448 ins_pipe(pipe_class_empty);
14449 %}
14450
14451 // ============================================================================
14452 // Logical Instructions
14453
14454 // Integer Logical Instructions
14455
14456 // And Instructions
14457
14458
14459 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14460 match(Set dst (AndI src1 src2));
14461
14462 format %{ "andw $dst, $src1, $src2\t# int" %}
14463
14464 ins_cost(INSN_COST);
14465 ins_encode %{
14466 __ andw(as_Register($dst$$reg),
14467 as_Register($src1$$reg),
14468 as_Register($src2$$reg));
14469 %}
14470
14471 ins_pipe(ialu_reg_reg);
14472 %}
14473
14474 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14475 match(Set dst (AndI src1 src2));
14476
14477 format %{ "andsw $dst, $src1, $src2\t# int" %}
14478
14479 ins_cost(INSN_COST);
14480 ins_encode %{
14481 __ andw(as_Register($dst$$reg),
14482 as_Register($src1$$reg),
14483 (uint64_t)($src2$$constant));
14484 %}
14485
14486 ins_pipe(ialu_reg_imm);
14487 %}
14488
14489 // Or Instructions
14490
14491 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14492 match(Set dst (OrI src1 src2));
14493
14494 format %{ "orrw $dst, $src1, $src2\t# int" %}
14495
14496 ins_cost(INSN_COST);
14497 ins_encode %{
14498 __ orrw(as_Register($dst$$reg),
14499 as_Register($src1$$reg),
14500 as_Register($src2$$reg));
14501 %}
14502
14503 ins_pipe(ialu_reg_reg);
14504 %}
14505
14506 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14507 match(Set dst (OrI src1 src2));
14508
14509 format %{ "orrw $dst, $src1, $src2\t# int" %}
14510
14511 ins_cost(INSN_COST);
14512 ins_encode %{
14513 __ orrw(as_Register($dst$$reg),
14514 as_Register($src1$$reg),
14515 (uint64_t)($src2$$constant));
14516 %}
14517
14518 ins_pipe(ialu_reg_imm);
14519 %}
14520
14521 // Xor Instructions
14522
14523 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14524 match(Set dst (XorI src1 src2));
14525
14526 format %{ "eorw $dst, $src1, $src2\t# int" %}
14527
14528 ins_cost(INSN_COST);
14529 ins_encode %{
14530 __ eorw(as_Register($dst$$reg),
14531 as_Register($src1$$reg),
14532 as_Register($src2$$reg));
14533 %}
14534
14535 ins_pipe(ialu_reg_reg);
14536 %}
14537
14538 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14539 match(Set dst (XorI src1 src2));
14540
14541 format %{ "eorw $dst, $src1, $src2\t# int" %}
14542
14543 ins_cost(INSN_COST);
14544 ins_encode %{
14545 __ eorw(as_Register($dst$$reg),
14546 as_Register($src1$$reg),
14547 (uint64_t)($src2$$constant));
14548 %}
14549
14550 ins_pipe(ialu_reg_imm);
14551 %}
14552
14553 // Long Logical Instructions
14554 // TODO
14555
14556 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14557 match(Set dst (AndL src1 src2));
14558
14559 format %{ "and $dst, $src1, $src2\t# int" %}
14560
14561 ins_cost(INSN_COST);
14562 ins_encode %{
14563 __ andr(as_Register($dst$$reg),
14564 as_Register($src1$$reg),
14565 as_Register($src2$$reg));
14566 %}
14567
14568 ins_pipe(ialu_reg_reg);
14569 %}
14570
14571 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14572 match(Set dst (AndL src1 src2));
14573
14574 format %{ "and $dst, $src1, $src2\t# int" %}
14575
14576 ins_cost(INSN_COST);
14577 ins_encode %{
14578 __ andr(as_Register($dst$$reg),
14579 as_Register($src1$$reg),
14580 (uint64_t)($src2$$constant));
14581 %}
14582
14583 ins_pipe(ialu_reg_imm);
14584 %}
14585
14586 // Or Instructions
14587
14588 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14589 match(Set dst (OrL src1 src2));
14590
14591 format %{ "orr $dst, $src1, $src2\t# int" %}
14592
14593 ins_cost(INSN_COST);
14594 ins_encode %{
14595 __ orr(as_Register($dst$$reg),
14596 as_Register($src1$$reg),
14597 as_Register($src2$$reg));
14598 %}
14599
14600 ins_pipe(ialu_reg_reg);
14601 %}
14602
14603 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14604 match(Set dst (OrL src1 src2));
14605
14606 format %{ "orr $dst, $src1, $src2\t# int" %}
14607
14608 ins_cost(INSN_COST);
14609 ins_encode %{
14610 __ orr(as_Register($dst$$reg),
14611 as_Register($src1$$reg),
14612 (uint64_t)($src2$$constant));
14613 %}
14614
14615 ins_pipe(ialu_reg_imm);
14616 %}
14617
14618 // Xor Instructions
14619
14620 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14621 match(Set dst (XorL src1 src2));
14622
14623 format %{ "eor $dst, $src1, $src2\t# int" %}
14624
14625 ins_cost(INSN_COST);
14626 ins_encode %{
14627 __ eor(as_Register($dst$$reg),
14628 as_Register($src1$$reg),
14629 as_Register($src2$$reg));
14630 %}
14631
14632 ins_pipe(ialu_reg_reg);
14633 %}
14634
14635 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14636 match(Set dst (XorL src1 src2));
14637
14638 ins_cost(INSN_COST);
14639 format %{ "eor $dst, $src1, $src2\t# int" %}
14640
14641 ins_encode %{
14642 __ eor(as_Register($dst$$reg),
14643 as_Register($src1$$reg),
14644 (uint64_t)($src2$$constant));
14645 %}
14646
14647 ins_pipe(ialu_reg_imm);
14648 %}
14649
14650 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14651 %{
14652 match(Set dst (ConvI2L src));
14653
14654 ins_cost(INSN_COST);
14655 format %{ "sxtw $dst, $src\t# i2l" %}
14656 ins_encode %{
14657 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14658 %}
14659 ins_pipe(ialu_reg_shift);
14660 %}
14661
14662 // this pattern occurs in bigmath arithmetic
14663 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14664 %{
14665 match(Set dst (AndL (ConvI2L src) mask));
14666
14667 ins_cost(INSN_COST);
14668 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14669 ins_encode %{
14670 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14671 %}
14672
14673 ins_pipe(ialu_reg_shift);
14674 %}
14675
14676 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14677 match(Set dst (ConvL2I src));
14678
14679 ins_cost(INSN_COST);
14680 format %{ "movw $dst, $src \t// l2i" %}
14681
14682 ins_encode %{
14683 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14684 %}
14685
14686 ins_pipe(ialu_reg);
14687 %}
14688
14689 instruct convD2F_reg(vRegF dst, vRegD src) %{
14690 match(Set dst (ConvD2F src));
14691
14692 ins_cost(INSN_COST * 5);
14693 format %{ "fcvtd $dst, $src \t// d2f" %}
14694
14695 ins_encode %{
14696 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14697 %}
14698
14699 ins_pipe(fp_d2f);
14700 %}
14701
14702 instruct convF2D_reg(vRegD dst, vRegF src) %{
14703 match(Set dst (ConvF2D src));
14704
14705 ins_cost(INSN_COST * 5);
14706 format %{ "fcvts $dst, $src \t// f2d" %}
14707
14708 ins_encode %{
14709 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14710 %}
14711
14712 ins_pipe(fp_f2d);
14713 %}
14714
14715 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14716 match(Set dst (ConvF2I src));
14717
14718 ins_cost(INSN_COST * 5);
14719 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14720
14721 ins_encode %{
14722 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14723 %}
14724
14725 ins_pipe(fp_f2i);
14726 %}
14727
14728 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14729 match(Set dst (ConvF2L src));
14730
14731 ins_cost(INSN_COST * 5);
14732 format %{ "fcvtzs $dst, $src \t// f2l" %}
14733
14734 ins_encode %{
14735 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14736 %}
14737
14738 ins_pipe(fp_f2l);
14739 %}
14740
14741 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14742 match(Set dst (ConvF2HF src));
14743 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14744 "smov $dst, $tmp\t# move result from $tmp to $dst"
14745 %}
14746 effect(TEMP tmp);
14747 ins_encode %{
14748 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14749 %}
14750 ins_pipe(pipe_slow);
14751 %}
14752
14753 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14754 match(Set dst (ConvHF2F src));
14755 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14756 "fcvt $dst, $tmp\t# convert half to single precision"
14757 %}
14758 effect(TEMP tmp);
14759 ins_encode %{
14760 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14761 %}
14762 ins_pipe(pipe_slow);
14763 %}
14764
14765 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14766 match(Set dst (ConvI2F src));
14767
14768 ins_cost(INSN_COST * 5);
14769 format %{ "scvtfws $dst, $src \t// i2f" %}
14770
14771 ins_encode %{
14772 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14773 %}
14774
14775 ins_pipe(fp_i2f);
14776 %}
14777
14778 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14779 match(Set dst (ConvL2F src));
14780
14781 ins_cost(INSN_COST * 5);
14782 format %{ "scvtfs $dst, $src \t// l2f" %}
14783
14784 ins_encode %{
14785 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14786 %}
14787
14788 ins_pipe(fp_l2f);
14789 %}
14790
14791 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14792 match(Set dst (ConvD2I src));
14793
14794 ins_cost(INSN_COST * 5);
14795 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14796
14797 ins_encode %{
14798 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14799 %}
14800
14801 ins_pipe(fp_d2i);
14802 %}
14803
14804 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14805 match(Set dst (ConvD2L src));
14806
14807 ins_cost(INSN_COST * 5);
14808 format %{ "fcvtzd $dst, $src \t// d2l" %}
14809
14810 ins_encode %{
14811 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14812 %}
14813
14814 ins_pipe(fp_d2l);
14815 %}
14816
14817 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14818 match(Set dst (ConvI2D src));
14819
14820 ins_cost(INSN_COST * 5);
14821 format %{ "scvtfwd $dst, $src \t// i2d" %}
14822
14823 ins_encode %{
14824 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14825 %}
14826
14827 ins_pipe(fp_i2d);
14828 %}
14829
14830 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14831 match(Set dst (ConvL2D src));
14832
14833 ins_cost(INSN_COST * 5);
14834 format %{ "scvtfd $dst, $src \t// l2d" %}
14835
14836 ins_encode %{
14837 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14838 %}
14839
14840 ins_pipe(fp_l2d);
14841 %}
14842
14843 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14844 %{
14845 match(Set dst (RoundD src));
14846 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14847 format %{ "java_round_double $dst,$src"%}
14848 ins_encode %{
14849 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14850 as_FloatRegister($ftmp$$reg));
14851 %}
14852 ins_pipe(pipe_slow);
14853 %}
14854
14855 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14856 %{
14857 match(Set dst (RoundF src));
14858 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14859 format %{ "java_round_float $dst,$src"%}
14860 ins_encode %{
14861 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14862 as_FloatRegister($ftmp$$reg));
14863 %}
14864 ins_pipe(pipe_slow);
14865 %}
14866
14867 // stack <-> reg and reg <-> reg shuffles with no conversion
14868
14869 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14870
14871 match(Set dst (MoveF2I src));
14872
14873 effect(DEF dst, USE src);
14874
14875 ins_cost(4 * INSN_COST);
14876
14877 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14878
14879 ins_encode %{
14880 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14881 %}
14882
14883 ins_pipe(iload_reg_reg);
14884
14885 %}
14886
14887 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14888
14889 match(Set dst (MoveI2F src));
14890
14891 effect(DEF dst, USE src);
14892
14893 ins_cost(4 * INSN_COST);
14894
14895 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14896
14897 ins_encode %{
14898 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14899 %}
14900
14901 ins_pipe(pipe_class_memory);
14902
14903 %}
14904
14905 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14906
14907 match(Set dst (MoveD2L src));
14908
14909 effect(DEF dst, USE src);
14910
14911 ins_cost(4 * INSN_COST);
14912
14913 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14914
14915 ins_encode %{
14916 __ ldr($dst$$Register, Address(sp, $src$$disp));
14917 %}
14918
14919 ins_pipe(iload_reg_reg);
14920
14921 %}
14922
14923 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14924
14925 match(Set dst (MoveL2D src));
14926
14927 effect(DEF dst, USE src);
14928
14929 ins_cost(4 * INSN_COST);
14930
14931 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14932
14933 ins_encode %{
14934 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14935 %}
14936
14937 ins_pipe(pipe_class_memory);
14938
14939 %}
14940
14941 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14942
14943 match(Set dst (MoveF2I src));
14944
14945 effect(DEF dst, USE src);
14946
14947 ins_cost(INSN_COST);
14948
14949 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14950
14951 ins_encode %{
14952 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14953 %}
14954
14955 ins_pipe(pipe_class_memory);
14956
14957 %}
14958
14959 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14960
14961 match(Set dst (MoveI2F src));
14962
14963 effect(DEF dst, USE src);
14964
14965 ins_cost(INSN_COST);
14966
14967 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14968
14969 ins_encode %{
14970 __ strw($src$$Register, Address(sp, $dst$$disp));
14971 %}
14972
14973 ins_pipe(istore_reg_reg);
14974
14975 %}
14976
14977 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14978
14979 match(Set dst (MoveD2L src));
14980
14981 effect(DEF dst, USE src);
14982
14983 ins_cost(INSN_COST);
14984
14985 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14986
14987 ins_encode %{
14988 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14989 %}
14990
14991 ins_pipe(pipe_class_memory);
14992
14993 %}
14994
14995 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14996
14997 match(Set dst (MoveL2D src));
14998
14999 effect(DEF dst, USE src);
15000
15001 ins_cost(INSN_COST);
15002
15003 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15004
15005 ins_encode %{
15006 __ str($src$$Register, Address(sp, $dst$$disp));
15007 %}
15008
15009 ins_pipe(istore_reg_reg);
15010
15011 %}
15012
15013 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15014
15015 match(Set dst (MoveF2I src));
15016
15017 effect(DEF dst, USE src);
15018
15019 ins_cost(INSN_COST);
15020
15021 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15022
15023 ins_encode %{
15024 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15025 %}
15026
15027 ins_pipe(fp_f2i);
15028
15029 %}
15030
15031 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15032
15033 match(Set dst (MoveI2F src));
15034
15035 effect(DEF dst, USE src);
15036
15037 ins_cost(INSN_COST);
15038
15039 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15040
15041 ins_encode %{
15042 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15043 %}
15044
15045 ins_pipe(fp_i2f);
15046
15047 %}
15048
15049 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15050
15051 match(Set dst (MoveD2L src));
15052
15053 effect(DEF dst, USE src);
15054
15055 ins_cost(INSN_COST);
15056
15057 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15058
15059 ins_encode %{
15060 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15061 %}
15062
15063 ins_pipe(fp_d2l);
15064
15065 %}
15066
15067 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15068
15069 match(Set dst (MoveL2D src));
15070
15071 effect(DEF dst, USE src);
15072
15073 ins_cost(INSN_COST);
15074
15075 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15076
15077 ins_encode %{
15078 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15079 %}
15080
15081 ins_pipe(fp_l2d);
15082
15083 %}
15084
15085 // ============================================================================
15086 // clearing of an array
15087
15088 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15089 %{
15090 match(Set dummy (ClearArray cnt base));
15091 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15092
15093 ins_cost(4 * INSN_COST);
15094 format %{ "ClearArray $cnt, $base" %}
15095
15096 ins_encode %{
15097 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15098 if (tpc == nullptr) {
15099 ciEnv::current()->record_failure("CodeCache is full");
15100 return;
15101 }
15102 %}
15103
15104 ins_pipe(pipe_class_memory);
15105 %}
15106
15107 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15108 %{
15109 predicate((uint64_t)n->in(2)->get_long()
15110 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15111 match(Set dummy (ClearArray cnt base));
15112 effect(TEMP temp, USE_KILL base, KILL cr);
15113
15114 ins_cost(4 * INSN_COST);
15115 format %{ "ClearArray $cnt, $base" %}
15116
15117 ins_encode %{
15118 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15119 if (tpc == nullptr) {
15120 ciEnv::current()->record_failure("CodeCache is full");
15121 return;
15122 }
15123 %}
15124
15125 ins_pipe(pipe_class_memory);
15126 %}
15127
15128 // ============================================================================
15129 // Overflow Math Instructions
15130
15131 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15132 %{
15133 match(Set cr (OverflowAddI op1 op2));
15134
15135 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15136 ins_cost(INSN_COST);
15137 ins_encode %{
15138 __ cmnw($op1$$Register, $op2$$Register);
15139 %}
15140
15141 ins_pipe(icmp_reg_reg);
15142 %}
15143
15144 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15145 %{
15146 match(Set cr (OverflowAddI op1 op2));
15147
15148 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15149 ins_cost(INSN_COST);
15150 ins_encode %{
15151 __ cmnw($op1$$Register, $op2$$constant);
15152 %}
15153
15154 ins_pipe(icmp_reg_imm);
15155 %}
15156
15157 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15158 %{
15159 match(Set cr (OverflowAddL op1 op2));
15160
15161 format %{ "cmn $op1, $op2\t# overflow check long" %}
15162 ins_cost(INSN_COST);
15163 ins_encode %{
15164 __ cmn($op1$$Register, $op2$$Register);
15165 %}
15166
15167 ins_pipe(icmp_reg_reg);
15168 %}
15169
15170 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15171 %{
15172 match(Set cr (OverflowAddL op1 op2));
15173
15174 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15175 ins_cost(INSN_COST);
15176 ins_encode %{
15177 __ adds(zr, $op1$$Register, $op2$$constant);
15178 %}
15179
15180 ins_pipe(icmp_reg_imm);
15181 %}
15182
15183 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15184 %{
15185 match(Set cr (OverflowSubI op1 op2));
15186
15187 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15188 ins_cost(INSN_COST);
15189 ins_encode %{
15190 __ cmpw($op1$$Register, $op2$$Register);
15191 %}
15192
15193 ins_pipe(icmp_reg_reg);
15194 %}
15195
15196 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15197 %{
15198 match(Set cr (OverflowSubI op1 op2));
15199
15200 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15201 ins_cost(INSN_COST);
15202 ins_encode %{
15203 __ cmpw($op1$$Register, $op2$$constant);
15204 %}
15205
15206 ins_pipe(icmp_reg_imm);
15207 %}
15208
15209 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15210 %{
15211 match(Set cr (OverflowSubL op1 op2));
15212
15213 format %{ "cmp $op1, $op2\t# overflow check long" %}
15214 ins_cost(INSN_COST);
15215 ins_encode %{
15216 __ cmp($op1$$Register, $op2$$Register);
15217 %}
15218
15219 ins_pipe(icmp_reg_reg);
15220 %}
15221
15222 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15223 %{
15224 match(Set cr (OverflowSubL op1 op2));
15225
15226 format %{ "cmp $op1, $op2\t# overflow check long" %}
15227 ins_cost(INSN_COST);
15228 ins_encode %{
15229 __ subs(zr, $op1$$Register, $op2$$constant);
15230 %}
15231
15232 ins_pipe(icmp_reg_imm);
15233 %}
15234
15235 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15236 %{
15237 match(Set cr (OverflowSubI zero op1));
15238
15239 format %{ "cmpw zr, $op1\t# overflow check int" %}
15240 ins_cost(INSN_COST);
15241 ins_encode %{
15242 __ cmpw(zr, $op1$$Register);
15243 %}
15244
15245 ins_pipe(icmp_reg_imm);
15246 %}
15247
15248 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15249 %{
15250 match(Set cr (OverflowSubL zero op1));
15251
15252 format %{ "cmp zr, $op1\t# overflow check long" %}
15253 ins_cost(INSN_COST);
15254 ins_encode %{
15255 __ cmp(zr, $op1$$Register);
15256 %}
15257
15258 ins_pipe(icmp_reg_imm);
15259 %}
15260
15261 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15262 %{
15263 match(Set cr (OverflowMulI op1 op2));
15264
15265 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15266 "cmp rscratch1, rscratch1, sxtw\n\t"
15267 "movw rscratch1, #0x80000000\n\t"
15268 "cselw rscratch1, rscratch1, zr, NE\n\t"
15269 "cmpw rscratch1, #1" %}
15270 ins_cost(5 * INSN_COST);
15271 ins_encode %{
15272 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15273 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15274 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15275 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15276 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15277 %}
15278
15279 ins_pipe(pipe_slow);
15280 %}
15281
15282 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15283 %{
15284 match(If cmp (OverflowMulI op1 op2));
15285 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15286 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15287 effect(USE labl, KILL cr);
15288
15289 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15290 "cmp rscratch1, rscratch1, sxtw\n\t"
15291 "b$cmp $labl" %}
15292 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15293 ins_encode %{
15294 Label* L = $labl$$label;
15295 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15296 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15297 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15298 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15299 %}
15300
15301 ins_pipe(pipe_serial);
15302 %}
15303
15304 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15305 %{
15306 match(Set cr (OverflowMulL op1 op2));
15307
15308 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15309 "smulh rscratch2, $op1, $op2\n\t"
15310 "cmp rscratch2, rscratch1, ASR #63\n\t"
15311 "movw rscratch1, #0x80000000\n\t"
15312 "cselw rscratch1, rscratch1, zr, NE\n\t"
15313 "cmpw rscratch1, #1" %}
15314 ins_cost(6 * INSN_COST);
15315 ins_encode %{
15316 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15317 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15318 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15319 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15320 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15321 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15322 %}
15323
15324 ins_pipe(pipe_slow);
15325 %}
15326
15327 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15328 %{
15329 match(If cmp (OverflowMulL op1 op2));
15330 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15331 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15332 effect(USE labl, KILL cr);
15333
15334 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15335 "smulh rscratch2, $op1, $op2\n\t"
15336 "cmp rscratch2, rscratch1, ASR #63\n\t"
15337 "b$cmp $labl" %}
15338 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15339 ins_encode %{
15340 Label* L = $labl$$label;
15341 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15342 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15343 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15344 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15345 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15346 %}
15347
15348 ins_pipe(pipe_serial);
15349 %}
15350
15351 // ============================================================================
15352 // Compare Instructions
15353
15354 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15355 %{
15356 match(Set cr (CmpI op1 op2));
15357
15358 effect(DEF cr, USE op1, USE op2);
15359
15360 ins_cost(INSN_COST);
15361 format %{ "cmpw $op1, $op2" %}
15362
15363 ins_encode(aarch64_enc_cmpw(op1, op2));
15364
15365 ins_pipe(icmp_reg_reg);
15366 %}
15367
15368 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15369 %{
15370 match(Set cr (CmpI op1 zero));
15371
15372 effect(DEF cr, USE op1);
15373
15374 ins_cost(INSN_COST);
15375 format %{ "cmpw $op1, 0" %}
15376
15377 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15378
15379 ins_pipe(icmp_reg_imm);
15380 %}
15381
15382 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15383 %{
15384 match(Set cr (CmpI op1 op2));
15385
15386 effect(DEF cr, USE op1);
15387
15388 ins_cost(INSN_COST);
15389 format %{ "cmpw $op1, $op2" %}
15390
15391 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15392
15393 ins_pipe(icmp_reg_imm);
15394 %}
15395
15396 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15397 %{
15398 match(Set cr (CmpI op1 op2));
15399
15400 effect(DEF cr, USE op1);
15401
15402 ins_cost(INSN_COST * 2);
15403 format %{ "cmpw $op1, $op2" %}
15404
15405 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15406
15407 ins_pipe(icmp_reg_imm);
15408 %}
15409
15410 // Unsigned compare Instructions; really, same as signed compare
15411 // except it should only be used to feed an If or a CMovI which takes a
15412 // cmpOpU.
15413
15414 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15415 %{
15416 match(Set cr (CmpU op1 op2));
15417
15418 effect(DEF cr, USE op1, USE op2);
15419
15420 ins_cost(INSN_COST);
15421 format %{ "cmpw $op1, $op2\t# unsigned" %}
15422
15423 ins_encode(aarch64_enc_cmpw(op1, op2));
15424
15425 ins_pipe(icmp_reg_reg);
15426 %}
15427
15428 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15429 %{
15430 match(Set cr (CmpU op1 zero));
15431
15432 effect(DEF cr, USE op1);
15433
15434 ins_cost(INSN_COST);
15435 format %{ "cmpw $op1, #0\t# unsigned" %}
15436
15437 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15438
15439 ins_pipe(icmp_reg_imm);
15440 %}
15441
15442 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15443 %{
15444 match(Set cr (CmpU op1 op2));
15445
15446 effect(DEF cr, USE op1);
15447
15448 ins_cost(INSN_COST);
15449 format %{ "cmpw $op1, $op2\t# unsigned" %}
15450
15451 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15452
15453 ins_pipe(icmp_reg_imm);
15454 %}
15455
15456 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15457 %{
15458 match(Set cr (CmpU op1 op2));
15459
15460 effect(DEF cr, USE op1);
15461
15462 ins_cost(INSN_COST * 2);
15463 format %{ "cmpw $op1, $op2\t# unsigned" %}
15464
15465 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15466
15467 ins_pipe(icmp_reg_imm);
15468 %}
15469
15470 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15471 %{
15472 match(Set cr (CmpL op1 op2));
15473
15474 effect(DEF cr, USE op1, USE op2);
15475
15476 ins_cost(INSN_COST);
15477 format %{ "cmp $op1, $op2" %}
15478
15479 ins_encode(aarch64_enc_cmp(op1, op2));
15480
15481 ins_pipe(icmp_reg_reg);
15482 %}
15483
15484 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15485 %{
15486 match(Set cr (CmpL op1 zero));
15487
15488 effect(DEF cr, USE op1);
15489
15490 ins_cost(INSN_COST);
15491 format %{ "tst $op1" %}
15492
15493 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15494
15495 ins_pipe(icmp_reg_imm);
15496 %}
15497
15498 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15499 %{
15500 match(Set cr (CmpL op1 op2));
15501
15502 effect(DEF cr, USE op1);
15503
15504 ins_cost(INSN_COST);
15505 format %{ "cmp $op1, $op2" %}
15506
15507 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15508
15509 ins_pipe(icmp_reg_imm);
15510 %}
15511
15512 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15513 %{
15514 match(Set cr (CmpL op1 op2));
15515
15516 effect(DEF cr, USE op1);
15517
15518 ins_cost(INSN_COST * 2);
15519 format %{ "cmp $op1, $op2" %}
15520
15521 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15522
15523 ins_pipe(icmp_reg_imm);
15524 %}
15525
15526 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15527 %{
15528 match(Set cr (CmpUL op1 op2));
15529
15530 effect(DEF cr, USE op1, USE op2);
15531
15532 ins_cost(INSN_COST);
15533 format %{ "cmp $op1, $op2" %}
15534
15535 ins_encode(aarch64_enc_cmp(op1, op2));
15536
15537 ins_pipe(icmp_reg_reg);
15538 %}
15539
15540 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15541 %{
15542 match(Set cr (CmpUL op1 zero));
15543
15544 effect(DEF cr, USE op1);
15545
15546 ins_cost(INSN_COST);
15547 format %{ "tst $op1" %}
15548
15549 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15550
15551 ins_pipe(icmp_reg_imm);
15552 %}
15553
15554 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15555 %{
15556 match(Set cr (CmpUL op1 op2));
15557
15558 effect(DEF cr, USE op1);
15559
15560 ins_cost(INSN_COST);
15561 format %{ "cmp $op1, $op2" %}
15562
15563 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15564
15565 ins_pipe(icmp_reg_imm);
15566 %}
15567
15568 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15569 %{
15570 match(Set cr (CmpUL op1 op2));
15571
15572 effect(DEF cr, USE op1);
15573
15574 ins_cost(INSN_COST * 2);
15575 format %{ "cmp $op1, $op2" %}
15576
15577 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15578
15579 ins_pipe(icmp_reg_imm);
15580 %}
15581
15582 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15583 %{
15584 match(Set cr (CmpP op1 op2));
15585
15586 effect(DEF cr, USE op1, USE op2);
15587
15588 ins_cost(INSN_COST);
15589 format %{ "cmp $op1, $op2\t // ptr" %}
15590
15591 ins_encode(aarch64_enc_cmpp(op1, op2));
15592
15593 ins_pipe(icmp_reg_reg);
15594 %}
15595
15596 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15597 %{
15598 match(Set cr (CmpN op1 op2));
15599
15600 effect(DEF cr, USE op1, USE op2);
15601
15602 ins_cost(INSN_COST);
15603 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15604
15605 ins_encode(aarch64_enc_cmpn(op1, op2));
15606
15607 ins_pipe(icmp_reg_reg);
15608 %}
15609
15610 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15611 %{
15612 match(Set cr (CmpP op1 zero));
15613
15614 effect(DEF cr, USE op1, USE zero);
15615
15616 ins_cost(INSN_COST);
15617 format %{ "cmp $op1, 0\t // ptr" %}
15618
15619 ins_encode(aarch64_enc_testp(op1));
15620
15621 ins_pipe(icmp_reg_imm);
15622 %}
15623
15624 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15625 %{
15626 match(Set cr (CmpN op1 zero));
15627
15628 effect(DEF cr, USE op1, USE zero);
15629
15630 ins_cost(INSN_COST);
15631 format %{ "cmp $op1, 0\t // compressed ptr" %}
15632
15633 ins_encode(aarch64_enc_testn(op1));
15634
15635 ins_pipe(icmp_reg_imm);
15636 %}
15637
15638 // FP comparisons
15639 //
15640 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15641 // using normal cmpOp. See declaration of rFlagsReg for details.
15642
15643 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15644 %{
15645 match(Set cr (CmpF src1 src2));
15646
15647 ins_cost(3 * INSN_COST);
15648 format %{ "fcmps $src1, $src2" %}
15649
15650 ins_encode %{
15651 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15652 %}
15653
15654 ins_pipe(pipe_class_compare);
15655 %}
15656
15657 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15658 %{
15659 match(Set cr (CmpF src1 src2));
15660
15661 ins_cost(3 * INSN_COST);
15662 format %{ "fcmps $src1, 0.0" %}
15663
15664 ins_encode %{
15665 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15666 %}
15667
15668 ins_pipe(pipe_class_compare);
15669 %}
15670 // FROM HERE
15671
15672 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15673 %{
15674 match(Set cr (CmpD src1 src2));
15675
15676 ins_cost(3 * INSN_COST);
15677 format %{ "fcmpd $src1, $src2" %}
15678
15679 ins_encode %{
15680 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15681 %}
15682
15683 ins_pipe(pipe_class_compare);
15684 %}
15685
15686 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15687 %{
15688 match(Set cr (CmpD src1 src2));
15689
15690 ins_cost(3 * INSN_COST);
15691 format %{ "fcmpd $src1, 0.0" %}
15692
15693 ins_encode %{
15694 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15695 %}
15696
15697 ins_pipe(pipe_class_compare);
15698 %}
15699
15700 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15701 %{
15702 match(Set dst (CmpF3 src1 src2));
15703 effect(KILL cr);
15704
15705 ins_cost(5 * INSN_COST);
15706 format %{ "fcmps $src1, $src2\n\t"
15707 "csinvw($dst, zr, zr, eq\n\t"
15708 "csnegw($dst, $dst, $dst, lt)"
15709 %}
15710
15711 ins_encode %{
15712 Label done;
15713 FloatRegister s1 = as_FloatRegister($src1$$reg);
15714 FloatRegister s2 = as_FloatRegister($src2$$reg);
15715 Register d = as_Register($dst$$reg);
15716 __ fcmps(s1, s2);
15717 // installs 0 if EQ else -1
15718 __ csinvw(d, zr, zr, Assembler::EQ);
15719 // keeps -1 if less or unordered else installs 1
15720 __ csnegw(d, d, d, Assembler::LT);
15721 __ bind(done);
15722 %}
15723
15724 ins_pipe(pipe_class_default);
15725
15726 %}
15727
15728 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15729 %{
15730 match(Set dst (CmpD3 src1 src2));
15731 effect(KILL cr);
15732
15733 ins_cost(5 * INSN_COST);
15734 format %{ "fcmpd $src1, $src2\n\t"
15735 "csinvw($dst, zr, zr, eq\n\t"
15736 "csnegw($dst, $dst, $dst, lt)"
15737 %}
15738
15739 ins_encode %{
15740 Label done;
15741 FloatRegister s1 = as_FloatRegister($src1$$reg);
15742 FloatRegister s2 = as_FloatRegister($src2$$reg);
15743 Register d = as_Register($dst$$reg);
15744 __ fcmpd(s1, s2);
15745 // installs 0 if EQ else -1
15746 __ csinvw(d, zr, zr, Assembler::EQ);
15747 // keeps -1 if less or unordered else installs 1
15748 __ csnegw(d, d, d, Assembler::LT);
15749 __ bind(done);
15750 %}
15751 ins_pipe(pipe_class_default);
15752
15753 %}
15754
15755 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15756 %{
15757 match(Set dst (CmpF3 src1 zero));
15758 effect(KILL cr);
15759
15760 ins_cost(5 * INSN_COST);
15761 format %{ "fcmps $src1, 0.0\n\t"
15762 "csinvw($dst, zr, zr, eq\n\t"
15763 "csnegw($dst, $dst, $dst, lt)"
15764 %}
15765
15766 ins_encode %{
15767 Label done;
15768 FloatRegister s1 = as_FloatRegister($src1$$reg);
15769 Register d = as_Register($dst$$reg);
15770 __ fcmps(s1, 0.0);
15771 // installs 0 if EQ else -1
15772 __ csinvw(d, zr, zr, Assembler::EQ);
15773 // keeps -1 if less or unordered else installs 1
15774 __ csnegw(d, d, d, Assembler::LT);
15775 __ bind(done);
15776 %}
15777
15778 ins_pipe(pipe_class_default);
15779
15780 %}
15781
15782 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15783 %{
15784 match(Set dst (CmpD3 src1 zero));
15785 effect(KILL cr);
15786
15787 ins_cost(5 * INSN_COST);
15788 format %{ "fcmpd $src1, 0.0\n\t"
15789 "csinvw($dst, zr, zr, eq\n\t"
15790 "csnegw($dst, $dst, $dst, lt)"
15791 %}
15792
15793 ins_encode %{
15794 Label done;
15795 FloatRegister s1 = as_FloatRegister($src1$$reg);
15796 Register d = as_Register($dst$$reg);
15797 __ fcmpd(s1, 0.0);
15798 // installs 0 if EQ else -1
15799 __ csinvw(d, zr, zr, Assembler::EQ);
15800 // keeps -1 if less or unordered else installs 1
15801 __ csnegw(d, d, d, Assembler::LT);
15802 __ bind(done);
15803 %}
15804 ins_pipe(pipe_class_default);
15805
15806 %}
15807
15808 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15809 %{
15810 match(Set dst (CmpLTMask p q));
15811 effect(KILL cr);
15812
15813 ins_cost(3 * INSN_COST);
15814
15815 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15816 "csetw $dst, lt\n\t"
15817 "subw $dst, zr, $dst"
15818 %}
15819
15820 ins_encode %{
15821 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15822 __ csetw(as_Register($dst$$reg), Assembler::LT);
15823 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15824 %}
15825
15826 ins_pipe(ialu_reg_reg);
15827 %}
15828
15829 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15830 %{
15831 match(Set dst (CmpLTMask src zero));
15832 effect(KILL cr);
15833
15834 ins_cost(INSN_COST);
15835
15836 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15837
15838 ins_encode %{
15839 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15840 %}
15841
15842 ins_pipe(ialu_reg_shift);
15843 %}
15844
15845 // ============================================================================
15846 // Max and Min
15847
15848 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15849
15850 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15851 %{
15852 effect(DEF cr, USE src);
15853 ins_cost(INSN_COST);
15854 format %{ "cmpw $src, 0" %}
15855
15856 ins_encode %{
15857 __ cmpw($src$$Register, 0);
15858 %}
15859 ins_pipe(icmp_reg_imm);
15860 %}
15861
15862 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15863 %{
15864 match(Set dst (MinI src1 src2));
15865 ins_cost(INSN_COST * 3);
15866
15867 expand %{
15868 rFlagsReg cr;
15869 compI_reg_reg(cr, src1, src2);
15870 cmovI_reg_reg_lt(dst, src1, src2, cr);
15871 %}
15872 %}
15873
15874 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15875 %{
15876 match(Set dst (MaxI src1 src2));
15877 ins_cost(INSN_COST * 3);
15878
15879 expand %{
15880 rFlagsReg cr;
15881 compI_reg_reg(cr, src1, src2);
15882 cmovI_reg_reg_gt(dst, src1, src2, cr);
15883 %}
15884 %}
15885
15886
15887 // ============================================================================
15888 // Branch Instructions
15889
15890 // Direct Branch.
15891 instruct branch(label lbl)
15892 %{
15893 match(Goto);
15894
15895 effect(USE lbl);
15896
15897 ins_cost(BRANCH_COST);
15898 format %{ "b $lbl" %}
15899
15900 ins_encode(aarch64_enc_b(lbl));
15901
15902 ins_pipe(pipe_branch);
15903 %}
15904
15905 // Conditional Near Branch
15906 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15907 %{
15908 // Same match rule as `branchConFar'.
15909 match(If cmp cr);
15910
15911 effect(USE lbl);
15912
15913 ins_cost(BRANCH_COST);
15914 // If set to 1 this indicates that the current instruction is a
15915 // short variant of a long branch. This avoids using this
15916 // instruction in first-pass matching. It will then only be used in
15917 // the `Shorten_branches' pass.
15918 // ins_short_branch(1);
15919 format %{ "b$cmp $lbl" %}
15920
15921 ins_encode(aarch64_enc_br_con(cmp, lbl));
15922
15923 ins_pipe(pipe_branch_cond);
15924 %}
15925
15926 // Conditional Near Branch Unsigned
15927 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15928 %{
15929 // Same match rule as `branchConFar'.
15930 match(If cmp cr);
15931
15932 effect(USE lbl);
15933
15934 ins_cost(BRANCH_COST);
15935 // If set to 1 this indicates that the current instruction is a
15936 // short variant of a long branch. This avoids using this
15937 // instruction in first-pass matching. It will then only be used in
15938 // the `Shorten_branches' pass.
15939 // ins_short_branch(1);
15940 format %{ "b$cmp $lbl\t# unsigned" %}
15941
15942 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15943
15944 ins_pipe(pipe_branch_cond);
15945 %}
15946
15947 // Make use of CBZ and CBNZ. These instructions, as well as being
15948 // shorter than (cmp; branch), have the additional benefit of not
15949 // killing the flags.
15950
15951 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15952 match(If cmp (CmpI op1 op2));
15953 effect(USE labl);
15954
15955 ins_cost(BRANCH_COST);
15956 format %{ "cbw$cmp $op1, $labl" %}
15957 ins_encode %{
15958 Label* L = $labl$$label;
15959 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15960 if (cond == Assembler::EQ)
15961 __ cbzw($op1$$Register, *L);
15962 else
15963 __ cbnzw($op1$$Register, *L);
15964 %}
15965 ins_pipe(pipe_cmp_branch);
15966 %}
15967
15968 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15969 match(If cmp (CmpL op1 op2));
15970 effect(USE labl);
15971
15972 ins_cost(BRANCH_COST);
15973 format %{ "cb$cmp $op1, $labl" %}
15974 ins_encode %{
15975 Label* L = $labl$$label;
15976 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15977 if (cond == Assembler::EQ)
15978 __ cbz($op1$$Register, *L);
15979 else
15980 __ cbnz($op1$$Register, *L);
15981 %}
15982 ins_pipe(pipe_cmp_branch);
15983 %}
15984
15985 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15986 match(If cmp (CmpP op1 op2));
15987 effect(USE labl);
15988
15989 ins_cost(BRANCH_COST);
15990 format %{ "cb$cmp $op1, $labl" %}
15991 ins_encode %{
15992 Label* L = $labl$$label;
15993 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15994 if (cond == Assembler::EQ)
15995 __ cbz($op1$$Register, *L);
15996 else
15997 __ cbnz($op1$$Register, *L);
15998 %}
15999 ins_pipe(pipe_cmp_branch);
16000 %}
16001
16002 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16003 match(If cmp (CmpN op1 op2));
16004 effect(USE labl);
16005
16006 ins_cost(BRANCH_COST);
16007 format %{ "cbw$cmp $op1, $labl" %}
16008 ins_encode %{
16009 Label* L = $labl$$label;
16010 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16011 if (cond == Assembler::EQ)
16012 __ cbzw($op1$$Register, *L);
16013 else
16014 __ cbnzw($op1$$Register, *L);
16015 %}
16016 ins_pipe(pipe_cmp_branch);
16017 %}
16018
16019 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16020 match(If cmp (CmpP (DecodeN oop) zero));
16021 effect(USE labl);
16022
16023 ins_cost(BRANCH_COST);
16024 format %{ "cb$cmp $oop, $labl" %}
16025 ins_encode %{
16026 Label* L = $labl$$label;
16027 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16028 if (cond == Assembler::EQ)
16029 __ cbzw($oop$$Register, *L);
16030 else
16031 __ cbnzw($oop$$Register, *L);
16032 %}
16033 ins_pipe(pipe_cmp_branch);
16034 %}
16035
16036 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16037 match(If cmp (CmpU op1 op2));
16038 effect(USE labl);
16039
16040 ins_cost(BRANCH_COST);
16041 format %{ "cbw$cmp $op1, $labl" %}
16042 ins_encode %{
16043 Label* L = $labl$$label;
16044 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16045 if (cond == Assembler::EQ || cond == Assembler::LS) {
16046 __ cbzw($op1$$Register, *L);
16047 } else {
16048 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16049 __ cbnzw($op1$$Register, *L);
16050 }
16051 %}
16052 ins_pipe(pipe_cmp_branch);
16053 %}
16054
16055 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16056 match(If cmp (CmpUL op1 op2));
16057 effect(USE labl);
16058
16059 ins_cost(BRANCH_COST);
16060 format %{ "cb$cmp $op1, $labl" %}
16061 ins_encode %{
16062 Label* L = $labl$$label;
16063 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16064 if (cond == Assembler::EQ || cond == Assembler::LS) {
16065 __ cbz($op1$$Register, *L);
16066 } else {
16067 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16068 __ cbnz($op1$$Register, *L);
16069 }
16070 %}
16071 ins_pipe(pipe_cmp_branch);
16072 %}
16073
16074 // Test bit and Branch
16075
16076 // Patterns for short (< 32KiB) variants
16077 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16078 match(If cmp (CmpL op1 op2));
16079 effect(USE labl);
16080
16081 ins_cost(BRANCH_COST);
16082 format %{ "cb$cmp $op1, $labl # long" %}
16083 ins_encode %{
16084 Label* L = $labl$$label;
16085 Assembler::Condition cond =
16086 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16087 __ tbr(cond, $op1$$Register, 63, *L);
16088 %}
16089 ins_pipe(pipe_cmp_branch);
16090 ins_short_branch(1);
16091 %}
16092
16093 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16094 match(If cmp (CmpI op1 op2));
16095 effect(USE labl);
16096
16097 ins_cost(BRANCH_COST);
16098 format %{ "cb$cmp $op1, $labl # int" %}
16099 ins_encode %{
16100 Label* L = $labl$$label;
16101 Assembler::Condition cond =
16102 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16103 __ tbr(cond, $op1$$Register, 31, *L);
16104 %}
16105 ins_pipe(pipe_cmp_branch);
16106 ins_short_branch(1);
16107 %}
16108
16109 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16110 match(If cmp (CmpL (AndL op1 op2) op3));
16111 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16112 effect(USE labl);
16113
16114 ins_cost(BRANCH_COST);
16115 format %{ "tb$cmp $op1, $op2, $labl" %}
16116 ins_encode %{
16117 Label* L = $labl$$label;
16118 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16119 int bit = exact_log2_long($op2$$constant);
16120 __ tbr(cond, $op1$$Register, bit, *L);
16121 %}
16122 ins_pipe(pipe_cmp_branch);
16123 ins_short_branch(1);
16124 %}
16125
16126 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16127 match(If cmp (CmpI (AndI op1 op2) op3));
16128 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16129 effect(USE labl);
16130
16131 ins_cost(BRANCH_COST);
16132 format %{ "tb$cmp $op1, $op2, $labl" %}
16133 ins_encode %{
16134 Label* L = $labl$$label;
16135 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16136 int bit = exact_log2((juint)$op2$$constant);
16137 __ tbr(cond, $op1$$Register, bit, *L);
16138 %}
16139 ins_pipe(pipe_cmp_branch);
16140 ins_short_branch(1);
16141 %}
16142
16143 // And far variants
16144 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16145 match(If cmp (CmpL op1 op2));
16146 effect(USE labl);
16147
16148 ins_cost(BRANCH_COST);
16149 format %{ "cb$cmp $op1, $labl # long" %}
16150 ins_encode %{
16151 Label* L = $labl$$label;
16152 Assembler::Condition cond =
16153 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16154 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16155 %}
16156 ins_pipe(pipe_cmp_branch);
16157 %}
16158
16159 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16160 match(If cmp (CmpI op1 op2));
16161 effect(USE labl);
16162
16163 ins_cost(BRANCH_COST);
16164 format %{ "cb$cmp $op1, $labl # int" %}
16165 ins_encode %{
16166 Label* L = $labl$$label;
16167 Assembler::Condition cond =
16168 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16169 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16170 %}
16171 ins_pipe(pipe_cmp_branch);
16172 %}
16173
16174 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16175 match(If cmp (CmpL (AndL op1 op2) op3));
16176 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16177 effect(USE labl);
16178
16179 ins_cost(BRANCH_COST);
16180 format %{ "tb$cmp $op1, $op2, $labl" %}
16181 ins_encode %{
16182 Label* L = $labl$$label;
16183 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16184 int bit = exact_log2_long($op2$$constant);
16185 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16186 %}
16187 ins_pipe(pipe_cmp_branch);
16188 %}
16189
16190 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16191 match(If cmp (CmpI (AndI op1 op2) op3));
16192 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16193 effect(USE labl);
16194
16195 ins_cost(BRANCH_COST);
16196 format %{ "tb$cmp $op1, $op2, $labl" %}
16197 ins_encode %{
16198 Label* L = $labl$$label;
16199 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16200 int bit = exact_log2((juint)$op2$$constant);
16201 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16202 %}
16203 ins_pipe(pipe_cmp_branch);
16204 %}
16205
16206 // Test bits
16207
16208 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16209 match(Set cr (CmpL (AndL op1 op2) op3));
16210 predicate(Assembler::operand_valid_for_logical_immediate
16211 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16212
16213 ins_cost(INSN_COST);
16214 format %{ "tst $op1, $op2 # long" %}
16215 ins_encode %{
16216 __ tst($op1$$Register, $op2$$constant);
16217 %}
16218 ins_pipe(ialu_reg_reg);
16219 %}
16220
16221 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16222 match(Set cr (CmpI (AndI op1 op2) op3));
16223 predicate(Assembler::operand_valid_for_logical_immediate
16224 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16225
16226 ins_cost(INSN_COST);
16227 format %{ "tst $op1, $op2 # int" %}
16228 ins_encode %{
16229 __ tstw($op1$$Register, $op2$$constant);
16230 %}
16231 ins_pipe(ialu_reg_reg);
16232 %}
16233
16234 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16235 match(Set cr (CmpL (AndL op1 op2) op3));
16236
16237 ins_cost(INSN_COST);
16238 format %{ "tst $op1, $op2 # long" %}
16239 ins_encode %{
16240 __ tst($op1$$Register, $op2$$Register);
16241 %}
16242 ins_pipe(ialu_reg_reg);
16243 %}
16244
16245 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16246 match(Set cr (CmpI (AndI op1 op2) op3));
16247
16248 ins_cost(INSN_COST);
16249 format %{ "tstw $op1, $op2 # int" %}
16250 ins_encode %{
16251 __ tstw($op1$$Register, $op2$$Register);
16252 %}
16253 ins_pipe(ialu_reg_reg);
16254 %}
16255
16256
16257 // Conditional Far Branch
16258 // Conditional Far Branch Unsigned
16259 // TODO: fixme
16260
16261 // counted loop end branch near
16262 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16263 %{
16264 match(CountedLoopEnd cmp cr);
16265
16266 effect(USE lbl);
16267
16268 ins_cost(BRANCH_COST);
16269 // short variant.
16270 // ins_short_branch(1);
16271 format %{ "b$cmp $lbl \t// counted loop end" %}
16272
16273 ins_encode(aarch64_enc_br_con(cmp, lbl));
16274
16275 ins_pipe(pipe_branch);
16276 %}
16277
16278 // counted loop end branch far
16279 // TODO: fixme
16280
16281 // ============================================================================
16282 // inlined locking and unlocking
16283
16284 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16285 %{
16286 match(Set cr (FastLock object box));
16287 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16288
16289 ins_cost(5 * INSN_COST);
16290 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16291
16292 ins_encode %{
16293 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16294 %}
16295
16296 ins_pipe(pipe_serial);
16297 %}
16298
16299 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16300 %{
16301 match(Set cr (FastUnlock object box));
16302 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16303
16304 ins_cost(5 * INSN_COST);
16305 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16306
16307 ins_encode %{
16308 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16309 %}
16310
16311 ins_pipe(pipe_serial);
16312 %}
16313
16314 // ============================================================================
16315 // Safepoint Instructions
16316
16317 // TODO
16318 // provide a near and far version of this code
16319
16320 instruct safePoint(rFlagsReg cr, iRegP poll)
16321 %{
16322 match(SafePoint poll);
16323 effect(KILL cr);
16324
16325 format %{
16326 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16327 %}
16328 ins_encode %{
16329 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16330 %}
16331 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16332 %}
16333
16334
16335 // ============================================================================
16336 // Procedure Call/Return Instructions
16337
16338 // Call Java Static Instruction
16339
16340 instruct CallStaticJavaDirect(method meth)
16341 %{
16342 match(CallStaticJava);
16343
16344 effect(USE meth);
16345
16346 ins_cost(CALL_COST);
16347
16348 format %{ "call,static $meth \t// ==> " %}
16349
16350 ins_encode(aarch64_enc_java_static_call(meth),
16351 aarch64_enc_call_epilog);
16352
16353 ins_pipe(pipe_class_call);
16354 %}
16355
16356 // TO HERE
16357
16358 // Call Java Dynamic Instruction
16359 instruct CallDynamicJavaDirect(method meth)
16360 %{
16361 match(CallDynamicJava);
16362
16363 effect(USE meth);
16364
16365 ins_cost(CALL_COST);
16366
16367 format %{ "CALL,dynamic $meth \t// ==> " %}
16368
16369 ins_encode(aarch64_enc_java_dynamic_call(meth),
16370 aarch64_enc_call_epilog);
16371
16372 ins_pipe(pipe_class_call);
16373 %}
16374
16375 // Call Runtime Instruction
16376
16377 instruct CallRuntimeDirect(method meth)
16378 %{
16379 match(CallRuntime);
16380
16381 effect(USE meth);
16382
16383 ins_cost(CALL_COST);
16384
16385 format %{ "CALL, runtime $meth" %}
16386
16387 ins_encode( aarch64_enc_java_to_runtime(meth) );
16388
16389 ins_pipe(pipe_class_call);
16390 %}
16391
16392 // Call Runtime Instruction
16393
16394 instruct CallLeafDirect(method meth)
16395 %{
16396 match(CallLeaf);
16397
16398 effect(USE meth);
16399
16400 ins_cost(CALL_COST);
16401
16402 format %{ "CALL, runtime leaf $meth" %}
16403
16404 ins_encode( aarch64_enc_java_to_runtime(meth) );
16405
16406 ins_pipe(pipe_class_call);
16407 %}
16408
16409 // Call Runtime Instruction without safepoint and with vector arguments
16410 instruct CallLeafDirectVector(method meth)
16411 %{
16412 match(CallLeafVector);
16413
16414 effect(USE meth);
16415
16416 ins_cost(CALL_COST);
16417
16418 format %{ "CALL, runtime leaf vector $meth" %}
16419
16420 ins_encode(aarch64_enc_java_to_runtime(meth));
16421
16422 ins_pipe(pipe_class_call);
16423 %}
16424
16425 // Call Runtime Instruction
16426
16427 instruct CallLeafNoFPDirect(method meth)
16428 %{
16429 match(CallLeafNoFP);
16430
16431 effect(USE meth);
16432
16433 ins_cost(CALL_COST);
16434
16435 format %{ "CALL, runtime leaf nofp $meth" %}
16436
16437 ins_encode( aarch64_enc_java_to_runtime(meth) );
16438
16439 ins_pipe(pipe_class_call);
16440 %}
16441
16442 // Tail Call; Jump from runtime stub to Java code.
16443 // Also known as an 'interprocedural jump'.
16444 // Target of jump will eventually return to caller.
16445 // TailJump below removes the return address.
16446 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16447 // emitted just above the TailCall which has reset rfp to the caller state.
16448 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16449 %{
16450 match(TailCall jump_target method_ptr);
16451
16452 ins_cost(CALL_COST);
16453
16454 format %{ "br $jump_target\t# $method_ptr holds method" %}
16455
16456 ins_encode(aarch64_enc_tail_call(jump_target));
16457
16458 ins_pipe(pipe_class_call);
16459 %}
16460
16461 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16462 %{
16463 match(TailJump jump_target ex_oop);
16464
16465 ins_cost(CALL_COST);
16466
16467 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16468
16469 ins_encode(aarch64_enc_tail_jmp(jump_target));
16470
16471 ins_pipe(pipe_class_call);
16472 %}
16473
16474 // Forward exception.
16475 instruct ForwardExceptionjmp()
16476 %{
16477 match(ForwardException);
16478 ins_cost(CALL_COST);
16479
16480 format %{ "b forward_exception_stub" %}
16481 ins_encode %{
16482 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16483 %}
16484 ins_pipe(pipe_class_call);
16485 %}
16486
16487 // Create exception oop: created by stack-crawling runtime code.
16488 // Created exception is now available to this handler, and is setup
16489 // just prior to jumping to this handler. No code emitted.
16490 // TODO check
16491 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16492 instruct CreateException(iRegP_R0 ex_oop)
16493 %{
16494 match(Set ex_oop (CreateEx));
16495
16496 format %{ " -- \t// exception oop; no code emitted" %}
16497
16498 size(0);
16499
16500 ins_encode( /*empty*/ );
16501
16502 ins_pipe(pipe_class_empty);
16503 %}
16504
16505 // Rethrow exception: The exception oop will come in the first
16506 // argument position. Then JUMP (not call) to the rethrow stub code.
16507 instruct RethrowException() %{
16508 match(Rethrow);
16509 ins_cost(CALL_COST);
16510
16511 format %{ "b rethrow_stub" %}
16512
16513 ins_encode( aarch64_enc_rethrow() );
16514
16515 ins_pipe(pipe_class_call);
16516 %}
16517
16518
16519 // Return Instruction
16520 // epilog node loads ret address into lr as part of frame pop
16521 instruct Ret()
16522 %{
16523 match(Return);
16524
16525 format %{ "ret\t// return register" %}
16526
16527 ins_encode( aarch64_enc_ret() );
16528
16529 ins_pipe(pipe_branch);
16530 %}
16531
16532 // Die now.
16533 instruct ShouldNotReachHere() %{
16534 match(Halt);
16535
16536 ins_cost(CALL_COST);
16537 format %{ "ShouldNotReachHere" %}
16538
16539 ins_encode %{
16540 if (is_reachable()) {
16541 const char* str = __ code_string(_halt_reason);
16542 __ stop(str);
16543 }
16544 %}
16545
16546 ins_pipe(pipe_class_default);
16547 %}
16548
16549 // ============================================================================
16550 // Partial Subtype Check
16551 //
16552 // superklass array for an instance of the superklass. Set a hidden
16553 // internal cache on a hit (cache is checked with exposed code in
16554 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16555 // encoding ALSO sets flags.
16556
16557 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16558 %{
16559 match(Set result (PartialSubtypeCheck sub super));
16560 predicate(!UseSecondarySupersTable);
16561 effect(KILL cr, KILL temp);
16562
16563 ins_cost(20 * INSN_COST); // slightly larger than the next version
16564 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16565
16566 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16567
16568 opcode(0x1); // Force zero of result reg on hit
16569
16570 ins_pipe(pipe_class_memory);
16571 %}
16572
16573 // Two versions of partialSubtypeCheck, both used when we need to
16574 // search for a super class in the secondary supers array. The first
16575 // is used when we don't know _a priori_ the class being searched
16576 // for. The second, far more common, is used when we do know: this is
16577 // used for instanceof, checkcast, and any case where C2 can determine
16578 // it by constant propagation.
16579
16580 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16581 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16582 rFlagsReg cr)
16583 %{
16584 match(Set result (PartialSubtypeCheck sub super));
16585 predicate(UseSecondarySupersTable);
16586 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16587
16588 ins_cost(10 * INSN_COST); // slightly larger than the next version
16589 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16590
16591 ins_encode %{
16592 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16593 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16594 $vtemp$$FloatRegister,
16595 $result$$Register, /*L_success*/nullptr);
16596 %}
16597
16598 ins_pipe(pipe_class_memory);
16599 %}
16600
16601 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16602 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16603 rFlagsReg cr)
16604 %{
16605 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16606 predicate(UseSecondarySupersTable);
16607 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16608
16609 ins_cost(5 * INSN_COST); // smaller than the next version
16610 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16611
16612 ins_encode %{
16613 bool success = false;
16614 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16615 if (InlineSecondarySupersTest) {
16616 success =
16617 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16618 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16619 $vtemp$$FloatRegister,
16620 $result$$Register,
16621 super_klass_slot);
16622 } else {
16623 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16624 success = (call != nullptr);
16625 }
16626 if (!success) {
16627 ciEnv::current()->record_failure("CodeCache is full");
16628 return;
16629 }
16630 %}
16631
16632 ins_pipe(pipe_class_memory);
16633 %}
16634
16635 // Intrisics for String.compareTo()
16636
16637 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16638 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16639 %{
16640 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16641 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16642 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16643
16644 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16645 ins_encode %{
16646 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16647 __ string_compare($str1$$Register, $str2$$Register,
16648 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16649 $tmp1$$Register, $tmp2$$Register,
16650 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16651 %}
16652 ins_pipe(pipe_class_memory);
16653 %}
16654
16655 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16656 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16657 %{
16658 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16659 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16660 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16661
16662 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16663 ins_encode %{
16664 __ string_compare($str1$$Register, $str2$$Register,
16665 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16666 $tmp1$$Register, $tmp2$$Register,
16667 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16668 %}
16669 ins_pipe(pipe_class_memory);
16670 %}
16671
16672 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16673 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16674 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16675 %{
16676 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16677 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16678 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16679 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16680
16681 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16682 ins_encode %{
16683 __ string_compare($str1$$Register, $str2$$Register,
16684 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16685 $tmp1$$Register, $tmp2$$Register,
16686 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16687 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16688 %}
16689 ins_pipe(pipe_class_memory);
16690 %}
16691
16692 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16693 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16694 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16695 %{
16696 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16697 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16698 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16699 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16700
16701 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16702 ins_encode %{
16703 __ string_compare($str1$$Register, $str2$$Register,
16704 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16705 $tmp1$$Register, $tmp2$$Register,
16706 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16707 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16708 %}
16709 ins_pipe(pipe_class_memory);
16710 %}
16711
16712 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16713 // these string_compare variants as NEON register type for convenience so that the prototype of
16714 // string_compare can be shared with all variants.
16715
16716 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16717 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16718 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16719 pRegGov_P1 pgtmp2, rFlagsReg cr)
16720 %{
16721 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16722 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16723 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16724 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16725
16726 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16727 ins_encode %{
16728 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16729 __ string_compare($str1$$Register, $str2$$Register,
16730 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16731 $tmp1$$Register, $tmp2$$Register,
16732 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16733 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16734 StrIntrinsicNode::LL);
16735 %}
16736 ins_pipe(pipe_class_memory);
16737 %}
16738
16739 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16740 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16741 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16742 pRegGov_P1 pgtmp2, rFlagsReg cr)
16743 %{
16744 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16745 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16746 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16747 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16748
16749 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16750 ins_encode %{
16751 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16752 __ string_compare($str1$$Register, $str2$$Register,
16753 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16754 $tmp1$$Register, $tmp2$$Register,
16755 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16756 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16757 StrIntrinsicNode::LU);
16758 %}
16759 ins_pipe(pipe_class_memory);
16760 %}
16761
16762 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16763 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16764 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16765 pRegGov_P1 pgtmp2, rFlagsReg cr)
16766 %{
16767 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16768 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16769 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16770 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16771
16772 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16773 ins_encode %{
16774 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16775 __ string_compare($str1$$Register, $str2$$Register,
16776 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16777 $tmp1$$Register, $tmp2$$Register,
16778 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16779 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16780 StrIntrinsicNode::UL);
16781 %}
16782 ins_pipe(pipe_class_memory);
16783 %}
16784
16785 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16786 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16787 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16788 pRegGov_P1 pgtmp2, rFlagsReg cr)
16789 %{
16790 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16791 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16792 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16793 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16794
16795 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16796 ins_encode %{
16797 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16798 __ string_compare($str1$$Register, $str2$$Register,
16799 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16800 $tmp1$$Register, $tmp2$$Register,
16801 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16802 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16803 StrIntrinsicNode::UU);
16804 %}
16805 ins_pipe(pipe_class_memory);
16806 %}
16807
16808 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16809 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16810 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16811 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16812 %{
16813 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16814 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16815 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16816 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16817 TEMP vtmp0, TEMP vtmp1, KILL cr);
16818 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16819 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16820
16821 ins_encode %{
16822 __ string_indexof($str1$$Register, $str2$$Register,
16823 $cnt1$$Register, $cnt2$$Register,
16824 $tmp1$$Register, $tmp2$$Register,
16825 $tmp3$$Register, $tmp4$$Register,
16826 $tmp5$$Register, $tmp6$$Register,
16827 -1, $result$$Register, StrIntrinsicNode::UU);
16828 %}
16829 ins_pipe(pipe_class_memory);
16830 %}
16831
16832 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16833 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16834 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16835 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16836 %{
16837 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16838 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16839 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16840 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16841 TEMP vtmp0, TEMP vtmp1, KILL cr);
16842 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16843 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16844
16845 ins_encode %{
16846 __ string_indexof($str1$$Register, $str2$$Register,
16847 $cnt1$$Register, $cnt2$$Register,
16848 $tmp1$$Register, $tmp2$$Register,
16849 $tmp3$$Register, $tmp4$$Register,
16850 $tmp5$$Register, $tmp6$$Register,
16851 -1, $result$$Register, StrIntrinsicNode::LL);
16852 %}
16853 ins_pipe(pipe_class_memory);
16854 %}
16855
16856 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16857 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16858 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16859 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16860 %{
16861 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16862 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16863 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16864 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16865 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16866 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16867 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16868
16869 ins_encode %{
16870 __ string_indexof($str1$$Register, $str2$$Register,
16871 $cnt1$$Register, $cnt2$$Register,
16872 $tmp1$$Register, $tmp2$$Register,
16873 $tmp3$$Register, $tmp4$$Register,
16874 $tmp5$$Register, $tmp6$$Register,
16875 -1, $result$$Register, StrIntrinsicNode::UL);
16876 %}
16877 ins_pipe(pipe_class_memory);
16878 %}
16879
16880 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16881 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16882 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16883 %{
16884 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16885 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16886 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16887 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16888 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16889 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16890
16891 ins_encode %{
16892 int icnt2 = (int)$int_cnt2$$constant;
16893 __ string_indexof($str1$$Register, $str2$$Register,
16894 $cnt1$$Register, zr,
16895 $tmp1$$Register, $tmp2$$Register,
16896 $tmp3$$Register, $tmp4$$Register, zr, zr,
16897 icnt2, $result$$Register, StrIntrinsicNode::UU);
16898 %}
16899 ins_pipe(pipe_class_memory);
16900 %}
16901
16902 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16903 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16904 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16905 %{
16906 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16907 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16908 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16909 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16910 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16911 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16912
16913 ins_encode %{
16914 int icnt2 = (int)$int_cnt2$$constant;
16915 __ string_indexof($str1$$Register, $str2$$Register,
16916 $cnt1$$Register, zr,
16917 $tmp1$$Register, $tmp2$$Register,
16918 $tmp3$$Register, $tmp4$$Register, zr, zr,
16919 icnt2, $result$$Register, StrIntrinsicNode::LL);
16920 %}
16921 ins_pipe(pipe_class_memory);
16922 %}
16923
16924 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16925 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16926 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16927 %{
16928 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16929 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16930 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16931 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16932 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16933 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16934
16935 ins_encode %{
16936 int icnt2 = (int)$int_cnt2$$constant;
16937 __ string_indexof($str1$$Register, $str2$$Register,
16938 $cnt1$$Register, zr,
16939 $tmp1$$Register, $tmp2$$Register,
16940 $tmp3$$Register, $tmp4$$Register, zr, zr,
16941 icnt2, $result$$Register, StrIntrinsicNode::UL);
16942 %}
16943 ins_pipe(pipe_class_memory);
16944 %}
16945
16946 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16947 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16948 iRegINoSp tmp3, rFlagsReg cr)
16949 %{
16950 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16951 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16952 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16953 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16954
16955 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16956
16957 ins_encode %{
16958 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16959 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16960 $tmp3$$Register);
16961 %}
16962 ins_pipe(pipe_class_memory);
16963 %}
16964
16965 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16966 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16967 iRegINoSp tmp3, rFlagsReg cr)
16968 %{
16969 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16970 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16971 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16972 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16973
16974 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16975
16976 ins_encode %{
16977 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16978 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16979 $tmp3$$Register);
16980 %}
16981 ins_pipe(pipe_class_memory);
16982 %}
16983
16984 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16985 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16986 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16987 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16988 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16989 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16990 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16991 ins_encode %{
16992 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16993 $result$$Register, $ztmp1$$FloatRegister,
16994 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16995 $ptmp$$PRegister, true /* isL */);
16996 %}
16997 ins_pipe(pipe_class_memory);
16998 %}
16999
17000 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17001 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17002 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17003 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17004 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17005 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17006 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17007 ins_encode %{
17008 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17009 $result$$Register, $ztmp1$$FloatRegister,
17010 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17011 $ptmp$$PRegister, false /* isL */);
17012 %}
17013 ins_pipe(pipe_class_memory);
17014 %}
17015
17016 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17017 iRegI_R0 result, rFlagsReg cr)
17018 %{
17019 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17020 match(Set result (StrEquals (Binary str1 str2) cnt));
17021 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17022
17023 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17024 ins_encode %{
17025 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17026 __ string_equals($str1$$Register, $str2$$Register,
17027 $result$$Register, $cnt$$Register);
17028 %}
17029 ins_pipe(pipe_class_memory);
17030 %}
17031
17032 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17033 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17034 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17035 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17036 iRegP_R10 tmp, rFlagsReg cr)
17037 %{
17038 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17039 match(Set result (AryEq ary1 ary2));
17040 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17041 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17042 TEMP vtmp6, TEMP vtmp7, KILL cr);
17043
17044 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17045 ins_encode %{
17046 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17047 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17048 $result$$Register, $tmp$$Register, 1);
17049 if (tpc == nullptr) {
17050 ciEnv::current()->record_failure("CodeCache is full");
17051 return;
17052 }
17053 %}
17054 ins_pipe(pipe_class_memory);
17055 %}
17056
17057 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17058 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17059 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17060 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17061 iRegP_R10 tmp, rFlagsReg cr)
17062 %{
17063 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17064 match(Set result (AryEq ary1 ary2));
17065 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17066 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17067 TEMP vtmp6, TEMP vtmp7, KILL cr);
17068
17069 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17070 ins_encode %{
17071 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17072 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17073 $result$$Register, $tmp$$Register, 2);
17074 if (tpc == nullptr) {
17075 ciEnv::current()->record_failure("CodeCache is full");
17076 return;
17077 }
17078 %}
17079 ins_pipe(pipe_class_memory);
17080 %}
17081
17082 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17083 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17084 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17085 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17086 %{
17087 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17088 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17089 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17090
17091 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
17092 ins_encode %{
17093 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17094 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17095 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17096 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17097 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17098 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17099 (BasicType)$basic_type$$constant);
17100 if (tpc == nullptr) {
17101 ciEnv::current()->record_failure("CodeCache is full");
17102 return;
17103 }
17104 %}
17105 ins_pipe(pipe_class_memory);
17106 %}
17107
17108 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17109 %{
17110 match(Set result (CountPositives ary1 len));
17111 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17112 format %{ "count positives byte[] $ary1,$len -> $result" %}
17113 ins_encode %{
17114 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17115 if (tpc == nullptr) {
17116 ciEnv::current()->record_failure("CodeCache is full");
17117 return;
17118 }
17119 %}
17120 ins_pipe( pipe_slow );
17121 %}
17122
17123 // fast char[] to byte[] compression
17124 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17125 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17126 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17127 iRegI_R0 result, rFlagsReg cr)
17128 %{
17129 match(Set result (StrCompressedCopy src (Binary dst len)));
17130 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17131 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17132
17133 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17134 ins_encode %{
17135 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17136 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17137 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17138 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17139 %}
17140 ins_pipe(pipe_slow);
17141 %}
17142
17143 // fast byte[] to char[] inflation
17144 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17145 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17146 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17147 %{
17148 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17149 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17150 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17151 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17152
17153 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17154 ins_encode %{
17155 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17156 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17157 $vtmp2$$FloatRegister, $tmp$$Register);
17158 if (tpc == nullptr) {
17159 ciEnv::current()->record_failure("CodeCache is full");
17160 return;
17161 }
17162 %}
17163 ins_pipe(pipe_class_memory);
17164 %}
17165
17166 // encode char[] to byte[] in ISO_8859_1
17167 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17168 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17169 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17170 iRegI_R0 result, rFlagsReg cr)
17171 %{
17172 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17173 match(Set result (EncodeISOArray src (Binary dst len)));
17174 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17175 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17176
17177 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17178 ins_encode %{
17179 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17180 $result$$Register, false,
17181 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17182 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17183 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17184 %}
17185 ins_pipe(pipe_class_memory);
17186 %}
17187
17188 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17189 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17190 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17191 iRegI_R0 result, rFlagsReg cr)
17192 %{
17193 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17194 match(Set result (EncodeISOArray src (Binary dst len)));
17195 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17196 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17197
17198 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17199 ins_encode %{
17200 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17201 $result$$Register, true,
17202 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17203 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17204 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17205 %}
17206 ins_pipe(pipe_class_memory);
17207 %}
17208
17209 //----------------------------- CompressBits/ExpandBits ------------------------
17210
17211 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17212 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17213 match(Set dst (CompressBits src mask));
17214 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17215 format %{ "mov $tsrc, $src\n\t"
17216 "mov $tmask, $mask\n\t"
17217 "bext $tdst, $tsrc, $tmask\n\t"
17218 "mov $dst, $tdst"
17219 %}
17220 ins_encode %{
17221 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17222 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17223 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17224 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17225 %}
17226 ins_pipe(pipe_slow);
17227 %}
17228
17229 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17230 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17231 match(Set dst (CompressBits (LoadI mem) mask));
17232 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17233 format %{ "ldrs $tsrc, $mem\n\t"
17234 "ldrs $tmask, $mask\n\t"
17235 "bext $tdst, $tsrc, $tmask\n\t"
17236 "mov $dst, $tdst"
17237 %}
17238 ins_encode %{
17239 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17240 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17241 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17242 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17243 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17244 %}
17245 ins_pipe(pipe_slow);
17246 %}
17247
17248 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17249 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17250 match(Set dst (CompressBits src mask));
17251 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17252 format %{ "mov $tsrc, $src\n\t"
17253 "mov $tmask, $mask\n\t"
17254 "bext $tdst, $tsrc, $tmask\n\t"
17255 "mov $dst, $tdst"
17256 %}
17257 ins_encode %{
17258 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17259 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17260 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17261 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17262 %}
17263 ins_pipe(pipe_slow);
17264 %}
17265
17266 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17267 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17268 match(Set dst (CompressBits (LoadL mem) mask));
17269 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17270 format %{ "ldrd $tsrc, $mem\n\t"
17271 "ldrd $tmask, $mask\n\t"
17272 "bext $tdst, $tsrc, $tmask\n\t"
17273 "mov $dst, $tdst"
17274 %}
17275 ins_encode %{
17276 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17277 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17278 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17279 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17280 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17281 %}
17282 ins_pipe(pipe_slow);
17283 %}
17284
17285 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17286 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17287 match(Set dst (ExpandBits src mask));
17288 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17289 format %{ "mov $tsrc, $src\n\t"
17290 "mov $tmask, $mask\n\t"
17291 "bdep $tdst, $tsrc, $tmask\n\t"
17292 "mov $dst, $tdst"
17293 %}
17294 ins_encode %{
17295 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17296 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17297 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17298 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17299 %}
17300 ins_pipe(pipe_slow);
17301 %}
17302
17303 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17304 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17305 match(Set dst (ExpandBits (LoadI mem) mask));
17306 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17307 format %{ "ldrs $tsrc, $mem\n\t"
17308 "ldrs $tmask, $mask\n\t"
17309 "bdep $tdst, $tsrc, $tmask\n\t"
17310 "mov $dst, $tdst"
17311 %}
17312 ins_encode %{
17313 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17314 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17315 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17316 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17317 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17318 %}
17319 ins_pipe(pipe_slow);
17320 %}
17321
17322 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17323 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17324 match(Set dst (ExpandBits src mask));
17325 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17326 format %{ "mov $tsrc, $src\n\t"
17327 "mov $tmask, $mask\n\t"
17328 "bdep $tdst, $tsrc, $tmask\n\t"
17329 "mov $dst, $tdst"
17330 %}
17331 ins_encode %{
17332 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17333 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17334 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17335 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17336 %}
17337 ins_pipe(pipe_slow);
17338 %}
17339
17340
17341 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17342 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17343 match(Set dst (ExpandBits (LoadL mem) mask));
17344 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17345 format %{ "ldrd $tsrc, $mem\n\t"
17346 "ldrd $tmask, $mask\n\t"
17347 "bdep $tdst, $tsrc, $tmask\n\t"
17348 "mov $dst, $tdst"
17349 %}
17350 ins_encode %{
17351 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17352 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17353 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17354 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17355 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17356 %}
17357 ins_pipe(pipe_slow);
17358 %}
17359
17360 //----------------------------- Reinterpret ----------------------------------
17361 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17362 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17363 match(Set dst (ReinterpretHF2S src));
17364 format %{ "reinterpretHF2S $dst, $src" %}
17365 ins_encode %{
17366 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17367 %}
17368 ins_pipe(pipe_slow);
17369 %}
17370
17371 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17372 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17373 match(Set dst (ReinterpretS2HF src));
17374 format %{ "reinterpretS2HF $dst, $src" %}
17375 ins_encode %{
17376 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17377 %}
17378 ins_pipe(pipe_slow);
17379 %}
17380
17381 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17382 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17383 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
17384 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
17385 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
17386 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17387 // can be omitted in this pattern, resulting in -
17388 // fcvt $dst, $src // Convert float to half-precision float
17389 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17390 %{
17391 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17392 format %{ "convF2HFAndS2HF $dst, $src" %}
17393 ins_encode %{
17394 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17395 %}
17396 ins_pipe(pipe_slow);
17397 %}
17398
17399 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17400 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17401 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
17402 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17403 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
17404 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17405 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17406 // resulting in -
17407 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
17408 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17409 %{
17410 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17411 format %{ "convHF2SAndHF2F $dst, $src" %}
17412 ins_encode %{
17413 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17414 %}
17415 ins_pipe(pipe_slow);
17416 %}
17417
17418 // ============================================================================
17419 // This name is KNOWN by the ADLC and cannot be changed.
17420 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17421 // for this guy.
17422 instruct tlsLoadP(thread_RegP dst)
17423 %{
17424 match(Set dst (ThreadLocal));
17425
17426 ins_cost(0);
17427
17428 format %{ " -- \t// $dst=Thread::current(), empty" %}
17429
17430 size(0);
17431
17432 ins_encode( /*empty*/ );
17433
17434 ins_pipe(pipe_class_empty);
17435 %}
17436
17437 //----------PEEPHOLE RULES-----------------------------------------------------
17438 // These must follow all instruction definitions as they use the names
17439 // defined in the instructions definitions.
17440 //
17441 // peepmatch ( root_instr_name [preceding_instruction]* );
17442 //
17443 // peepconstraint %{
17444 // (instruction_number.operand_name relational_op instruction_number.operand_name
17445 // [, ...] );
17446 // // instruction numbers are zero-based using left to right order in peepmatch
17447 //
17448 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17449 // // provide an instruction_number.operand_name for each operand that appears
17450 // // in the replacement instruction's match rule
17451 //
17452 // ---------VM FLAGS---------------------------------------------------------
17453 //
17454 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17455 //
17456 // Each peephole rule is given an identifying number starting with zero and
17457 // increasing by one in the order seen by the parser. An individual peephole
17458 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17459 // on the command-line.
17460 //
17461 // ---------CURRENT LIMITATIONS----------------------------------------------
17462 //
17463 // Only match adjacent instructions in same basic block
17464 // Only equality constraints
17465 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17466 // Only one replacement instruction
17467 //
17468 // ---------EXAMPLE----------------------------------------------------------
17469 //
17470 // // pertinent parts of existing instructions in architecture description
17471 // instruct movI(iRegINoSp dst, iRegI src)
17472 // %{
17473 // match(Set dst (CopyI src));
17474 // %}
17475 //
17476 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17477 // %{
17478 // match(Set dst (AddI dst src));
17479 // effect(KILL cr);
17480 // %}
17481 //
17482 // // Change (inc mov) to lea
17483 // peephole %{
17484 // // increment preceded by register-register move
17485 // peepmatch ( incI_iReg movI );
17486 // // require that the destination register of the increment
17487 // // match the destination register of the move
17488 // peepconstraint ( 0.dst == 1.dst );
17489 // // construct a replacement instruction that sets
17490 // // the destination to ( move's source register + one )
17491 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17492 // %}
17493 //
17494
17495 // Implementation no longer uses movX instructions since
17496 // machine-independent system no longer uses CopyX nodes.
17497 //
17498 // peephole
17499 // %{
17500 // peepmatch (incI_iReg movI);
17501 // peepconstraint (0.dst == 1.dst);
17502 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17503 // %}
17504
17505 // peephole
17506 // %{
17507 // peepmatch (decI_iReg movI);
17508 // peepconstraint (0.dst == 1.dst);
17509 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17510 // %}
17511
17512 // peephole
17513 // %{
17514 // peepmatch (addI_iReg_imm movI);
17515 // peepconstraint (0.dst == 1.dst);
17516 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17517 // %}
17518
17519 // peephole
17520 // %{
17521 // peepmatch (incL_iReg movL);
17522 // peepconstraint (0.dst == 1.dst);
17523 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17524 // %}
17525
17526 // peephole
17527 // %{
17528 // peepmatch (decL_iReg movL);
17529 // peepconstraint (0.dst == 1.dst);
17530 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17531 // %}
17532
17533 // peephole
17534 // %{
17535 // peepmatch (addL_iReg_imm movL);
17536 // peepconstraint (0.dst == 1.dst);
17537 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17538 // %}
17539
17540 // peephole
17541 // %{
17542 // peepmatch (addP_iReg_imm movP);
17543 // peepconstraint (0.dst == 1.dst);
17544 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17545 // %}
17546
17547 // // Change load of spilled value to only a spill
17548 // instruct storeI(memory mem, iRegI src)
17549 // %{
17550 // match(Set mem (StoreI mem src));
17551 // %}
17552 //
17553 // instruct loadI(iRegINoSp dst, memory mem)
17554 // %{
17555 // match(Set dst (LoadI mem));
17556 // %}
17557 //
17558
17559 //----------SMARTSPILL RULES---------------------------------------------------
17560 // These must follow all instruction definitions as they use the names
17561 // defined in the instructions definitions.
17562
17563 // Local Variables:
17564 // mode: c++
17565 // End: