1 //
2 // Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for vector register V10
885 reg_class v10_veca_reg(
886 V10, V10_H, V10_J, V10_K
887 );
888
889 // Class for vector register V11
890 reg_class v11_veca_reg(
891 V11, V11_H, V11_J, V11_K
892 );
893
894 // Class for vector register V12
895 reg_class v12_veca_reg(
896 V12, V12_H, V12_J, V12_K
897 );
898
899 // Class for vector register V13
900 reg_class v13_veca_reg(
901 V13, V13_H, V13_J, V13_K
902 );
903
904 // Class for vector register V17
905 reg_class v17_veca_reg(
906 V17, V17_H, V17_J, V17_K
907 );
908
909 // Class for vector register V18
910 reg_class v18_veca_reg(
911 V18, V18_H, V18_J, V18_K
912 );
913
914 // Class for vector register V23
915 reg_class v23_veca_reg(
916 V23, V23_H, V23_J, V23_K
917 );
918
919 // Class for vector register V24
920 reg_class v24_veca_reg(
921 V24, V24_H, V24_J, V24_K
922 );
923
924 // Class for 128 bit register v0
925 reg_class v0_reg(
926 V0, V0_H
927 );
928
929 // Class for 128 bit register v1
930 reg_class v1_reg(
931 V1, V1_H
932 );
933
934 // Class for 128 bit register v2
935 reg_class v2_reg(
936 V2, V2_H
937 );
938
939 // Class for 128 bit register v3
940 reg_class v3_reg(
941 V3, V3_H
942 );
943
944 // Class for 128 bit register v4
945 reg_class v4_reg(
946 V4, V4_H
947 );
948
949 // Class for 128 bit register v5
950 reg_class v5_reg(
951 V5, V5_H
952 );
953
954 // Class for 128 bit register v6
955 reg_class v6_reg(
956 V6, V6_H
957 );
958
959 // Class for 128 bit register v7
960 reg_class v7_reg(
961 V7, V7_H
962 );
963
964 // Class for 128 bit register v8
965 reg_class v8_reg(
966 V8, V8_H
967 );
968
969 // Class for 128 bit register v9
970 reg_class v9_reg(
971 V9, V9_H
972 );
973
974 // Class for 128 bit register v10
975 reg_class v10_reg(
976 V10, V10_H
977 );
978
979 // Class for 128 bit register v11
980 reg_class v11_reg(
981 V11, V11_H
982 );
983
984 // Class for 128 bit register v12
985 reg_class v12_reg(
986 V12, V12_H
987 );
988
989 // Class for 128 bit register v13
990 reg_class v13_reg(
991 V13, V13_H
992 );
993
994 // Class for 128 bit register v14
995 reg_class v14_reg(
996 V14, V14_H
997 );
998
999 // Class for 128 bit register v15
1000 reg_class v15_reg(
1001 V15, V15_H
1002 );
1003
1004 // Class for 128 bit register v16
1005 reg_class v16_reg(
1006 V16, V16_H
1007 );
1008
1009 // Class for 128 bit register v17
1010 reg_class v17_reg(
1011 V17, V17_H
1012 );
1013
1014 // Class for 128 bit register v18
1015 reg_class v18_reg(
1016 V18, V18_H
1017 );
1018
1019 // Class for 128 bit register v19
1020 reg_class v19_reg(
1021 V19, V19_H
1022 );
1023
1024 // Class for 128 bit register v20
1025 reg_class v20_reg(
1026 V20, V20_H
1027 );
1028
1029 // Class for 128 bit register v21
1030 reg_class v21_reg(
1031 V21, V21_H
1032 );
1033
1034 // Class for 128 bit register v22
1035 reg_class v22_reg(
1036 V22, V22_H
1037 );
1038
1039 // Class for 128 bit register v23
1040 reg_class v23_reg(
1041 V23, V23_H
1042 );
1043
1044 // Class for 128 bit register v24
1045 reg_class v24_reg(
1046 V24, V24_H
1047 );
1048
1049 // Class for 128 bit register v25
1050 reg_class v25_reg(
1051 V25, V25_H
1052 );
1053
1054 // Class for 128 bit register v26
1055 reg_class v26_reg(
1056 V26, V26_H
1057 );
1058
1059 // Class for 128 bit register v27
1060 reg_class v27_reg(
1061 V27, V27_H
1062 );
1063
1064 // Class for 128 bit register v28
1065 reg_class v28_reg(
1066 V28, V28_H
1067 );
1068
1069 // Class for 128 bit register v29
1070 reg_class v29_reg(
1071 V29, V29_H
1072 );
1073
1074 // Class for 128 bit register v30
1075 reg_class v30_reg(
1076 V30, V30_H
1077 );
1078
1079 // Class for 128 bit register v31
1080 reg_class v31_reg(
1081 V31, V31_H
1082 );
1083
1084 // Class for all SVE predicate registers.
1085 reg_class pr_reg (
1086 P0,
1087 P1,
1088 P2,
1089 P3,
1090 P4,
1091 P5,
1092 P6,
1093 // P7, non-allocatable, preserved with all elements preset to TRUE.
1094 P8,
1095 P9,
1096 P10,
1097 P11,
1098 P12,
1099 P13,
1100 P14,
1101 P15
1102 );
1103
1104 // Class for SVE governing predicate registers, which are used
1105 // to determine the active elements of a predicated instruction.
1106 reg_class gov_pr (
1107 P0,
1108 P1,
1109 P2,
1110 P3,
1111 P4,
1112 P5,
1113 P6,
1114 // P7, non-allocatable, preserved with all elements preset to TRUE.
1115 );
1116
1117 reg_class p0_reg(P0);
1118 reg_class p1_reg(P1);
1119
1120 // Singleton class for condition codes
1121 reg_class int_flags(RFLAGS);
1122
1123 %}
1124
1125 //----------DEFINITION BLOCK---------------------------------------------------
1126 // Define name --> value mappings to inform the ADLC of an integer valued name
1127 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1128 // Format:
1129 // int_def <name> ( <int_value>, <expression>);
1130 // Generated Code in ad_<arch>.hpp
1131 // #define <name> (<expression>)
1132 // // value == <int_value>
1133 // Generated code in ad_<arch>.cpp adlc_verification()
1134 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1135 //
1136
1137 // we follow the ppc-aix port in using a simple cost model which ranks
1138 // register operations as cheap, memory ops as more expensive and
1139 // branches as most expensive. the first two have a low as well as a
1140 // normal cost. huge cost appears to be a way of saying don't do
1141 // something
1142
1143 definitions %{
1144 // The default cost (of a register move instruction).
1145 int_def INSN_COST ( 100, 100);
1146 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1147 int_def CALL_COST ( 200, 2 * INSN_COST);
1148 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1149 %}
1150
1151
1152 //----------SOURCE BLOCK-------------------------------------------------------
1153 // This is a block of C++ code which provides values, functions, and
1154 // definitions necessary in the rest of the architecture description
1155
1156 source_hpp %{
1157
1158 #include "asm/macroAssembler.hpp"
1159 #include "gc/shared/barrierSetAssembler.hpp"
1160 #include "gc/shared/cardTable.hpp"
1161 #include "gc/shared/cardTableBarrierSet.hpp"
1162 #include "gc/shared/collectedHeap.hpp"
1163 #include "opto/addnode.hpp"
1164 #include "opto/convertnode.hpp"
1165 #include "runtime/objectMonitor.hpp"
1166
1167 extern RegMask _ANY_REG32_mask;
1168 extern RegMask _ANY_REG_mask;
1169 extern RegMask _PTR_REG_mask;
1170 extern RegMask _NO_SPECIAL_REG32_mask;
1171 extern RegMask _NO_SPECIAL_REG_mask;
1172 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1173 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1174
1175 class CallStubImpl {
1176
1177 //--------------------------------------------------------------
1178 //---< Used for optimization in Compile::shorten_branches >---
1179 //--------------------------------------------------------------
1180
1181 public:
1182 // Size of call trampoline stub.
1183 static uint size_call_trampoline() {
1184 return MacroAssembler::max_trampoline_stub_size(); // no call trampolines on this platform
1185 }
1186
1187 // number of relocations needed by a call trampoline stub
1188 static uint reloc_call_trampoline() {
1189 return 0; // no call trampolines on this platform
1190 }
1191 };
1192
1193 class HandlerImpl {
1194
1195 public:
1196
1197 static int emit_exception_handler(C2_MacroAssembler *masm);
1198 static int emit_deopt_handler(C2_MacroAssembler* masm);
1199
1200 static uint size_exception_handler() {
1201 return MacroAssembler::far_codestub_branch_size();
1202 }
1203
1204 static uint size_deopt_handler() {
1205 // count one adr and one far branch instruction
1206 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1207 }
1208 };
1209
1210 class Node::PD {
1211 public:
1212 enum NodeFlags {
1213 _last_flag = Node::_last_flag
1214 };
1215 };
1216
1217 bool is_CAS(int opcode, bool maybe_volatile);
1218
1219 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1220
1221 bool unnecessary_acquire(const Node *barrier);
1222 bool needs_acquiring_load(const Node *load);
1223
1224 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1225
1226 bool unnecessary_release(const Node *barrier);
1227 bool unnecessary_volatile(const Node *barrier);
1228 bool needs_releasing_store(const Node *store);
1229
1230 // predicate controlling translation of CompareAndSwapX
1231 bool needs_acquiring_load_exclusive(const Node *load);
1232
1233 // predicate controlling addressing modes
1234 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1235
1236 // Convert BootTest condition to Assembler condition.
1237 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1238 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1239 %}
1240
1241 source %{
1242
1243 // Derived RegMask with conditionally allocatable registers
1244
1245 void PhaseOutput::pd_perform_mach_node_analysis() {
1246 }
1247
1248 int MachNode::pd_alignment_required() const {
1249 return 1;
1250 }
1251
1252 int MachNode::compute_padding(int current_offset) const {
1253 return 0;
1254 }
1255
1256 RegMask _ANY_REG32_mask;
1257 RegMask _ANY_REG_mask;
1258 RegMask _PTR_REG_mask;
1259 RegMask _NO_SPECIAL_REG32_mask;
1260 RegMask _NO_SPECIAL_REG_mask;
1261 RegMask _NO_SPECIAL_PTR_REG_mask;
1262 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1263
1264 void reg_mask_init() {
1265 // We derive below RegMask(s) from the ones which are auto-generated from
1266 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1267 // registers conditionally reserved.
1268
1269 _ANY_REG32_mask.assignFrom(_ALL_REG32_mask);
1270 _ANY_REG32_mask.remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1271
1272 _ANY_REG_mask.assignFrom(_ALL_REG_mask);
1273
1274 _PTR_REG_mask.assignFrom(_ALL_REG_mask);
1275
1276 _NO_SPECIAL_REG32_mask.assignFrom(_ALL_REG32_mask);
1277 _NO_SPECIAL_REG32_mask.subtract(_NON_ALLOCATABLE_REG32_mask);
1278
1279 _NO_SPECIAL_REG_mask.assignFrom(_ALL_REG_mask);
1280 _NO_SPECIAL_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1281
1282 _NO_SPECIAL_PTR_REG_mask.assignFrom(_ALL_REG_mask);
1283 _NO_SPECIAL_PTR_REG_mask.subtract(_NON_ALLOCATABLE_REG_mask);
1284
1285 // r27 is not allocatable when compressed oops is on and heapbase is not
1286 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1287 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1288 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1289 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1290 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1291 }
1292
1293 // r29 is not allocatable when PreserveFramePointer is on
1294 if (PreserveFramePointer) {
1295 _NO_SPECIAL_REG32_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1296 _NO_SPECIAL_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1297 _NO_SPECIAL_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1298 }
1299
1300 _NO_SPECIAL_NO_RFP_PTR_REG_mask.assignFrom(_NO_SPECIAL_PTR_REG_mask);
1301 _NO_SPECIAL_NO_RFP_PTR_REG_mask.remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1302 }
1303
1304 // Optimizaton of volatile gets and puts
1305 // -------------------------------------
1306 //
1307 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1308 // use to implement volatile reads and writes. For a volatile read
1309 // we simply need
1310 //
1311 // ldar<x>
1312 //
1313 // and for a volatile write we need
1314 //
1315 // stlr<x>
1316 //
1317 // Alternatively, we can implement them by pairing a normal
1318 // load/store with a memory barrier. For a volatile read we need
1319 //
1320 // ldr<x>
1321 // dmb ishld
1322 //
1323 // for a volatile write
1324 //
1325 // dmb ish
1326 // str<x>
1327 // dmb ish
1328 //
1329 // We can also use ldaxr and stlxr to implement compare and swap CAS
1330 // sequences. These are normally translated to an instruction
1331 // sequence like the following
1332 //
1333 // dmb ish
1334 // retry:
1335 // ldxr<x> rval raddr
1336 // cmp rval rold
1337 // b.ne done
1338 // stlxr<x> rval, rnew, rold
1339 // cbnz rval retry
1340 // done:
1341 // cset r0, eq
1342 // dmb ishld
1343 //
1344 // Note that the exclusive store is already using an stlxr
1345 // instruction. That is required to ensure visibility to other
1346 // threads of the exclusive write (assuming it succeeds) before that
1347 // of any subsequent writes.
1348 //
1349 // The following instruction sequence is an improvement on the above
1350 //
1351 // retry:
1352 // ldaxr<x> rval raddr
1353 // cmp rval rold
1354 // b.ne done
1355 // stlxr<x> rval, rnew, rold
1356 // cbnz rval retry
1357 // done:
1358 // cset r0, eq
1359 //
1360 // We don't need the leading dmb ish since the stlxr guarantees
1361 // visibility of prior writes in the case that the swap is
1362 // successful. Crucially we don't have to worry about the case where
1363 // the swap is not successful since no valid program should be
1364 // relying on visibility of prior changes by the attempting thread
1365 // in the case where the CAS fails.
1366 //
1367 // Similarly, we don't need the trailing dmb ishld if we substitute
1368 // an ldaxr instruction since that will provide all the guarantees we
1369 // require regarding observation of changes made by other threads
1370 // before any change to the CAS address observed by the load.
1371 //
1372 // In order to generate the desired instruction sequence we need to
1373 // be able to identify specific 'signature' ideal graph node
1374 // sequences which i) occur as a translation of a volatile reads or
1375 // writes or CAS operations and ii) do not occur through any other
1376 // translation or graph transformation. We can then provide
1377 // alternative aldc matching rules which translate these node
1378 // sequences to the desired machine code sequences. Selection of the
1379 // alternative rules can be implemented by predicates which identify
1380 // the relevant node sequences.
1381 //
1382 // The ideal graph generator translates a volatile read to the node
1383 // sequence
1384 //
1385 // LoadX[mo_acquire]
1386 // MemBarAcquire
1387 //
1388 // As a special case when using the compressed oops optimization we
1389 // may also see this variant
1390 //
1391 // LoadN[mo_acquire]
1392 // DecodeN
1393 // MemBarAcquire
1394 //
1395 // A volatile write is translated to the node sequence
1396 //
1397 // MemBarRelease
1398 // StoreX[mo_release] {CardMark}-optional
1399 // MemBarVolatile
1400 //
1401 // n.b. the above node patterns are generated with a strict
1402 // 'signature' configuration of input and output dependencies (see
1403 // the predicates below for exact details). The card mark may be as
1404 // simple as a few extra nodes or, in a few GC configurations, may
1405 // include more complex control flow between the leading and
1406 // trailing memory barriers. However, whatever the card mark
1407 // configuration these signatures are unique to translated volatile
1408 // reads/stores -- they will not appear as a result of any other
1409 // bytecode translation or inlining nor as a consequence of
1410 // optimizing transforms.
1411 //
1412 // We also want to catch inlined unsafe volatile gets and puts and
1413 // be able to implement them using either ldar<x>/stlr<x> or some
1414 // combination of ldr<x>/stlr<x> and dmb instructions.
1415 //
1416 // Inlined unsafe volatiles puts manifest as a minor variant of the
1417 // normal volatile put node sequence containing an extra cpuorder
1418 // membar
1419 //
1420 // MemBarRelease
1421 // MemBarCPUOrder
1422 // StoreX[mo_release] {CardMark}-optional
1423 // MemBarCPUOrder
1424 // MemBarVolatile
1425 //
1426 // n.b. as an aside, a cpuorder membar is not itself subject to
1427 // matching and translation by adlc rules. However, the rule
1428 // predicates need to detect its presence in order to correctly
1429 // select the desired adlc rules.
1430 //
1431 // Inlined unsafe volatile gets manifest as a slightly different
1432 // node sequence to a normal volatile get because of the
1433 // introduction of some CPUOrder memory barriers to bracket the
1434 // Load. However, but the same basic skeleton of a LoadX feeding a
1435 // MemBarAcquire, possibly through an optional DecodeN, is still
1436 // present
1437 //
1438 // MemBarCPUOrder
1439 // || \\
1440 // MemBarCPUOrder LoadX[mo_acquire]
1441 // || |
1442 // || {DecodeN} optional
1443 // || /
1444 // MemBarAcquire
1445 //
1446 // In this case the acquire membar does not directly depend on the
1447 // load. However, we can be sure that the load is generated from an
1448 // inlined unsafe volatile get if we see it dependent on this unique
1449 // sequence of membar nodes. Similarly, given an acquire membar we
1450 // can know that it was added because of an inlined unsafe volatile
1451 // get if it is fed and feeds a cpuorder membar and if its feed
1452 // membar also feeds an acquiring load.
1453 //
1454 // Finally an inlined (Unsafe) CAS operation is translated to the
1455 // following ideal graph
1456 //
1457 // MemBarRelease
1458 // MemBarCPUOrder
1459 // CompareAndSwapX {CardMark}-optional
1460 // MemBarCPUOrder
1461 // MemBarAcquire
1462 //
1463 // So, where we can identify these volatile read and write
1464 // signatures we can choose to plant either of the above two code
1465 // sequences. For a volatile read we can simply plant a normal
1466 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1467 // also choose to inhibit translation of the MemBarAcquire and
1468 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1469 //
1470 // When we recognise a volatile store signature we can choose to
1471 // plant at a dmb ish as a translation for the MemBarRelease, a
1472 // normal str<x> and then a dmb ish for the MemBarVolatile.
1473 // Alternatively, we can inhibit translation of the MemBarRelease
1474 // and MemBarVolatile and instead plant a simple stlr<x>
1475 // instruction.
1476 //
1477 // when we recognise a CAS signature we can choose to plant a dmb
1478 // ish as a translation for the MemBarRelease, the conventional
1479 // macro-instruction sequence for the CompareAndSwap node (which
1480 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1481 // Alternatively, we can elide generation of the dmb instructions
1482 // and plant the alternative CompareAndSwap macro-instruction
1483 // sequence (which uses ldaxr<x>).
1484 //
1485 // Of course, the above only applies when we see these signature
1486 // configurations. We still want to plant dmb instructions in any
1487 // other cases where we may see a MemBarAcquire, MemBarRelease or
1488 // MemBarVolatile. For example, at the end of a constructor which
1489 // writes final/volatile fields we will see a MemBarRelease
1490 // instruction and this needs a 'dmb ish' lest we risk the
1491 // constructed object being visible without making the
1492 // final/volatile field writes visible.
1493 //
1494 // n.b. the translation rules below which rely on detection of the
1495 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1496 // If we see anything other than the signature configurations we
1497 // always just translate the loads and stores to ldr<x> and str<x>
1498 // and translate acquire, release and volatile membars to the
1499 // relevant dmb instructions.
1500 //
1501
1502 // is_CAS(int opcode, bool maybe_volatile)
1503 //
1504 // return true if opcode is one of the possible CompareAndSwapX
1505 // values otherwise false.
1506
1507 bool is_CAS(int opcode, bool maybe_volatile)
1508 {
1509 switch(opcode) {
1510 // We handle these
1511 case Op_CompareAndSwapI:
1512 case Op_CompareAndSwapL:
1513 case Op_CompareAndSwapP:
1514 case Op_CompareAndSwapN:
1515 case Op_ShenandoahCompareAndSwapP:
1516 case Op_ShenandoahCompareAndSwapN:
1517 case Op_CompareAndSwapB:
1518 case Op_CompareAndSwapS:
1519 case Op_GetAndSetI:
1520 case Op_GetAndSetL:
1521 case Op_GetAndSetP:
1522 case Op_GetAndSetN:
1523 case Op_GetAndAddI:
1524 case Op_GetAndAddL:
1525 return true;
1526 case Op_CompareAndExchangeI:
1527 case Op_CompareAndExchangeN:
1528 case Op_CompareAndExchangeB:
1529 case Op_CompareAndExchangeS:
1530 case Op_CompareAndExchangeL:
1531 case Op_CompareAndExchangeP:
1532 case Op_WeakCompareAndSwapB:
1533 case Op_WeakCompareAndSwapS:
1534 case Op_WeakCompareAndSwapI:
1535 case Op_WeakCompareAndSwapL:
1536 case Op_WeakCompareAndSwapP:
1537 case Op_WeakCompareAndSwapN:
1538 case Op_ShenandoahWeakCompareAndSwapP:
1539 case Op_ShenandoahWeakCompareAndSwapN:
1540 case Op_ShenandoahCompareAndExchangeP:
1541 case Op_ShenandoahCompareAndExchangeN:
1542 return maybe_volatile;
1543 default:
1544 return false;
1545 }
1546 }
1547
1548 // helper to determine the maximum number of Phi nodes we may need to
1549 // traverse when searching from a card mark membar for the merge mem
1550 // feeding a trailing membar or vice versa
1551
1552 // predicates controlling emit of ldr<x>/ldar<x>
1553
1554 bool unnecessary_acquire(const Node *barrier)
1555 {
1556 assert(barrier->is_MemBar(), "expecting a membar");
1557
1558 MemBarNode* mb = barrier->as_MemBar();
1559
1560 if (mb->trailing_load()) {
1561 return true;
1562 }
1563
1564 if (mb->trailing_load_store()) {
1565 Node* load_store = mb->in(MemBarNode::Precedent);
1566 assert(load_store->is_LoadStore(), "unexpected graph shape");
1567 return is_CAS(load_store->Opcode(), true);
1568 }
1569
1570 return false;
1571 }
1572
1573 bool needs_acquiring_load(const Node *n)
1574 {
1575 assert(n->is_Load(), "expecting a load");
1576 LoadNode *ld = n->as_Load();
1577 return ld->is_acquire();
1578 }
1579
1580 bool unnecessary_release(const Node *n)
1581 {
1582 assert((n->is_MemBar() &&
1583 n->Opcode() == Op_MemBarRelease),
1584 "expecting a release membar");
1585
1586 MemBarNode *barrier = n->as_MemBar();
1587 if (!barrier->leading()) {
1588 return false;
1589 } else {
1590 Node* trailing = barrier->trailing_membar();
1591 MemBarNode* trailing_mb = trailing->as_MemBar();
1592 assert(trailing_mb->trailing(), "Not a trailing membar?");
1593 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1594
1595 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1596 if (mem->is_Store()) {
1597 assert(mem->as_Store()->is_release(), "");
1598 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1599 return true;
1600 } else {
1601 assert(mem->is_LoadStore(), "");
1602 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1603 return is_CAS(mem->Opcode(), true);
1604 }
1605 }
1606 return false;
1607 }
1608
1609 bool unnecessary_volatile(const Node *n)
1610 {
1611 // assert n->is_MemBar();
1612 MemBarNode *mbvol = n->as_MemBar();
1613
1614 bool release = mbvol->trailing_store();
1615 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1616 #ifdef ASSERT
1617 if (release) {
1618 Node* leading = mbvol->leading_membar();
1619 assert(leading->Opcode() == Op_MemBarRelease, "");
1620 assert(leading->as_MemBar()->leading_store(), "");
1621 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1622 }
1623 #endif
1624
1625 return release;
1626 }
1627
1628 // predicates controlling emit of str<x>/stlr<x>
1629
1630 bool needs_releasing_store(const Node *n)
1631 {
1632 // assert n->is_Store();
1633 StoreNode *st = n->as_Store();
1634 return st->trailing_membar() != nullptr;
1635 }
1636
1637 // predicate controlling translation of CAS
1638 //
1639 // returns true if CAS needs to use an acquiring load otherwise false
1640
1641 bool needs_acquiring_load_exclusive(const Node *n)
1642 {
1643 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1644 LoadStoreNode* ldst = n->as_LoadStore();
1645 if (is_CAS(n->Opcode(), false)) {
1646 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1647 } else {
1648 return ldst->trailing_membar() != nullptr;
1649 }
1650
1651 // so we can just return true here
1652 return true;
1653 }
1654
1655 #define __ masm->
1656
1657 // advance declarations for helper functions to convert register
1658 // indices to register objects
1659
1660 // the ad file has to provide implementations of certain methods
1661 // expected by the generic code
1662 //
1663 // REQUIRED FUNCTIONALITY
1664
1665 //=============================================================================
1666
1667 // !!!!! Special hack to get all types of calls to specify the byte offset
1668 // from the start of the call to the point where the return address
1669 // will point.
1670
1671 int MachCallStaticJavaNode::ret_addr_offset()
1672 {
1673 // call should be a simple bl
1674 int off = 4;
1675 return off;
1676 }
1677
1678 int MachCallDynamicJavaNode::ret_addr_offset()
1679 {
1680 return 16; // movz, movk, movk, bl
1681 }
1682
1683 int MachCallRuntimeNode::ret_addr_offset() {
1684 // for generated stubs the call will be
1685 // bl(addr)
1686 // or with far branches
1687 // bl(trampoline_stub)
1688 // for real runtime callouts it will be six instructions
1689 // see aarch64_enc_java_to_runtime
1690 // adr(rscratch2, retaddr)
1691 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1692 // lea(rscratch1, RuntimeAddress(addr)
1693 // blr(rscratch1)
1694 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1695 if (cb) {
1696 return 1 * NativeInstruction::instruction_size;
1697 } else {
1698 return 6 * NativeInstruction::instruction_size;
1699 }
1700 }
1701
1702 //=============================================================================
1703
1704 #ifndef PRODUCT
1705 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1706 st->print("BREAKPOINT");
1707 }
1708 #endif
1709
1710 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1711 __ brk(0);
1712 }
1713
1714 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1715 return MachNode::size(ra_);
1716 }
1717
1718 //=============================================================================
1719
1720 #ifndef PRODUCT
1721 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1722 st->print("nop \t# %d bytes pad for loops and calls", _count);
1723 }
1724 #endif
1725
1726 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1727 for (int i = 0; i < _count; i++) {
1728 __ nop();
1729 }
1730 }
1731
1732 uint MachNopNode::size(PhaseRegAlloc*) const {
1733 return _count * NativeInstruction::instruction_size;
1734 }
1735
1736 //=============================================================================
1737 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::EMPTY;
1738
1739 int ConstantTable::calculate_table_base_offset() const {
1740 return 0; // absolute addressing, no offset
1741 }
1742
1743 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1744 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1745 ShouldNotReachHere();
1746 }
1747
1748 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1749 // Empty encoding
1750 }
1751
1752 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1753 return 0;
1754 }
1755
1756 #ifndef PRODUCT
1757 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1758 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1759 }
1760 #endif
1761
1762 #ifndef PRODUCT
1763 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1764 Compile* C = ra_->C;
1765
1766 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1767
1768 if (C->output()->need_stack_bang(framesize))
1769 st->print("# stack bang size=%d\n\t", framesize);
1770
1771 if (VM_Version::use_rop_protection()) {
1772 st->print("ldr zr, [lr]\n\t");
1773 st->print("paciaz\n\t");
1774 }
1775 if (framesize < ((1 << 9) + 2 * wordSize)) {
1776 st->print("sub sp, sp, #%d\n\t", framesize);
1777 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1778 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1779 } else {
1780 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1781 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1782 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1783 st->print("sub sp, sp, rscratch1");
1784 }
1785 if (C->stub_function() == nullptr) {
1786 st->print("\n\t");
1787 st->print("ldr rscratch1, [guard]\n\t");
1788 st->print("dmb ishld\n\t");
1789 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1790 st->print("cmp rscratch1, rscratch2\n\t");
1791 st->print("b.eq skip");
1792 st->print("\n\t");
1793 st->print("blr #nmethod_entry_barrier_stub\n\t");
1794 st->print("b skip\n\t");
1795 st->print("guard: int\n\t");
1796 st->print("\n\t");
1797 st->print("skip:\n\t");
1798 }
1799 }
1800 #endif
1801
1802 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1803 Compile* C = ra_->C;
1804
1805 // n.b. frame size includes space for return pc and rfp
1806 const int framesize = C->output()->frame_size_in_bytes();
1807
1808 if (C->clinit_barrier_on_entry()) {
1809 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1810
1811 Label L_skip_barrier;
1812
1813 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1814 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1815 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1816 __ bind(L_skip_barrier);
1817 }
1818
1819 if (C->max_vector_size() > 0) {
1820 __ reinitialize_ptrue();
1821 }
1822
1823 int bangsize = C->output()->bang_size_in_bytes();
1824 if (C->output()->need_stack_bang(bangsize))
1825 __ generate_stack_overflow_check(bangsize);
1826
1827 __ build_frame(framesize);
1828
1829 if (C->stub_function() == nullptr) {
1830 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1831 // Dummy labels for just measuring the code size
1832 Label dummy_slow_path;
1833 Label dummy_continuation;
1834 Label dummy_guard;
1835 Label* slow_path = &dummy_slow_path;
1836 Label* continuation = &dummy_continuation;
1837 Label* guard = &dummy_guard;
1838 if (!Compile::current()->output()->in_scratch_emit_size()) {
1839 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1840 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1841 Compile::current()->output()->add_stub(stub);
1842 slow_path = &stub->entry();
1843 continuation = &stub->continuation();
1844 guard = &stub->guard();
1845 }
1846 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1847 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1848 }
1849
1850 if (VerifyStackAtCalls) {
1851 Unimplemented();
1852 }
1853
1854 C->output()->set_frame_complete(__ offset());
1855
1856 if (C->has_mach_constant_base_node()) {
1857 // NOTE: We set the table base offset here because users might be
1858 // emitted before MachConstantBaseNode.
1859 ConstantTable& constant_table = C->output()->constant_table();
1860 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1861 }
1862 }
1863
1864 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1865 {
1866 return MachNode::size(ra_); // too many variables; just compute it
1867 // the hard way
1868 }
1869
1870 int MachPrologNode::reloc() const
1871 {
1872 return 0;
1873 }
1874
1875 //=============================================================================
1876
1877 #ifndef PRODUCT
1878 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1879 Compile* C = ra_->C;
1880 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1881
1882 st->print("# pop frame %d\n\t",framesize);
1883
1884 if (framesize == 0) {
1885 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1886 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1887 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1888 st->print("add sp, sp, #%d\n\t", framesize);
1889 } else {
1890 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1891 st->print("add sp, sp, rscratch1\n\t");
1892 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1893 }
1894 if (VM_Version::use_rop_protection()) {
1895 st->print("autiaz\n\t");
1896 st->print("ldr zr, [lr]\n\t");
1897 }
1898
1899 if (do_polling() && C->is_method_compilation()) {
1900 st->print("# test polling word\n\t");
1901 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1902 st->print("cmp sp, rscratch1\n\t");
1903 st->print("bhi #slow_path");
1904 }
1905 }
1906 #endif
1907
1908 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1909 Compile* C = ra_->C;
1910 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1911
1912 __ remove_frame(framesize);
1913
1914 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1915 __ reserved_stack_check();
1916 }
1917
1918 if (do_polling() && C->is_method_compilation()) {
1919 Label dummy_label;
1920 Label* code_stub = &dummy_label;
1921 if (!C->output()->in_scratch_emit_size()) {
1922 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1923 C->output()->add_stub(stub);
1924 code_stub = &stub->entry();
1925 }
1926 __ relocate(relocInfo::poll_return_type);
1927 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1928 }
1929 }
1930
1931 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1932 // Variable size. Determine dynamically.
1933 return MachNode::size(ra_);
1934 }
1935
1936 int MachEpilogNode::reloc() const {
1937 // Return number of relocatable values contained in this instruction.
1938 return 1; // 1 for polling page.
1939 }
1940
1941 const Pipeline * MachEpilogNode::pipeline() const {
1942 return MachNode::pipeline_class();
1943 }
1944
1945 //=============================================================================
1946
1947 static enum RC rc_class(OptoReg::Name reg) {
1948
1949 if (reg == OptoReg::Bad) {
1950 return rc_bad;
1951 }
1952
1953 // we have 32 int registers * 2 halves
1954 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1955
1956 if (reg < slots_of_int_registers) {
1957 return rc_int;
1958 }
1959
1960 // we have 32 float register * 8 halves
1961 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1962 if (reg < slots_of_int_registers + slots_of_float_registers) {
1963 return rc_float;
1964 }
1965
1966 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1967 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1968 return rc_predicate;
1969 }
1970
1971 // Between predicate regs & stack is the flags.
1972 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1973
1974 return rc_stack;
1975 }
1976
1977 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1978 Compile* C = ra_->C;
1979
1980 // Get registers to move.
1981 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1982 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1983 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1984 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1985
1986 enum RC src_hi_rc = rc_class(src_hi);
1987 enum RC src_lo_rc = rc_class(src_lo);
1988 enum RC dst_hi_rc = rc_class(dst_hi);
1989 enum RC dst_lo_rc = rc_class(dst_lo);
1990
1991 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1992
1993 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1994 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1995 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1996 "expected aligned-adjacent pairs");
1997 }
1998
1999 if (src_lo == dst_lo && src_hi == dst_hi) {
2000 return 0; // Self copy, no move.
2001 }
2002
2003 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
2004 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
2005 int src_offset = ra_->reg2offset(src_lo);
2006 int dst_offset = ra_->reg2offset(dst_lo);
2007
2008 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2009 uint ireg = ideal_reg();
2010 if (ireg == Op_VecA && masm) {
2011 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
2012 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2013 // stack->stack
2014 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
2015 sve_vector_reg_size_in_bytes);
2016 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2017 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2018 sve_vector_reg_size_in_bytes);
2019 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2020 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2021 sve_vector_reg_size_in_bytes);
2022 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2023 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2024 as_FloatRegister(Matcher::_regEncode[src_lo]),
2025 as_FloatRegister(Matcher::_regEncode[src_lo]));
2026 } else {
2027 ShouldNotReachHere();
2028 }
2029 } else if (masm) {
2030 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
2031 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
2032 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
2033 // stack->stack
2034 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2035 if (ireg == Op_VecD) {
2036 __ unspill(rscratch1, true, src_offset);
2037 __ spill(rscratch1, true, dst_offset);
2038 } else {
2039 __ spill_copy128(src_offset, dst_offset);
2040 }
2041 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2042 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2043 ireg == Op_VecD ? __ T8B : __ T16B,
2044 as_FloatRegister(Matcher::_regEncode[src_lo]));
2045 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2046 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2047 ireg == Op_VecD ? __ D : __ Q,
2048 ra_->reg2offset(dst_lo));
2049 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2050 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2051 ireg == Op_VecD ? __ D : __ Q,
2052 ra_->reg2offset(src_lo));
2053 } else {
2054 ShouldNotReachHere();
2055 }
2056 }
2057 } else if (masm) {
2058 switch (src_lo_rc) {
2059 case rc_int:
2060 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2061 if (is64) {
2062 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2063 as_Register(Matcher::_regEncode[src_lo]));
2064 } else {
2065 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2066 as_Register(Matcher::_regEncode[src_lo]));
2067 }
2068 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2069 if (is64) {
2070 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2071 as_Register(Matcher::_regEncode[src_lo]));
2072 } else {
2073 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2074 as_Register(Matcher::_regEncode[src_lo]));
2075 }
2076 } else { // gpr --> stack spill
2077 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2078 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2079 }
2080 break;
2081 case rc_float:
2082 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2083 if (is64) {
2084 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2085 as_FloatRegister(Matcher::_regEncode[src_lo]));
2086 } else {
2087 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2088 as_FloatRegister(Matcher::_regEncode[src_lo]));
2089 }
2090 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2091 if (is64) {
2092 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2093 as_FloatRegister(Matcher::_regEncode[src_lo]));
2094 } else {
2095 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2096 as_FloatRegister(Matcher::_regEncode[src_lo]));
2097 }
2098 } else { // fpr --> stack spill
2099 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2100 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2101 is64 ? __ D : __ S, dst_offset);
2102 }
2103 break;
2104 case rc_stack:
2105 if (dst_lo_rc == rc_int) { // stack --> gpr load
2106 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2107 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2108 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2109 is64 ? __ D : __ S, src_offset);
2110 } else if (dst_lo_rc == rc_predicate) {
2111 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2112 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2113 } else { // stack --> stack copy
2114 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2115 if (ideal_reg() == Op_RegVectMask) {
2116 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2117 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2118 } else {
2119 __ unspill(rscratch1, is64, src_offset);
2120 __ spill(rscratch1, is64, dst_offset);
2121 }
2122 }
2123 break;
2124 case rc_predicate:
2125 if (dst_lo_rc == rc_predicate) {
2126 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2127 } else if (dst_lo_rc == rc_stack) {
2128 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2129 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2130 } else {
2131 assert(false, "bad src and dst rc_class combination.");
2132 ShouldNotReachHere();
2133 }
2134 break;
2135 default:
2136 assert(false, "bad rc_class for spill");
2137 ShouldNotReachHere();
2138 }
2139 }
2140
2141 if (st) {
2142 st->print("spill ");
2143 if (src_lo_rc == rc_stack) {
2144 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2145 } else {
2146 st->print("%s -> ", Matcher::regName[src_lo]);
2147 }
2148 if (dst_lo_rc == rc_stack) {
2149 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2150 } else {
2151 st->print("%s", Matcher::regName[dst_lo]);
2152 }
2153 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2154 int vsize = 0;
2155 switch (ideal_reg()) {
2156 case Op_VecD:
2157 vsize = 64;
2158 break;
2159 case Op_VecX:
2160 vsize = 128;
2161 break;
2162 case Op_VecA:
2163 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2164 break;
2165 default:
2166 assert(false, "bad register type for spill");
2167 ShouldNotReachHere();
2168 }
2169 st->print("\t# vector spill size = %d", vsize);
2170 } else if (ideal_reg() == Op_RegVectMask) {
2171 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2172 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2173 st->print("\t# predicate spill size = %d", vsize);
2174 } else {
2175 st->print("\t# spill size = %d", is64 ? 64 : 32);
2176 }
2177 }
2178
2179 return 0;
2180
2181 }
2182
2183 #ifndef PRODUCT
2184 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2185 if (!ra_)
2186 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2187 else
2188 implementation(nullptr, ra_, false, st);
2189 }
2190 #endif
2191
2192 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2193 implementation(masm, ra_, false, nullptr);
2194 }
2195
2196 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2197 return MachNode::size(ra_);
2198 }
2199
2200 //=============================================================================
2201
2202 #ifndef PRODUCT
2203 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2204 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2205 int reg = ra_->get_reg_first(this);
2206 st->print("add %s, rsp, #%d]\t# box lock",
2207 Matcher::regName[reg], offset);
2208 }
2209 #endif
2210
2211 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2212 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2213 int reg = ra_->get_encode(this);
2214
2215 // This add will handle any 24-bit signed offset. 24 bits allows an
2216 // 8 megabyte stack frame.
2217 __ add(as_Register(reg), sp, offset);
2218 }
2219
2220 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2221 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2222 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2223
2224 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2225 return NativeInstruction::instruction_size;
2226 } else {
2227 return 2 * NativeInstruction::instruction_size;
2228 }
2229 }
2230
2231 //=============================================================================
2232
2233 #ifndef PRODUCT
2234 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2235 {
2236 st->print_cr("# MachUEPNode");
2237 if (UseCompressedClassPointers) {
2238 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2239 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2240 st->print_cr("\tcmpw rscratch1, r10");
2241 } else {
2242 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2243 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2244 st->print_cr("\tcmp rscratch1, r10");
2245 }
2246 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2247 }
2248 #endif
2249
2250 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2251 {
2252 __ ic_check(InteriorEntryAlignment);
2253 }
2254
2255 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2256 {
2257 return MachNode::size(ra_);
2258 }
2259
2260 // REQUIRED EMIT CODE
2261
2262 //=============================================================================
2263
2264 // Emit exception handler code.
2265 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2266 {
2267 // mov rscratch1 #exception_blob_entry_point
2268 // br rscratch1
2269 // Note that the code buffer's insts_mark is always relative to insts.
2270 // That's why we must use the macroassembler to generate a handler.
2271 address base = __ start_a_stub(size_exception_handler());
2272 if (base == nullptr) {
2273 ciEnv::current()->record_failure("CodeCache is full");
2274 return 0; // CodeBuffer::expand failed
2275 }
2276 int offset = __ offset();
2277 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2278 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2279 __ end_a_stub();
2280 return offset;
2281 }
2282
2283 // Emit deopt handler code.
2284 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2285 {
2286 // Note that the code buffer's insts_mark is always relative to insts.
2287 // That's why we must use the macroassembler to generate a handler.
2288 address base = __ start_a_stub(size_deopt_handler());
2289 if (base == nullptr) {
2290 ciEnv::current()->record_failure("CodeCache is full");
2291 return 0; // CodeBuffer::expand failed
2292 }
2293 int offset = __ offset();
2294
2295 __ adr(lr, __ pc());
2296 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2297
2298 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2299 __ end_a_stub();
2300 return offset;
2301 }
2302
2303 // REQUIRED MATCHER CODE
2304
2305 //=============================================================================
2306
2307 bool Matcher::match_rule_supported(int opcode) {
2308 if (!has_match_rule(opcode))
2309 return false;
2310
2311 switch (opcode) {
2312 case Op_OnSpinWait:
2313 return VM_Version::supports_on_spin_wait();
2314 case Op_CacheWB:
2315 case Op_CacheWBPreSync:
2316 case Op_CacheWBPostSync:
2317 if (!VM_Version::supports_data_cache_line_flush()) {
2318 return false;
2319 }
2320 break;
2321 case Op_ExpandBits:
2322 case Op_CompressBits:
2323 if (!VM_Version::supports_svebitperm()) {
2324 return false;
2325 }
2326 break;
2327 case Op_FmaF:
2328 case Op_FmaD:
2329 case Op_FmaVF:
2330 case Op_FmaVD:
2331 if (!UseFMA) {
2332 return false;
2333 }
2334 break;
2335 case Op_FmaHF:
2336 // UseFMA flag also needs to be checked along with FEAT_FP16
2337 if (!UseFMA || !is_feat_fp16_supported()) {
2338 return false;
2339 }
2340 break;
2341 case Op_AddHF:
2342 case Op_SubHF:
2343 case Op_MulHF:
2344 case Op_DivHF:
2345 case Op_MinHF:
2346 case Op_MaxHF:
2347 case Op_SqrtHF:
2348 // Half-precision floating point scalar operations require FEAT_FP16
2349 // to be available. FEAT_FP16 is enabled if both "fphp" and "asimdhp"
2350 // features are supported.
2351 if (!is_feat_fp16_supported()) {
2352 return false;
2353 }
2354 break;
2355 }
2356
2357 return true; // Per default match rules are supported.
2358 }
2359
2360 const RegMask* Matcher::predicate_reg_mask(void) {
2361 return &_PR_REG_mask;
2362 }
2363
2364 bool Matcher::supports_vector_calling_convention(void) {
2365 return EnableVectorSupport;
2366 }
2367
2368 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2369 assert(EnableVectorSupport, "sanity");
2370 int lo = V0_num;
2371 int hi = V0_H_num;
2372 if (ideal_reg == Op_VecX || ideal_reg == Op_VecA) {
2373 hi = V0_K_num;
2374 }
2375 return OptoRegPair(hi, lo);
2376 }
2377
2378 // Is this branch offset short enough that a short branch can be used?
2379 //
2380 // NOTE: If the platform does not provide any short branch variants, then
2381 // this method should return false for offset 0.
2382 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2383 // The passed offset is relative to address of the branch.
2384
2385 return (-32768 <= offset && offset < 32768);
2386 }
2387
2388 // Vector width in bytes.
2389 int Matcher::vector_width_in_bytes(BasicType bt) {
2390 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2391 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2392 // Minimum 2 values in vector
2393 if (size < 2*type2aelembytes(bt)) size = 0;
2394 // But never < 4
2395 if (size < 4) size = 0;
2396 return size;
2397 }
2398
2399 // Limits on vector size (number of elements) loaded into vector.
2400 int Matcher::max_vector_size(const BasicType bt) {
2401 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2402 }
2403
2404 int Matcher::min_vector_size(const BasicType bt) {
2405 // Usually, the shortest vector length supported by AArch64 ISA and
2406 // Vector API species is 64 bits. However, we allow 32-bit or 16-bit
2407 // vectors in a few special cases.
2408 int size;
2409 switch(bt) {
2410 case T_BOOLEAN:
2411 // Load/store a vector mask with only 2 elements for vector types
2412 // such as "2I/2F/2L/2D".
2413 size = 2;
2414 break;
2415 case T_BYTE:
2416 // Generate a "4B" vector, to support vector cast between "8B/16B"
2417 // and "4S/4I/4L/4F/4D".
2418 size = 4;
2419 break;
2420 case T_SHORT:
2421 // Generate a "2S" vector, to support vector cast between "4S/8S"
2422 // and "2I/2L/2F/2D".
2423 size = 2;
2424 break;
2425 default:
2426 // Limit the min vector length to 64-bit.
2427 size = 8 / type2aelembytes(bt);
2428 // The number of elements in a vector should be at least 2.
2429 size = MAX2(size, 2);
2430 }
2431
2432 int max_size = max_vector_size(bt);
2433 return MIN2(size, max_size);
2434 }
2435
2436 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2437 return Matcher::max_vector_size(bt);
2438 }
2439
2440 // Actual max scalable vector register length.
2441 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2442 return Matcher::max_vector_size(bt);
2443 }
2444
2445 // Vector ideal reg.
2446 uint Matcher::vector_ideal_reg(int len) {
2447 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2448 return Op_VecA;
2449 }
2450 switch(len) {
2451 // For 16-bit/32-bit mask vector, reuse VecD.
2452 case 2:
2453 case 4:
2454 case 8: return Op_VecD;
2455 case 16: return Op_VecX;
2456 }
2457 ShouldNotReachHere();
2458 return 0;
2459 }
2460
2461 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2462 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2463 switch (ideal_reg) {
2464 case Op_VecA: return new vecAOper();
2465 case Op_VecD: return new vecDOper();
2466 case Op_VecX: return new vecXOper();
2467 }
2468 ShouldNotReachHere();
2469 return nullptr;
2470 }
2471
2472 bool Matcher::is_reg2reg_move(MachNode* m) {
2473 return false;
2474 }
2475
2476 bool Matcher::is_generic_vector(MachOper* opnd) {
2477 return opnd->opcode() == VREG;
2478 }
2479
2480 // Return whether or not this register is ever used as an argument.
2481 // This function is used on startup to build the trampoline stubs in
2482 // generateOptoStub. Registers not mentioned will be killed by the VM
2483 // call in the trampoline, and arguments in those registers not be
2484 // available to the callee.
2485 bool Matcher::can_be_java_arg(int reg)
2486 {
2487 return
2488 reg == R0_num || reg == R0_H_num ||
2489 reg == R1_num || reg == R1_H_num ||
2490 reg == R2_num || reg == R2_H_num ||
2491 reg == R3_num || reg == R3_H_num ||
2492 reg == R4_num || reg == R4_H_num ||
2493 reg == R5_num || reg == R5_H_num ||
2494 reg == R6_num || reg == R6_H_num ||
2495 reg == R7_num || reg == R7_H_num ||
2496 reg == V0_num || reg == V0_H_num ||
2497 reg == V1_num || reg == V1_H_num ||
2498 reg == V2_num || reg == V2_H_num ||
2499 reg == V3_num || reg == V3_H_num ||
2500 reg == V4_num || reg == V4_H_num ||
2501 reg == V5_num || reg == V5_H_num ||
2502 reg == V6_num || reg == V6_H_num ||
2503 reg == V7_num || reg == V7_H_num;
2504 }
2505
2506 bool Matcher::is_spillable_arg(int reg)
2507 {
2508 return can_be_java_arg(reg);
2509 }
2510
2511 uint Matcher::int_pressure_limit()
2512 {
2513 // JDK-8183543: When taking the number of available registers as int
2514 // register pressure threshold, the jtreg test:
2515 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2516 // failed due to C2 compilation failure with
2517 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2518 //
2519 // A derived pointer is live at CallNode and then is flagged by RA
2520 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2521 // derived pointers and lastly fail to spill after reaching maximum
2522 // number of iterations. Lowering the default pressure threshold to
2523 // (_NO_SPECIAL_REG32_mask.size() minus 1) forces CallNode to become
2524 // a high register pressure area of the code so that split_DEF can
2525 // generate DefinitionSpillCopy for the derived pointer.
2526 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.size() - 1;
2527 if (!PreserveFramePointer) {
2528 // When PreserveFramePointer is off, frame pointer is allocatable,
2529 // but different from other SOC registers, it is excluded from
2530 // fatproj's mask because its save type is No-Save. Decrease 1 to
2531 // ensure high pressure at fatproj when PreserveFramePointer is off.
2532 // See check_pressure_at_fatproj().
2533 default_int_pressure_threshold--;
2534 }
2535 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2536 }
2537
2538 uint Matcher::float_pressure_limit()
2539 {
2540 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2541 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.size() : FLOATPRESSURE;
2542 }
2543
2544 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2545 return false;
2546 }
2547
2548 const RegMask& Matcher::divI_proj_mask() {
2549 ShouldNotReachHere();
2550 return RegMask::EMPTY;
2551 }
2552
2553 // Register for MODI projection of divmodI.
2554 const RegMask& Matcher::modI_proj_mask() {
2555 ShouldNotReachHere();
2556 return RegMask::EMPTY;
2557 }
2558
2559 // Register for DIVL projection of divmodL.
2560 const RegMask& Matcher::divL_proj_mask() {
2561 ShouldNotReachHere();
2562 return RegMask::EMPTY;
2563 }
2564
2565 // Register for MODL projection of divmodL.
2566 const RegMask& Matcher::modL_proj_mask() {
2567 ShouldNotReachHere();
2568 return RegMask::EMPTY;
2569 }
2570
2571 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2572 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2573 Node* u = addp->fast_out(i);
2574 if (u->is_LoadStore()) {
2575 // On AArch64, LoadStoreNodes (i.e. compare and swap
2576 // instructions) only take register indirect as an operand, so
2577 // any attempt to use an AddPNode as an input to a LoadStoreNode
2578 // must fail.
2579 return false;
2580 }
2581 if (u->is_Mem()) {
2582 int opsize = u->as_Mem()->memory_size();
2583 assert(opsize > 0, "unexpected memory operand size");
2584 if (u->as_Mem()->memory_size() != (1<<shift)) {
2585 return false;
2586 }
2587 }
2588 }
2589 return true;
2590 }
2591
2592 // Convert BootTest condition to Assembler condition.
2593 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2594 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2595 Assembler::Condition result;
2596 switch(cond) {
2597 case BoolTest::eq:
2598 result = Assembler::EQ; break;
2599 case BoolTest::ne:
2600 result = Assembler::NE; break;
2601 case BoolTest::le:
2602 result = Assembler::LE; break;
2603 case BoolTest::ge:
2604 result = Assembler::GE; break;
2605 case BoolTest::lt:
2606 result = Assembler::LT; break;
2607 case BoolTest::gt:
2608 result = Assembler::GT; break;
2609 case BoolTest::ule:
2610 result = Assembler::LS; break;
2611 case BoolTest::uge:
2612 result = Assembler::HS; break;
2613 case BoolTest::ult:
2614 result = Assembler::LO; break;
2615 case BoolTest::ugt:
2616 result = Assembler::HI; break;
2617 case BoolTest::overflow:
2618 result = Assembler::VS; break;
2619 case BoolTest::no_overflow:
2620 result = Assembler::VC; break;
2621 default:
2622 ShouldNotReachHere();
2623 return Assembler::Condition(-1);
2624 }
2625
2626 // Check conversion
2627 if (cond & BoolTest::unsigned_compare) {
2628 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2629 } else {
2630 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2631 }
2632
2633 return result;
2634 }
2635
2636 // Binary src (Replicate con)
2637 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2638 if (n == nullptr || m == nullptr) {
2639 return false;
2640 }
2641
2642 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2643 return false;
2644 }
2645
2646 Node* imm_node = m->in(1);
2647 if (!imm_node->is_Con()) {
2648 return false;
2649 }
2650
2651 const Type* t = imm_node->bottom_type();
2652 if (!(t->isa_int() || t->isa_long())) {
2653 return false;
2654 }
2655
2656 switch (n->Opcode()) {
2657 case Op_AndV:
2658 case Op_OrV:
2659 case Op_XorV: {
2660 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2661 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2662 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2663 }
2664 case Op_AddVB:
2665 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2666 case Op_AddVS:
2667 case Op_AddVI:
2668 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2669 case Op_AddVL:
2670 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2671 default:
2672 return false;
2673 }
2674 }
2675
2676 // (XorV src (Replicate m1))
2677 // (XorVMask src (MaskAll m1))
2678 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2679 if (n != nullptr && m != nullptr) {
2680 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2681 VectorNode::is_all_ones_vector(m);
2682 }
2683 return false;
2684 }
2685
2686 // Should the matcher clone input 'm' of node 'n'?
2687 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2688 if (is_vshift_con_pattern(n, m) ||
2689 is_vector_bitwise_not_pattern(n, m) ||
2690 is_valid_sve_arith_imm_pattern(n, m) ||
2691 is_encode_and_store_pattern(n, m)) {
2692 mstack.push(m, Visit);
2693 return true;
2694 }
2695 return false;
2696 }
2697
2698 // Should the Matcher clone shifts on addressing modes, expecting them
2699 // to be subsumed into complex addressing expressions or compute them
2700 // into registers?
2701 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2702
2703 // Loads and stores with indirect memory input (e.g., volatile loads and
2704 // stores) do not subsume the input into complex addressing expressions. If
2705 // the addressing expression is input to at least one such load or store, do
2706 // not clone the addressing expression. Query needs_acquiring_load and
2707 // needs_releasing_store as a proxy for indirect memory input, as it is not
2708 // possible to directly query for indirect memory input at this stage.
2709 for (DUIterator_Fast imax, i = m->fast_outs(imax); i < imax; i++) {
2710 Node* n = m->fast_out(i);
2711 if (n->is_Load() && needs_acquiring_load(n)) {
2712 return false;
2713 }
2714 if (n->is_Store() && needs_releasing_store(n)) {
2715 return false;
2716 }
2717 }
2718
2719 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2720 return true;
2721 }
2722
2723 Node *off = m->in(AddPNode::Offset);
2724 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2725 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2726 // Are there other uses besides address expressions?
2727 !is_visited(off)) {
2728 address_visited.set(off->_idx); // Flag as address_visited
2729 mstack.push(off->in(2), Visit);
2730 Node *conv = off->in(1);
2731 if (conv->Opcode() == Op_ConvI2L &&
2732 // Are there other uses besides address expressions?
2733 !is_visited(conv)) {
2734 address_visited.set(conv->_idx); // Flag as address_visited
2735 mstack.push(conv->in(1), Pre_Visit);
2736 } else {
2737 mstack.push(conv, Pre_Visit);
2738 }
2739 address_visited.test_set(m->_idx); // Flag as address_visited
2740 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2741 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2742 return true;
2743 } else if (off->Opcode() == Op_ConvI2L &&
2744 // Are there other uses besides address expressions?
2745 !is_visited(off)) {
2746 address_visited.test_set(m->_idx); // Flag as address_visited
2747 address_visited.set(off->_idx); // Flag as address_visited
2748 mstack.push(off->in(1), Pre_Visit);
2749 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2750 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2751 return true;
2752 }
2753 return false;
2754 }
2755
2756 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2757 { \
2758 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2759 guarantee(DISP == 0, "mode not permitted for volatile"); \
2760 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2761 __ INSN(REG, as_Register(BASE)); \
2762 }
2763
2764
2765 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2766 {
2767 Address::extend scale;
2768
2769 // Hooboy, this is fugly. We need a way to communicate to the
2770 // encoder that the index needs to be sign extended, so we have to
2771 // enumerate all the cases.
2772 switch (opcode) {
2773 case INDINDEXSCALEDI2L:
2774 case INDINDEXSCALEDI2LN:
2775 case INDINDEXI2L:
2776 case INDINDEXI2LN:
2777 scale = Address::sxtw(size);
2778 break;
2779 default:
2780 scale = Address::lsl(size);
2781 }
2782
2783 if (index == -1) {
2784 return Address(base, disp);
2785 } else {
2786 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2787 return Address(base, as_Register(index), scale);
2788 }
2789 }
2790
2791
2792 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2793 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2794 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2795 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2796 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2797
2798 // Used for all non-volatile memory accesses. The use of
2799 // $mem->opcode() to discover whether this pattern uses sign-extended
2800 // offsets is something of a kludge.
2801 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2802 Register reg, int opcode,
2803 Register base, int index, int scale, int disp,
2804 int size_in_memory)
2805 {
2806 Address addr = mem2address(opcode, base, index, scale, disp);
2807 if (addr.getMode() == Address::base_plus_offset) {
2808 /* Fix up any out-of-range offsets. */
2809 assert_different_registers(rscratch1, base);
2810 assert_different_registers(rscratch1, reg);
2811 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2812 }
2813 (masm->*insn)(reg, addr);
2814 }
2815
2816 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2817 FloatRegister reg, int opcode,
2818 Register base, int index, int size, int disp,
2819 int size_in_memory)
2820 {
2821 Address::extend scale;
2822
2823 switch (opcode) {
2824 case INDINDEXSCALEDI2L:
2825 case INDINDEXSCALEDI2LN:
2826 scale = Address::sxtw(size);
2827 break;
2828 default:
2829 scale = Address::lsl(size);
2830 }
2831
2832 if (index == -1) {
2833 // Fix up any out-of-range offsets.
2834 assert_different_registers(rscratch1, base);
2835 Address addr = Address(base, disp);
2836 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2837 (masm->*insn)(reg, addr);
2838 } else {
2839 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2840 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2841 }
2842 }
2843
2844 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2845 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2846 int opcode, Register base, int index, int size, int disp)
2847 {
2848 if (index == -1) {
2849 (masm->*insn)(reg, T, Address(base, disp));
2850 } else {
2851 assert(disp == 0, "unsupported address mode");
2852 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2853 }
2854 }
2855
2856 %}
2857
2858
2859
2860 //----------ENCODING BLOCK-----------------------------------------------------
2861 // This block specifies the encoding classes used by the compiler to
2862 // output byte streams. Encoding classes are parameterized macros
2863 // used by Machine Instruction Nodes in order to generate the bit
2864 // encoding of the instruction. Operands specify their base encoding
2865 // interface with the interface keyword. There are currently
2866 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2867 // COND_INTER. REG_INTER causes an operand to generate a function
2868 // which returns its register number when queried. CONST_INTER causes
2869 // an operand to generate a function which returns the value of the
2870 // constant when queried. MEMORY_INTER causes an operand to generate
2871 // four functions which return the Base Register, the Index Register,
2872 // the Scale Value, and the Offset Value of the operand when queried.
2873 // COND_INTER causes an operand to generate six functions which return
2874 // the encoding code (ie - encoding bits for the instruction)
2875 // associated with each basic boolean condition for a conditional
2876 // instruction.
2877 //
2878 // Instructions specify two basic values for encoding. Again, a
2879 // function is available to check if the constant displacement is an
2880 // oop. They use the ins_encode keyword to specify their encoding
2881 // classes (which must be a sequence of enc_class names, and their
2882 // parameters, specified in the encoding block), and they use the
2883 // opcode keyword to specify, in order, their primary, secondary, and
2884 // tertiary opcode. Only the opcode sections which a particular
2885 // instruction needs for encoding need to be specified.
2886 encode %{
2887 // Build emit functions for each basic byte or larger field in the
2888 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2889 // from C++ code in the enc_class source block. Emit functions will
2890 // live in the main source block for now. In future, we can
2891 // generalize this by adding a syntax that specifies the sizes of
2892 // fields in an order, so that the adlc can build the emit functions
2893 // automagically
2894
2895 // catch all for unimplemented encodings
2896 enc_class enc_unimplemented %{
2897 __ unimplemented("C2 catch all");
2898 %}
2899
2900 // BEGIN Non-volatile memory access
2901
2902 // This encoding class is generated automatically from ad_encode.m4.
2903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2904 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2905 Register dst_reg = as_Register($dst$$reg);
2906 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2907 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2908 %}
2909
2910 // This encoding class is generated automatically from ad_encode.m4.
2911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2912 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2913 Register dst_reg = as_Register($dst$$reg);
2914 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2915 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2916 %}
2917
2918 // This encoding class is generated automatically from ad_encode.m4.
2919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2920 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2921 Register dst_reg = as_Register($dst$$reg);
2922 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2923 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2924 %}
2925
2926 // This encoding class is generated automatically from ad_encode.m4.
2927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2928 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2929 Register dst_reg = as_Register($dst$$reg);
2930 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2931 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2932 %}
2933
2934 // This encoding class is generated automatically from ad_encode.m4.
2935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2936 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2937 Register dst_reg = as_Register($dst$$reg);
2938 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2939 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2940 %}
2941
2942 // This encoding class is generated automatically from ad_encode.m4.
2943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2944 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2945 Register dst_reg = as_Register($dst$$reg);
2946 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2947 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2948 %}
2949
2950 // This encoding class is generated automatically from ad_encode.m4.
2951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2952 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2953 Register dst_reg = as_Register($dst$$reg);
2954 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2955 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2956 %}
2957
2958 // This encoding class is generated automatically from ad_encode.m4.
2959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2960 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2961 Register dst_reg = as_Register($dst$$reg);
2962 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2963 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2964 %}
2965
2966 // This encoding class is generated automatically from ad_encode.m4.
2967 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2968 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2969 Register dst_reg = as_Register($dst$$reg);
2970 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2971 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2972 %}
2973
2974 // This encoding class is generated automatically from ad_encode.m4.
2975 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2976 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2977 Register dst_reg = as_Register($dst$$reg);
2978 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2979 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2980 %}
2981
2982 // This encoding class is generated automatically from ad_encode.m4.
2983 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2984 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2985 Register dst_reg = as_Register($dst$$reg);
2986 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2987 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2988 %}
2989
2990 // This encoding class is generated automatically from ad_encode.m4.
2991 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2992 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2993 Register dst_reg = as_Register($dst$$reg);
2994 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2995 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2996 %}
2997
2998 // This encoding class is generated automatically from ad_encode.m4.
2999 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3000 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
3001 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3002 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
3003 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3004 %}
3005
3006 // This encoding class is generated automatically from ad_encode.m4.
3007 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3008 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
3009 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3010 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
3011 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3012 %}
3013
3014 // This encoding class is generated automatically from ad_encode.m4.
3015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3016 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
3017 Register src_reg = as_Register($src$$reg);
3018 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
3019 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3020 %}
3021
3022 // This encoding class is generated automatically from ad_encode.m4.
3023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3024 enc_class aarch64_enc_strb0(memory1 mem) %{
3025 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3026 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3027 %}
3028
3029 // This encoding class is generated automatically from ad_encode.m4.
3030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3031 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
3032 Register src_reg = as_Register($src$$reg);
3033 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
3034 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3035 %}
3036
3037 // This encoding class is generated automatically from ad_encode.m4.
3038 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3039 enc_class aarch64_enc_strh0(memory2 mem) %{
3040 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
3041 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
3042 %}
3043
3044 // This encoding class is generated automatically from ad_encode.m4.
3045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3046 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
3047 Register src_reg = as_Register($src$$reg);
3048 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
3049 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3050 %}
3051
3052 // This encoding class is generated automatically from ad_encode.m4.
3053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3054 enc_class aarch64_enc_strw0(memory4 mem) %{
3055 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
3056 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3057 %}
3058
3059 // This encoding class is generated automatically from ad_encode.m4.
3060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3061 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
3062 Register src_reg = as_Register($src$$reg);
3063 // we sometimes get asked to store the stack pointer into the
3064 // current thread -- we cannot do that directly on AArch64
3065 if (src_reg == r31_sp) {
3066 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3067 __ mov(rscratch2, sp);
3068 src_reg = rscratch2;
3069 }
3070 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
3071 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3072 %}
3073
3074 // This encoding class is generated automatically from ad_encode.m4.
3075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3076 enc_class aarch64_enc_str0(memory8 mem) %{
3077 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
3078 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3079 %}
3080
3081 // This encoding class is generated automatically from ad_encode.m4.
3082 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3083 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3084 FloatRegister src_reg = as_FloatRegister($src$$reg);
3085 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3086 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3087 %}
3088
3089 // This encoding class is generated automatically from ad_encode.m4.
3090 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3091 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3092 FloatRegister src_reg = as_FloatRegister($src$$reg);
3093 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3094 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3095 %}
3096
3097 // This encoding class is generated automatically from ad_encode.m4.
3098 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3099 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3100 __ membar(Assembler::StoreStore);
3101 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3102 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3103 %}
3104
3105 // END Non-volatile memory access
3106
3107 // Vector loads and stores
3108 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3109 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3110 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3111 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3112 %}
3113
3114 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3115 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3116 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3117 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3118 %}
3119
3120 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3121 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3122 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3123 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3124 %}
3125
3126 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3127 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3128 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3129 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3130 %}
3131
3132 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3133 FloatRegister src_reg = as_FloatRegister($src$$reg);
3134 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3135 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3136 %}
3137
3138 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3139 FloatRegister src_reg = as_FloatRegister($src$$reg);
3140 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3141 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3142 %}
3143
3144 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3145 FloatRegister src_reg = as_FloatRegister($src$$reg);
3146 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3147 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3148 %}
3149
3150 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3151 FloatRegister src_reg = as_FloatRegister($src$$reg);
3152 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3153 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3154 %}
3155
3156 // volatile loads and stores
3157
3158 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3159 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, stlrb);
3161 %}
3162
3163 enc_class aarch64_enc_stlrb0(memory mem) %{
3164 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3165 rscratch1, stlrb);
3166 %}
3167
3168 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3169 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3170 rscratch1, stlrh);
3171 %}
3172
3173 enc_class aarch64_enc_stlrh0(memory mem) %{
3174 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3175 rscratch1, stlrh);
3176 %}
3177
3178 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3179 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3180 rscratch1, stlrw);
3181 %}
3182
3183 enc_class aarch64_enc_stlrw0(memory mem) %{
3184 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3185 rscratch1, stlrw);
3186 %}
3187
3188 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3189 Register dst_reg = as_Register($dst$$reg);
3190 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3191 rscratch1, ldarb);
3192 __ sxtbw(dst_reg, dst_reg);
3193 %}
3194
3195 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3196 Register dst_reg = as_Register($dst$$reg);
3197 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3198 rscratch1, ldarb);
3199 __ sxtb(dst_reg, dst_reg);
3200 %}
3201
3202 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3203 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3204 rscratch1, ldarb);
3205 %}
3206
3207 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3208 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3209 rscratch1, ldarb);
3210 %}
3211
3212 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3213 Register dst_reg = as_Register($dst$$reg);
3214 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3215 rscratch1, ldarh);
3216 __ sxthw(dst_reg, dst_reg);
3217 %}
3218
3219 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3220 Register dst_reg = as_Register($dst$$reg);
3221 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3222 rscratch1, ldarh);
3223 __ sxth(dst_reg, dst_reg);
3224 %}
3225
3226 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3227 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3228 rscratch1, ldarh);
3229 %}
3230
3231 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3232 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3233 rscratch1, ldarh);
3234 %}
3235
3236 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3237 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3238 rscratch1, ldarw);
3239 %}
3240
3241 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3242 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3243 rscratch1, ldarw);
3244 %}
3245
3246 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3247 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3248 rscratch1, ldar);
3249 %}
3250
3251 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3252 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3253 rscratch1, ldarw);
3254 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3255 %}
3256
3257 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3258 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3259 rscratch1, ldar);
3260 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3261 %}
3262
3263 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3264 Register src_reg = as_Register($src$$reg);
3265 // we sometimes get asked to store the stack pointer into the
3266 // current thread -- we cannot do that directly on AArch64
3267 if (src_reg == r31_sp) {
3268 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3269 __ mov(rscratch2, sp);
3270 src_reg = rscratch2;
3271 }
3272 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3273 rscratch1, stlr);
3274 %}
3275
3276 enc_class aarch64_enc_stlr0(memory mem) %{
3277 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3278 rscratch1, stlr);
3279 %}
3280
3281 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3282 {
3283 FloatRegister src_reg = as_FloatRegister($src$$reg);
3284 __ fmovs(rscratch2, src_reg);
3285 }
3286 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3287 rscratch1, stlrw);
3288 %}
3289
3290 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3291 {
3292 FloatRegister src_reg = as_FloatRegister($src$$reg);
3293 __ fmovd(rscratch2, src_reg);
3294 }
3295 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3296 rscratch1, stlr);
3297 %}
3298
3299 // synchronized read/update encodings
3300
3301 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3302 Register dst_reg = as_Register($dst$$reg);
3303 Register base = as_Register($mem$$base);
3304 int index = $mem$$index;
3305 int scale = $mem$$scale;
3306 int disp = $mem$$disp;
3307 if (index == -1) {
3308 if (disp != 0) {
3309 __ lea(rscratch1, Address(base, disp));
3310 __ ldaxr(dst_reg, rscratch1);
3311 } else {
3312 // TODO
3313 // should we ever get anything other than this case?
3314 __ ldaxr(dst_reg, base);
3315 }
3316 } else {
3317 Register index_reg = as_Register(index);
3318 if (disp == 0) {
3319 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3320 __ ldaxr(dst_reg, rscratch1);
3321 } else {
3322 __ lea(rscratch1, Address(base, disp));
3323 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3324 __ ldaxr(dst_reg, rscratch1);
3325 }
3326 }
3327 %}
3328
3329 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3330 Register src_reg = as_Register($src$$reg);
3331 Register base = as_Register($mem$$base);
3332 int index = $mem$$index;
3333 int scale = $mem$$scale;
3334 int disp = $mem$$disp;
3335 if (index == -1) {
3336 if (disp != 0) {
3337 __ lea(rscratch2, Address(base, disp));
3338 __ stlxr(rscratch1, src_reg, rscratch2);
3339 } else {
3340 // TODO
3341 // should we ever get anything other than this case?
3342 __ stlxr(rscratch1, src_reg, base);
3343 }
3344 } else {
3345 Register index_reg = as_Register(index);
3346 if (disp == 0) {
3347 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3348 __ stlxr(rscratch1, src_reg, rscratch2);
3349 } else {
3350 __ lea(rscratch2, Address(base, disp));
3351 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3352 __ stlxr(rscratch1, src_reg, rscratch2);
3353 }
3354 }
3355 __ cmpw(rscratch1, zr);
3356 %}
3357
3358 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3359 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3360 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3361 Assembler::xword, /*acquire*/ false, /*release*/ true,
3362 /*weak*/ false, noreg);
3363 %}
3364
3365 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3366 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3367 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3368 Assembler::word, /*acquire*/ false, /*release*/ true,
3369 /*weak*/ false, noreg);
3370 %}
3371
3372 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3373 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3374 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3375 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3376 /*weak*/ false, noreg);
3377 %}
3378
3379 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3380 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3381 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3382 Assembler::byte, /*acquire*/ false, /*release*/ true,
3383 /*weak*/ false, noreg);
3384 %}
3385
3386
3387 // The only difference between aarch64_enc_cmpxchg and
3388 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3389 // CompareAndSwap sequence to serve as a barrier on acquiring a
3390 // lock.
3391 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3392 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3393 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3394 Assembler::xword, /*acquire*/ true, /*release*/ true,
3395 /*weak*/ false, noreg);
3396 %}
3397
3398 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3399 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3400 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3401 Assembler::word, /*acquire*/ true, /*release*/ true,
3402 /*weak*/ false, noreg);
3403 %}
3404
3405 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3406 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3407 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3408 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3409 /*weak*/ false, noreg);
3410 %}
3411
3412 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3413 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3414 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3415 Assembler::byte, /*acquire*/ true, /*release*/ true,
3416 /*weak*/ false, noreg);
3417 %}
3418
3419 // auxiliary used for CompareAndSwapX to set result register
3420 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3421 Register res_reg = as_Register($res$$reg);
3422 __ cset(res_reg, Assembler::EQ);
3423 %}
3424
3425 // prefetch encodings
3426
3427 enc_class aarch64_enc_prefetchw(memory mem) %{
3428 Register base = as_Register($mem$$base);
3429 int index = $mem$$index;
3430 int scale = $mem$$scale;
3431 int disp = $mem$$disp;
3432 if (index == -1) {
3433 // Fix up any out-of-range offsets.
3434 assert_different_registers(rscratch1, base);
3435 Address addr = Address(base, disp);
3436 addr = __ legitimize_address(addr, 8, rscratch1);
3437 __ prfm(addr, PSTL1KEEP);
3438 } else {
3439 Register index_reg = as_Register(index);
3440 if (disp == 0) {
3441 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3442 } else {
3443 __ lea(rscratch1, Address(base, disp));
3444 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3445 }
3446 }
3447 %}
3448
3449 // mov encodings
3450
3451 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3452 uint32_t con = (uint32_t)$src$$constant;
3453 Register dst_reg = as_Register($dst$$reg);
3454 if (con == 0) {
3455 __ movw(dst_reg, zr);
3456 } else {
3457 __ movw(dst_reg, con);
3458 }
3459 %}
3460
3461 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3462 Register dst_reg = as_Register($dst$$reg);
3463 uint64_t con = (uint64_t)$src$$constant;
3464 if (con == 0) {
3465 __ mov(dst_reg, zr);
3466 } else {
3467 __ mov(dst_reg, con);
3468 }
3469 %}
3470
3471 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3472 Register dst_reg = as_Register($dst$$reg);
3473 address con = (address)$src$$constant;
3474 if (con == nullptr || con == (address)1) {
3475 ShouldNotReachHere();
3476 } else {
3477 relocInfo::relocType rtype = $src->constant_reloc();
3478 if (rtype == relocInfo::oop_type) {
3479 __ movoop(dst_reg, (jobject)con);
3480 } else if (rtype == relocInfo::metadata_type) {
3481 __ mov_metadata(dst_reg, (Metadata*)con);
3482 } else {
3483 assert(rtype == relocInfo::none || rtype == relocInfo::external_word_type, "unexpected reloc type");
3484 if (! __ is_valid_AArch64_address(con) ||
3485 con < (address)(uintptr_t)os::vm_page_size()) {
3486 __ mov(dst_reg, con);
3487 } else {
3488 uint64_t offset;
3489 __ adrp(dst_reg, con, offset);
3490 __ add(dst_reg, dst_reg, offset);
3491 }
3492 }
3493 }
3494 %}
3495
3496 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3497 Register dst_reg = as_Register($dst$$reg);
3498 __ mov(dst_reg, zr);
3499 %}
3500
3501 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3502 Register dst_reg = as_Register($dst$$reg);
3503 __ mov(dst_reg, (uint64_t)1);
3504 %}
3505
3506 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3507 Register dst_reg = as_Register($dst$$reg);
3508 address con = (address)$src$$constant;
3509 if (con == nullptr) {
3510 ShouldNotReachHere();
3511 } else {
3512 relocInfo::relocType rtype = $src->constant_reloc();
3513 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3514 __ set_narrow_oop(dst_reg, (jobject)con);
3515 }
3516 %}
3517
3518 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3519 Register dst_reg = as_Register($dst$$reg);
3520 __ mov(dst_reg, zr);
3521 %}
3522
3523 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3524 Register dst_reg = as_Register($dst$$reg);
3525 address con = (address)$src$$constant;
3526 if (con == nullptr) {
3527 ShouldNotReachHere();
3528 } else {
3529 relocInfo::relocType rtype = $src->constant_reloc();
3530 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3531 __ set_narrow_klass(dst_reg, (Klass *)con);
3532 }
3533 %}
3534
3535 // arithmetic encodings
3536
3537 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3538 Register dst_reg = as_Register($dst$$reg);
3539 Register src_reg = as_Register($src1$$reg);
3540 int32_t con = (int32_t)$src2$$constant;
3541 // add has primary == 0, subtract has primary == 1
3542 if ($primary) { con = -con; }
3543 if (con < 0) {
3544 __ subw(dst_reg, src_reg, -con);
3545 } else {
3546 __ addw(dst_reg, src_reg, con);
3547 }
3548 %}
3549
3550 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3551 Register dst_reg = as_Register($dst$$reg);
3552 Register src_reg = as_Register($src1$$reg);
3553 int32_t con = (int32_t)$src2$$constant;
3554 // add has primary == 0, subtract has primary == 1
3555 if ($primary) { con = -con; }
3556 if (con < 0) {
3557 __ sub(dst_reg, src_reg, -con);
3558 } else {
3559 __ add(dst_reg, src_reg, con);
3560 }
3561 %}
3562
3563 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3564 Register dst_reg = as_Register($dst$$reg);
3565 Register src1_reg = as_Register($src1$$reg);
3566 Register src2_reg = as_Register($src2$$reg);
3567 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3568 %}
3569
3570 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3571 Register dst_reg = as_Register($dst$$reg);
3572 Register src1_reg = as_Register($src1$$reg);
3573 Register src2_reg = as_Register($src2$$reg);
3574 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3575 %}
3576
3577 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3578 Register dst_reg = as_Register($dst$$reg);
3579 Register src1_reg = as_Register($src1$$reg);
3580 Register src2_reg = as_Register($src2$$reg);
3581 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3582 %}
3583
3584 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3585 Register dst_reg = as_Register($dst$$reg);
3586 Register src1_reg = as_Register($src1$$reg);
3587 Register src2_reg = as_Register($src2$$reg);
3588 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3589 %}
3590
3591 // compare instruction encodings
3592
3593 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3594 Register reg1 = as_Register($src1$$reg);
3595 Register reg2 = as_Register($src2$$reg);
3596 __ cmpw(reg1, reg2);
3597 %}
3598
3599 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3600 Register reg = as_Register($src1$$reg);
3601 int32_t val = $src2$$constant;
3602 if (val >= 0) {
3603 __ subsw(zr, reg, val);
3604 } else {
3605 __ addsw(zr, reg, -val);
3606 }
3607 %}
3608
3609 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3610 Register reg1 = as_Register($src1$$reg);
3611 uint32_t val = (uint32_t)$src2$$constant;
3612 __ movw(rscratch1, val);
3613 __ cmpw(reg1, rscratch1);
3614 %}
3615
3616 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3617 Register reg1 = as_Register($src1$$reg);
3618 Register reg2 = as_Register($src2$$reg);
3619 __ cmp(reg1, reg2);
3620 %}
3621
3622 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3623 Register reg = as_Register($src1$$reg);
3624 int64_t val = $src2$$constant;
3625 if (val >= 0) {
3626 __ subs(zr, reg, val);
3627 } else if (val != -val) {
3628 __ adds(zr, reg, -val);
3629 } else {
3630 // aargh, Long.MIN_VALUE is a special case
3631 __ orr(rscratch1, zr, (uint64_t)val);
3632 __ subs(zr, reg, rscratch1);
3633 }
3634 %}
3635
3636 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3637 Register reg1 = as_Register($src1$$reg);
3638 uint64_t val = (uint64_t)$src2$$constant;
3639 __ mov(rscratch1, val);
3640 __ cmp(reg1, rscratch1);
3641 %}
3642
3643 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3644 Register reg1 = as_Register($src1$$reg);
3645 Register reg2 = as_Register($src2$$reg);
3646 __ cmp(reg1, reg2);
3647 %}
3648
3649 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3650 Register reg1 = as_Register($src1$$reg);
3651 Register reg2 = as_Register($src2$$reg);
3652 __ cmpw(reg1, reg2);
3653 %}
3654
3655 enc_class aarch64_enc_testp(iRegP src) %{
3656 Register reg = as_Register($src$$reg);
3657 __ cmp(reg, zr);
3658 %}
3659
3660 enc_class aarch64_enc_testn(iRegN src) %{
3661 Register reg = as_Register($src$$reg);
3662 __ cmpw(reg, zr);
3663 %}
3664
3665 enc_class aarch64_enc_b(label lbl) %{
3666 Label *L = $lbl$$label;
3667 __ b(*L);
3668 %}
3669
3670 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3671 Label *L = $lbl$$label;
3672 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3673 %}
3674
3675 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3676 Label *L = $lbl$$label;
3677 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3678 %}
3679
3680 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3681 %{
3682 Register sub_reg = as_Register($sub$$reg);
3683 Register super_reg = as_Register($super$$reg);
3684 Register temp_reg = as_Register($temp$$reg);
3685 Register result_reg = as_Register($result$$reg);
3686
3687 Label miss;
3688 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3689 nullptr, &miss,
3690 /*set_cond_codes:*/ true);
3691 if ($primary) {
3692 __ mov(result_reg, zr);
3693 }
3694 __ bind(miss);
3695 %}
3696
3697 enc_class aarch64_enc_java_static_call(method meth) %{
3698 address addr = (address)$meth$$method;
3699 address call;
3700 if (!_method) {
3701 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3702 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3703 if (call == nullptr) {
3704 ciEnv::current()->record_failure("CodeCache is full");
3705 return;
3706 }
3707 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3708 // The NOP here is purely to ensure that eliding a call to
3709 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3710 __ nop();
3711 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3712 } else {
3713 int method_index = resolved_method_index(masm);
3714 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3715 : static_call_Relocation::spec(method_index);
3716 call = __ trampoline_call(Address(addr, rspec));
3717 if (call == nullptr) {
3718 ciEnv::current()->record_failure("CodeCache is full");
3719 return;
3720 }
3721 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3722 // Calls of the same statically bound method can share
3723 // a stub to the interpreter.
3724 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3725 } else {
3726 // Emit stub for static call
3727 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3728 if (stub == nullptr) {
3729 ciEnv::current()->record_failure("CodeCache is full");
3730 return;
3731 }
3732 }
3733 }
3734
3735 __ post_call_nop();
3736
3737 // Only non uncommon_trap calls need to reinitialize ptrue.
3738 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3739 __ reinitialize_ptrue();
3740 }
3741 %}
3742
3743 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3744 int method_index = resolved_method_index(masm);
3745 address call = __ ic_call((address)$meth$$method, method_index);
3746 if (call == nullptr) {
3747 ciEnv::current()->record_failure("CodeCache is full");
3748 return;
3749 }
3750 __ post_call_nop();
3751 if (Compile::current()->max_vector_size() > 0) {
3752 __ reinitialize_ptrue();
3753 }
3754 %}
3755
3756 enc_class aarch64_enc_call_epilog() %{
3757 if (VerifyStackAtCalls) {
3758 // Check that stack depth is unchanged: find majik cookie on stack
3759 __ call_Unimplemented();
3760 }
3761 %}
3762
3763 enc_class aarch64_enc_java_to_runtime(method meth) %{
3764 // some calls to generated routines (arraycopy code) are scheduled
3765 // by C2 as runtime calls. if so we can call them using a br (they
3766 // will be in a reachable segment) otherwise we have to use a blr
3767 // which loads the absolute address into a register.
3768 address entry = (address)$meth$$method;
3769 CodeBlob *cb = CodeCache::find_blob(entry);
3770 if (cb) {
3771 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3772 if (call == nullptr) {
3773 ciEnv::current()->record_failure("CodeCache is full");
3774 return;
3775 }
3776 __ post_call_nop();
3777 } else {
3778 Label retaddr;
3779 // Make the anchor frame walkable
3780 __ adr(rscratch2, retaddr);
3781 __ str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
3782 __ lea(rscratch1, RuntimeAddress(entry));
3783 __ blr(rscratch1);
3784 __ bind(retaddr);
3785 __ post_call_nop();
3786 }
3787 if (Compile::current()->max_vector_size() > 0) {
3788 __ reinitialize_ptrue();
3789 }
3790 %}
3791
3792 enc_class aarch64_enc_rethrow() %{
3793 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3794 %}
3795
3796 enc_class aarch64_enc_ret() %{
3797 #ifdef ASSERT
3798 if (Compile::current()->max_vector_size() > 0) {
3799 __ verify_ptrue();
3800 }
3801 #endif
3802 __ ret(lr);
3803 %}
3804
3805 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3806 Register target_reg = as_Register($jump_target$$reg);
3807 __ br(target_reg);
3808 %}
3809
3810 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3811 Register target_reg = as_Register($jump_target$$reg);
3812 // exception oop should be in r0
3813 // ret addr has been popped into lr
3814 // callee expects it in r3
3815 __ mov(r3, lr);
3816 __ br(target_reg);
3817 %}
3818
3819 %}
3820
3821 //----------FRAME--------------------------------------------------------------
3822 // Definition of frame structure and management information.
3823 //
3824 // S T A C K L A Y O U T Allocators stack-slot number
3825 // | (to get allocators register number
3826 // G Owned by | | v add OptoReg::stack0())
3827 // r CALLER | |
3828 // o | +--------+ pad to even-align allocators stack-slot
3829 // w V | pad0 | numbers; owned by CALLER
3830 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3831 // h ^ | in | 5
3832 // | | args | 4 Holes in incoming args owned by SELF
3833 // | | | | 3
3834 // | | +--------+
3835 // V | | old out| Empty on Intel, window on Sparc
3836 // | old |preserve| Must be even aligned.
3837 // | SP-+--------+----> Matcher::_old_SP, even aligned
3838 // | | in | 3 area for Intel ret address
3839 // Owned by |preserve| Empty on Sparc.
3840 // SELF +--------+
3841 // | | pad2 | 2 pad to align old SP
3842 // | +--------+ 1
3843 // | | locks | 0
3844 // | +--------+----> OptoReg::stack0(), even aligned
3845 // | | pad1 | 11 pad to align new SP
3846 // | +--------+
3847 // | | | 10
3848 // | | spills | 9 spills
3849 // V | | 8 (pad0 slot for callee)
3850 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3851 // ^ | out | 7
3852 // | | args | 6 Holes in outgoing args owned by CALLEE
3853 // Owned by +--------+
3854 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3855 // | new |preserve| Must be even-aligned.
3856 // | SP-+--------+----> Matcher::_new_SP, even aligned
3857 // | | |
3858 //
3859 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3860 // known from SELF's arguments and the Java calling convention.
3861 // Region 6-7 is determined per call site.
3862 // Note 2: If the calling convention leaves holes in the incoming argument
3863 // area, those holes are owned by SELF. Holes in the outgoing area
3864 // are owned by the CALLEE. Holes should not be necessary in the
3865 // incoming area, as the Java calling convention is completely under
3866 // the control of the AD file. Doubles can be sorted and packed to
3867 // avoid holes. Holes in the outgoing arguments may be necessary for
3868 // varargs C calling conventions.
3869 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3870 // even aligned with pad0 as needed.
3871 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3872 // (the latter is true on Intel but is it false on AArch64?)
3873 // region 6-11 is even aligned; it may be padded out more so that
3874 // the region from SP to FP meets the minimum stack alignment.
3875 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3876 // alignment. Region 11, pad1, may be dynamically extended so that
3877 // SP meets the minimum alignment.
3878
3879 frame %{
3880 // These three registers define part of the calling convention
3881 // between compiled code and the interpreter.
3882
3883 // Inline Cache Register or Method for I2C.
3884 inline_cache_reg(R12);
3885
3886 // Number of stack slots consumed by locking an object
3887 sync_stack_slots(2);
3888
3889 // Compiled code's Frame Pointer
3890 frame_pointer(R31);
3891
3892 // Interpreter stores its frame pointer in a register which is
3893 // stored to the stack by I2CAdaptors.
3894 // I2CAdaptors convert from interpreted java to compiled java.
3895 interpreter_frame_pointer(R29);
3896
3897 // Stack alignment requirement
3898 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3899
3900 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3901 // for calls to C. Supports the var-args backing area for register parms.
3902 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3903
3904 // The after-PROLOG location of the return address. Location of
3905 // return address specifies a type (REG or STACK) and a number
3906 // representing the register number (i.e. - use a register name) or
3907 // stack slot.
3908 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3909 // Otherwise, it is above the locks and verification slot and alignment word
3910 // TODO this may well be correct but need to check why that - 2 is there
3911 // ppc port uses 0 but we definitely need to allow for fixed_slots
3912 // which folds in the space used for monitors
3913 return_addr(STACK - 2 +
3914 align_up((Compile::current()->in_preserve_stack_slots() +
3915 Compile::current()->fixed_slots()),
3916 stack_alignment_in_slots()));
3917
3918 // Location of compiled Java return values. Same as C for now.
3919 return_value
3920 %{
3921 // TODO do we allow ideal_reg == Op_RegN???
3922 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3923 "only return normal values");
3924
3925 static const int lo[Op_RegL + 1] = { // enum name
3926 0, // Op_Node
3927 0, // Op_Set
3928 R0_num, // Op_RegN
3929 R0_num, // Op_RegI
3930 R0_num, // Op_RegP
3931 V0_num, // Op_RegF
3932 V0_num, // Op_RegD
3933 R0_num // Op_RegL
3934 };
3935
3936 static const int hi[Op_RegL + 1] = { // enum name
3937 0, // Op_Node
3938 0, // Op_Set
3939 OptoReg::Bad, // Op_RegN
3940 OptoReg::Bad, // Op_RegI
3941 R0_H_num, // Op_RegP
3942 OptoReg::Bad, // Op_RegF
3943 V0_H_num, // Op_RegD
3944 R0_H_num // Op_RegL
3945 };
3946
3947 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3948 %}
3949 %}
3950
3951 //----------ATTRIBUTES---------------------------------------------------------
3952 //----------Operand Attributes-------------------------------------------------
3953 op_attrib op_cost(1); // Required cost attribute
3954
3955 //----------Instruction Attributes---------------------------------------------
3956 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3957 ins_attrib ins_size(32); // Required size attribute (in bits)
3958 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3959 // a non-matching short branch variant
3960 // of some long branch?
3961 ins_attrib ins_alignment(4); // Required alignment attribute (must
3962 // be a power of 2) specifies the
3963 // alignment that some part of the
3964 // instruction (not necessarily the
3965 // start) requires. If > 1, a
3966 // compute_padding() function must be
3967 // provided for the instruction
3968
3969 // Whether this node is expanded during code emission into a sequence of
3970 // instructions and the first instruction can perform an implicit null check.
3971 ins_attrib ins_is_late_expanded_null_check_candidate(false);
3972
3973 //----------OPERANDS-----------------------------------------------------------
3974 // Operand definitions must precede instruction definitions for correct parsing
3975 // in the ADLC because operands constitute user defined types which are used in
3976 // instruction definitions.
3977
3978 //----------Simple Operands----------------------------------------------------
3979
3980 // Integer operands 32 bit
3981 // 32 bit immediate
3982 operand immI()
3983 %{
3984 match(ConI);
3985
3986 op_cost(0);
3987 format %{ %}
3988 interface(CONST_INTER);
3989 %}
3990
3991 // 32 bit zero
3992 operand immI0()
3993 %{
3994 predicate(n->get_int() == 0);
3995 match(ConI);
3996
3997 op_cost(0);
3998 format %{ %}
3999 interface(CONST_INTER);
4000 %}
4001
4002 // 32 bit unit increment
4003 operand immI_1()
4004 %{
4005 predicate(n->get_int() == 1);
4006 match(ConI);
4007
4008 op_cost(0);
4009 format %{ %}
4010 interface(CONST_INTER);
4011 %}
4012
4013 // 32 bit unit decrement
4014 operand immI_M1()
4015 %{
4016 predicate(n->get_int() == -1);
4017 match(ConI);
4018
4019 op_cost(0);
4020 format %{ %}
4021 interface(CONST_INTER);
4022 %}
4023
4024 // Shift values for add/sub extension shift
4025 operand immIExt()
4026 %{
4027 predicate(0 <= n->get_int() && (n->get_int() <= 4));
4028 match(ConI);
4029
4030 op_cost(0);
4031 format %{ %}
4032 interface(CONST_INTER);
4033 %}
4034
4035 operand immI_gt_1()
4036 %{
4037 predicate(n->get_int() > 1);
4038 match(ConI);
4039
4040 op_cost(0);
4041 format %{ %}
4042 interface(CONST_INTER);
4043 %}
4044
4045 operand immI_le_4()
4046 %{
4047 predicate(n->get_int() <= 4);
4048 match(ConI);
4049
4050 op_cost(0);
4051 format %{ %}
4052 interface(CONST_INTER);
4053 %}
4054
4055 operand immI_16()
4056 %{
4057 predicate(n->get_int() == 16);
4058 match(ConI);
4059
4060 op_cost(0);
4061 format %{ %}
4062 interface(CONST_INTER);
4063 %}
4064
4065 operand immI_24()
4066 %{
4067 predicate(n->get_int() == 24);
4068 match(ConI);
4069
4070 op_cost(0);
4071 format %{ %}
4072 interface(CONST_INTER);
4073 %}
4074
4075 operand immI_32()
4076 %{
4077 predicate(n->get_int() == 32);
4078 match(ConI);
4079
4080 op_cost(0);
4081 format %{ %}
4082 interface(CONST_INTER);
4083 %}
4084
4085 operand immI_48()
4086 %{
4087 predicate(n->get_int() == 48);
4088 match(ConI);
4089
4090 op_cost(0);
4091 format %{ %}
4092 interface(CONST_INTER);
4093 %}
4094
4095 operand immI_56()
4096 %{
4097 predicate(n->get_int() == 56);
4098 match(ConI);
4099
4100 op_cost(0);
4101 format %{ %}
4102 interface(CONST_INTER);
4103 %}
4104
4105 operand immI_255()
4106 %{
4107 predicate(n->get_int() == 255);
4108 match(ConI);
4109
4110 op_cost(0);
4111 format %{ %}
4112 interface(CONST_INTER);
4113 %}
4114
4115 operand immI_65535()
4116 %{
4117 predicate(n->get_int() == 65535);
4118 match(ConI);
4119
4120 op_cost(0);
4121 format %{ %}
4122 interface(CONST_INTER);
4123 %}
4124
4125 operand immI_positive()
4126 %{
4127 predicate(n->get_int() > 0);
4128 match(ConI);
4129
4130 op_cost(0);
4131 format %{ %}
4132 interface(CONST_INTER);
4133 %}
4134
4135 // BoolTest condition for signed compare
4136 operand immI_cmp_cond()
4137 %{
4138 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4139 match(ConI);
4140
4141 op_cost(0);
4142 format %{ %}
4143 interface(CONST_INTER);
4144 %}
4145
4146 // BoolTest condition for unsigned compare
4147 operand immI_cmpU_cond()
4148 %{
4149 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4150 match(ConI);
4151
4152 op_cost(0);
4153 format %{ %}
4154 interface(CONST_INTER);
4155 %}
4156
4157 operand immL_255()
4158 %{
4159 predicate(n->get_long() == 255L);
4160 match(ConL);
4161
4162 op_cost(0);
4163 format %{ %}
4164 interface(CONST_INTER);
4165 %}
4166
4167 operand immL_65535()
4168 %{
4169 predicate(n->get_long() == 65535L);
4170 match(ConL);
4171
4172 op_cost(0);
4173 format %{ %}
4174 interface(CONST_INTER);
4175 %}
4176
4177 operand immL_4294967295()
4178 %{
4179 predicate(n->get_long() == 4294967295L);
4180 match(ConL);
4181
4182 op_cost(0);
4183 format %{ %}
4184 interface(CONST_INTER);
4185 %}
4186
4187 operand immL_bitmask()
4188 %{
4189 predicate((n->get_long() != 0)
4190 && ((n->get_long() & 0xc000000000000000l) == 0)
4191 && is_power_of_2(n->get_long() + 1));
4192 match(ConL);
4193
4194 op_cost(0);
4195 format %{ %}
4196 interface(CONST_INTER);
4197 %}
4198
4199 operand immI_bitmask()
4200 %{
4201 predicate((n->get_int() != 0)
4202 && ((n->get_int() & 0xc0000000) == 0)
4203 && is_power_of_2(n->get_int() + 1));
4204 match(ConI);
4205
4206 op_cost(0);
4207 format %{ %}
4208 interface(CONST_INTER);
4209 %}
4210
4211 operand immL_positive_bitmaskI()
4212 %{
4213 predicate((n->get_long() != 0)
4214 && ((julong)n->get_long() < 0x80000000ULL)
4215 && is_power_of_2(n->get_long() + 1));
4216 match(ConL);
4217
4218 op_cost(0);
4219 format %{ %}
4220 interface(CONST_INTER);
4221 %}
4222
4223 // Scale values for scaled offset addressing modes (up to long but not quad)
4224 operand immIScale()
4225 %{
4226 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4227 match(ConI);
4228
4229 op_cost(0);
4230 format %{ %}
4231 interface(CONST_INTER);
4232 %}
4233
4234 // 5 bit signed integer
4235 operand immI5()
4236 %{
4237 predicate(Assembler::is_simm(n->get_int(), 5));
4238 match(ConI);
4239
4240 op_cost(0);
4241 format %{ %}
4242 interface(CONST_INTER);
4243 %}
4244
4245 // 7 bit unsigned integer
4246 operand immIU7()
4247 %{
4248 predicate(Assembler::is_uimm(n->get_int(), 7));
4249 match(ConI);
4250
4251 op_cost(0);
4252 format %{ %}
4253 interface(CONST_INTER);
4254 %}
4255
4256 // Offset for scaled or unscaled immediate loads and stores
4257 operand immIOffset()
4258 %{
4259 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4260 match(ConI);
4261
4262 op_cost(0);
4263 format %{ %}
4264 interface(CONST_INTER);
4265 %}
4266
4267 operand immIOffset1()
4268 %{
4269 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4270 match(ConI);
4271
4272 op_cost(0);
4273 format %{ %}
4274 interface(CONST_INTER);
4275 %}
4276
4277 operand immIOffset2()
4278 %{
4279 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4280 match(ConI);
4281
4282 op_cost(0);
4283 format %{ %}
4284 interface(CONST_INTER);
4285 %}
4286
4287 operand immIOffset4()
4288 %{
4289 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4290 match(ConI);
4291
4292 op_cost(0);
4293 format %{ %}
4294 interface(CONST_INTER);
4295 %}
4296
4297 operand immIOffset8()
4298 %{
4299 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4300 match(ConI);
4301
4302 op_cost(0);
4303 format %{ %}
4304 interface(CONST_INTER);
4305 %}
4306
4307 operand immIOffset16()
4308 %{
4309 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4310 match(ConI);
4311
4312 op_cost(0);
4313 format %{ %}
4314 interface(CONST_INTER);
4315 %}
4316
4317 operand immLOffset()
4318 %{
4319 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4320 match(ConL);
4321
4322 op_cost(0);
4323 format %{ %}
4324 interface(CONST_INTER);
4325 %}
4326
4327 operand immLoffset1()
4328 %{
4329 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4330 match(ConL);
4331
4332 op_cost(0);
4333 format %{ %}
4334 interface(CONST_INTER);
4335 %}
4336
4337 operand immLoffset2()
4338 %{
4339 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4340 match(ConL);
4341
4342 op_cost(0);
4343 format %{ %}
4344 interface(CONST_INTER);
4345 %}
4346
4347 operand immLoffset4()
4348 %{
4349 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4350 match(ConL);
4351
4352 op_cost(0);
4353 format %{ %}
4354 interface(CONST_INTER);
4355 %}
4356
4357 operand immLoffset8()
4358 %{
4359 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4360 match(ConL);
4361
4362 op_cost(0);
4363 format %{ %}
4364 interface(CONST_INTER);
4365 %}
4366
4367 operand immLoffset16()
4368 %{
4369 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4370 match(ConL);
4371
4372 op_cost(0);
4373 format %{ %}
4374 interface(CONST_INTER);
4375 %}
4376
4377 // 5 bit signed long integer
4378 operand immL5()
4379 %{
4380 predicate(Assembler::is_simm(n->get_long(), 5));
4381 match(ConL);
4382
4383 op_cost(0);
4384 format %{ %}
4385 interface(CONST_INTER);
4386 %}
4387
4388 // 7 bit unsigned long integer
4389 operand immLU7()
4390 %{
4391 predicate(Assembler::is_uimm(n->get_long(), 7));
4392 match(ConL);
4393
4394 op_cost(0);
4395 format %{ %}
4396 interface(CONST_INTER);
4397 %}
4398
4399 // 8 bit signed value.
4400 operand immI8()
4401 %{
4402 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4403 match(ConI);
4404
4405 op_cost(0);
4406 format %{ %}
4407 interface(CONST_INTER);
4408 %}
4409
4410 // 8 bit signed value (simm8), or #simm8 LSL 8.
4411 operand immIDupV()
4412 %{
4413 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->get_int()));
4414 match(ConI);
4415
4416 op_cost(0);
4417 format %{ %}
4418 interface(CONST_INTER);
4419 %}
4420
4421 // 8 bit signed value (simm8), or #simm8 LSL 8.
4422 operand immLDupV()
4423 %{
4424 predicate(Assembler::operand_valid_for_sve_dup_immediate(n->get_long()));
4425 match(ConL);
4426
4427 op_cost(0);
4428 format %{ %}
4429 interface(CONST_INTER);
4430 %}
4431
4432 // 8 bit signed value (simm8), or #simm8 LSL 8.
4433 operand immHDupV()
4434 %{
4435 predicate(Assembler::operand_valid_for_sve_dup_immediate((int64_t)n->geth()));
4436 match(ConH);
4437
4438 op_cost(0);
4439 format %{ %}
4440 interface(CONST_INTER);
4441 %}
4442
4443 // 8 bit integer valid for vector add sub immediate
4444 operand immBAddSubV()
4445 %{
4446 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4447 match(ConI);
4448
4449 op_cost(0);
4450 format %{ %}
4451 interface(CONST_INTER);
4452 %}
4453
4454 // 32 bit integer valid for add sub immediate
4455 operand immIAddSub()
4456 %{
4457 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4458 match(ConI);
4459 op_cost(0);
4460 format %{ %}
4461 interface(CONST_INTER);
4462 %}
4463
4464 // 32 bit integer valid for vector add sub immediate
4465 operand immIAddSubV()
4466 %{
4467 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4468 match(ConI);
4469
4470 op_cost(0);
4471 format %{ %}
4472 interface(CONST_INTER);
4473 %}
4474
4475 // 32 bit unsigned integer valid for logical immediate
4476
4477 operand immBLog()
4478 %{
4479 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4480 match(ConI);
4481
4482 op_cost(0);
4483 format %{ %}
4484 interface(CONST_INTER);
4485 %}
4486
4487 operand immSLog()
4488 %{
4489 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4490 match(ConI);
4491
4492 op_cost(0);
4493 format %{ %}
4494 interface(CONST_INTER);
4495 %}
4496
4497 operand immILog()
4498 %{
4499 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4500 match(ConI);
4501
4502 op_cost(0);
4503 format %{ %}
4504 interface(CONST_INTER);
4505 %}
4506
4507 // Integer operands 64 bit
4508 // 64 bit immediate
4509 operand immL()
4510 %{
4511 match(ConL);
4512
4513 op_cost(0);
4514 format %{ %}
4515 interface(CONST_INTER);
4516 %}
4517
4518 // 64 bit zero
4519 operand immL0()
4520 %{
4521 predicate(n->get_long() == 0);
4522 match(ConL);
4523
4524 op_cost(0);
4525 format %{ %}
4526 interface(CONST_INTER);
4527 %}
4528
4529 // 64 bit unit decrement
4530 operand immL_M1()
4531 %{
4532 predicate(n->get_long() == -1);
4533 match(ConL);
4534
4535 op_cost(0);
4536 format %{ %}
4537 interface(CONST_INTER);
4538 %}
4539
4540 // 64 bit integer valid for add sub immediate
4541 operand immLAddSub()
4542 %{
4543 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4544 match(ConL);
4545 op_cost(0);
4546 format %{ %}
4547 interface(CONST_INTER);
4548 %}
4549
4550 // 64 bit integer valid for addv subv immediate
4551 operand immLAddSubV()
4552 %{
4553 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4554 match(ConL);
4555
4556 op_cost(0);
4557 format %{ %}
4558 interface(CONST_INTER);
4559 %}
4560
4561 // 64 bit integer valid for logical immediate
4562 operand immLLog()
4563 %{
4564 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4565 match(ConL);
4566 op_cost(0);
4567 format %{ %}
4568 interface(CONST_INTER);
4569 %}
4570
4571 // Long Immediate: low 32-bit mask
4572 operand immL_32bits()
4573 %{
4574 predicate(n->get_long() == 0xFFFFFFFFL);
4575 match(ConL);
4576 op_cost(0);
4577 format %{ %}
4578 interface(CONST_INTER);
4579 %}
4580
4581 // Pointer operands
4582 // Pointer Immediate
4583 operand immP()
4584 %{
4585 match(ConP);
4586
4587 op_cost(0);
4588 format %{ %}
4589 interface(CONST_INTER);
4590 %}
4591
4592 // nullptr Pointer Immediate
4593 operand immP0()
4594 %{
4595 predicate(n->get_ptr() == 0);
4596 match(ConP);
4597
4598 op_cost(0);
4599 format %{ %}
4600 interface(CONST_INTER);
4601 %}
4602
4603 // Pointer Immediate One
4604 // this is used in object initialization (initial object header)
4605 operand immP_1()
4606 %{
4607 predicate(n->get_ptr() == 1);
4608 match(ConP);
4609
4610 op_cost(0);
4611 format %{ %}
4612 interface(CONST_INTER);
4613 %}
4614
4615 // AOT Runtime Constants Address
4616 operand immAOTRuntimeConstantsAddress()
4617 %{
4618 // Check if the address is in the range of AOT Runtime Constants
4619 predicate(AOTRuntimeConstants::contains((address)(n->get_ptr())));
4620 match(ConP);
4621
4622 op_cost(0);
4623 format %{ %}
4624 interface(CONST_INTER);
4625 %}
4626
4627 // Float and Double operands
4628 // Double Immediate
4629 operand immD()
4630 %{
4631 match(ConD);
4632 op_cost(0);
4633 format %{ %}
4634 interface(CONST_INTER);
4635 %}
4636
4637 // Double Immediate: +0.0d
4638 operand immD0()
4639 %{
4640 predicate(jlong_cast(n->getd()) == 0);
4641 match(ConD);
4642
4643 op_cost(0);
4644 format %{ %}
4645 interface(CONST_INTER);
4646 %}
4647
4648 // constant 'double +0.0'.
4649 operand immDPacked()
4650 %{
4651 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4652 match(ConD);
4653 op_cost(0);
4654 format %{ %}
4655 interface(CONST_INTER);
4656 %}
4657
4658 // Float Immediate
4659 operand immF()
4660 %{
4661 match(ConF);
4662 op_cost(0);
4663 format %{ %}
4664 interface(CONST_INTER);
4665 %}
4666
4667 // Float Immediate: +0.0f.
4668 operand immF0()
4669 %{
4670 predicate(jint_cast(n->getf()) == 0);
4671 match(ConF);
4672
4673 op_cost(0);
4674 format %{ %}
4675 interface(CONST_INTER);
4676 %}
4677
4678 // Half Float (FP16) Immediate
4679 operand immH()
4680 %{
4681 match(ConH);
4682 op_cost(0);
4683 format %{ %}
4684 interface(CONST_INTER);
4685 %}
4686
4687 //
4688 operand immFPacked()
4689 %{
4690 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4691 match(ConF);
4692 op_cost(0);
4693 format %{ %}
4694 interface(CONST_INTER);
4695 %}
4696
4697 // Narrow pointer operands
4698 // Narrow Pointer Immediate
4699 operand immN()
4700 %{
4701 match(ConN);
4702
4703 op_cost(0);
4704 format %{ %}
4705 interface(CONST_INTER);
4706 %}
4707
4708 // Narrow nullptr Pointer Immediate
4709 operand immN0()
4710 %{
4711 predicate(n->get_narrowcon() == 0);
4712 match(ConN);
4713
4714 op_cost(0);
4715 format %{ %}
4716 interface(CONST_INTER);
4717 %}
4718
4719 operand immNKlass()
4720 %{
4721 match(ConNKlass);
4722
4723 op_cost(0);
4724 format %{ %}
4725 interface(CONST_INTER);
4726 %}
4727
4728 // Integer 32 bit Register Operands
4729 // Integer 32 bitRegister (excludes SP)
4730 operand iRegI()
4731 %{
4732 constraint(ALLOC_IN_RC(any_reg32));
4733 match(RegI);
4734 match(iRegINoSp);
4735 op_cost(0);
4736 format %{ %}
4737 interface(REG_INTER);
4738 %}
4739
4740 // Integer 32 bit Register not Special
4741 operand iRegINoSp()
4742 %{
4743 constraint(ALLOC_IN_RC(no_special_reg32));
4744 match(RegI);
4745 op_cost(0);
4746 format %{ %}
4747 interface(REG_INTER);
4748 %}
4749
4750 // Integer 64 bit Register Operands
4751 // Integer 64 bit Register (includes SP)
4752 operand iRegL()
4753 %{
4754 constraint(ALLOC_IN_RC(any_reg));
4755 match(RegL);
4756 match(iRegLNoSp);
4757 op_cost(0);
4758 format %{ %}
4759 interface(REG_INTER);
4760 %}
4761
4762 // Integer 64 bit Register not Special
4763 operand iRegLNoSp()
4764 %{
4765 constraint(ALLOC_IN_RC(no_special_reg));
4766 match(RegL);
4767 match(iRegL_R0);
4768 format %{ %}
4769 interface(REG_INTER);
4770 %}
4771
4772 // Pointer Register Operands
4773 // Pointer Register
4774 operand iRegP()
4775 %{
4776 constraint(ALLOC_IN_RC(ptr_reg));
4777 match(RegP);
4778 match(iRegPNoSp);
4779 match(iRegP_R0);
4780 //match(iRegP_R2);
4781 //match(iRegP_R4);
4782 match(iRegP_R5);
4783 match(thread_RegP);
4784 op_cost(0);
4785 format %{ %}
4786 interface(REG_INTER);
4787 %}
4788
4789 // Pointer 64 bit Register not Special
4790 operand iRegPNoSp()
4791 %{
4792 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4793 match(RegP);
4794 // match(iRegP);
4795 // match(iRegP_R0);
4796 // match(iRegP_R2);
4797 // match(iRegP_R4);
4798 // match(iRegP_R5);
4799 // match(thread_RegP);
4800 op_cost(0);
4801 format %{ %}
4802 interface(REG_INTER);
4803 %}
4804
4805 // This operand is not allowed to use rfp even if
4806 // rfp is not used to hold the frame pointer.
4807 operand iRegPNoSpNoRfp()
4808 %{
4809 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4810 match(RegP);
4811 match(iRegPNoSp);
4812 op_cost(0);
4813 format %{ %}
4814 interface(REG_INTER);
4815 %}
4816
4817 // Pointer 64 bit Register R0 only
4818 operand iRegP_R0()
4819 %{
4820 constraint(ALLOC_IN_RC(r0_reg));
4821 match(RegP);
4822 // match(iRegP);
4823 match(iRegPNoSp);
4824 op_cost(0);
4825 format %{ %}
4826 interface(REG_INTER);
4827 %}
4828
4829 // Pointer 64 bit Register R1 only
4830 operand iRegP_R1()
4831 %{
4832 constraint(ALLOC_IN_RC(r1_reg));
4833 match(RegP);
4834 // match(iRegP);
4835 match(iRegPNoSp);
4836 op_cost(0);
4837 format %{ %}
4838 interface(REG_INTER);
4839 %}
4840
4841 // Pointer 64 bit Register R2 only
4842 operand iRegP_R2()
4843 %{
4844 constraint(ALLOC_IN_RC(r2_reg));
4845 match(RegP);
4846 // match(iRegP);
4847 match(iRegPNoSp);
4848 op_cost(0);
4849 format %{ %}
4850 interface(REG_INTER);
4851 %}
4852
4853 // Pointer 64 bit Register R3 only
4854 operand iRegP_R3()
4855 %{
4856 constraint(ALLOC_IN_RC(r3_reg));
4857 match(RegP);
4858 // match(iRegP);
4859 match(iRegPNoSp);
4860 op_cost(0);
4861 format %{ %}
4862 interface(REG_INTER);
4863 %}
4864
4865 // Pointer 64 bit Register R4 only
4866 operand iRegP_R4()
4867 %{
4868 constraint(ALLOC_IN_RC(r4_reg));
4869 match(RegP);
4870 // match(iRegP);
4871 match(iRegPNoSp);
4872 op_cost(0);
4873 format %{ %}
4874 interface(REG_INTER);
4875 %}
4876
4877 // Pointer 64 bit Register R5 only
4878 operand iRegP_R5()
4879 %{
4880 constraint(ALLOC_IN_RC(r5_reg));
4881 match(RegP);
4882 // match(iRegP);
4883 match(iRegPNoSp);
4884 op_cost(0);
4885 format %{ %}
4886 interface(REG_INTER);
4887 %}
4888
4889 // Pointer 64 bit Register R10 only
4890 operand iRegP_R10()
4891 %{
4892 constraint(ALLOC_IN_RC(r10_reg));
4893 match(RegP);
4894 // match(iRegP);
4895 match(iRegPNoSp);
4896 op_cost(0);
4897 format %{ %}
4898 interface(REG_INTER);
4899 %}
4900
4901 // Long 64 bit Register R0 only
4902 operand iRegL_R0()
4903 %{
4904 constraint(ALLOC_IN_RC(r0_reg));
4905 match(RegL);
4906 match(iRegLNoSp);
4907 op_cost(0);
4908 format %{ %}
4909 interface(REG_INTER);
4910 %}
4911
4912 // Long 64 bit Register R11 only
4913 operand iRegL_R11()
4914 %{
4915 constraint(ALLOC_IN_RC(r11_reg));
4916 match(RegL);
4917 match(iRegLNoSp);
4918 op_cost(0);
4919 format %{ %}
4920 interface(REG_INTER);
4921 %}
4922
4923 // Register R0 only
4924 operand iRegI_R0()
4925 %{
4926 constraint(ALLOC_IN_RC(int_r0_reg));
4927 match(RegI);
4928 match(iRegINoSp);
4929 op_cost(0);
4930 format %{ %}
4931 interface(REG_INTER);
4932 %}
4933
4934 // Register R2 only
4935 operand iRegI_R2()
4936 %{
4937 constraint(ALLOC_IN_RC(int_r2_reg));
4938 match(RegI);
4939 match(iRegINoSp);
4940 op_cost(0);
4941 format %{ %}
4942 interface(REG_INTER);
4943 %}
4944
4945 // Register R3 only
4946 operand iRegI_R3()
4947 %{
4948 constraint(ALLOC_IN_RC(int_r3_reg));
4949 match(RegI);
4950 match(iRegINoSp);
4951 op_cost(0);
4952 format %{ %}
4953 interface(REG_INTER);
4954 %}
4955
4956
4957 // Register R4 only
4958 operand iRegI_R4()
4959 %{
4960 constraint(ALLOC_IN_RC(int_r4_reg));
4961 match(RegI);
4962 match(iRegINoSp);
4963 op_cost(0);
4964 format %{ %}
4965 interface(REG_INTER);
4966 %}
4967
4968
4969 // Pointer Register Operands
4970 // Narrow Pointer Register
4971 operand iRegN()
4972 %{
4973 constraint(ALLOC_IN_RC(any_reg32));
4974 match(RegN);
4975 match(iRegNNoSp);
4976 op_cost(0);
4977 format %{ %}
4978 interface(REG_INTER);
4979 %}
4980
4981 // Integer 64 bit Register not Special
4982 operand iRegNNoSp()
4983 %{
4984 constraint(ALLOC_IN_RC(no_special_reg32));
4985 match(RegN);
4986 op_cost(0);
4987 format %{ %}
4988 interface(REG_INTER);
4989 %}
4990
4991 // Float Register
4992 // Float register operands
4993 operand vRegF()
4994 %{
4995 constraint(ALLOC_IN_RC(float_reg));
4996 match(RegF);
4997
4998 op_cost(0);
4999 format %{ %}
5000 interface(REG_INTER);
5001 %}
5002
5003 // Double Register
5004 // Double register operands
5005 operand vRegD()
5006 %{
5007 constraint(ALLOC_IN_RC(double_reg));
5008 match(RegD);
5009
5010 op_cost(0);
5011 format %{ %}
5012 interface(REG_INTER);
5013 %}
5014
5015 // Generic vector class. This will be used for
5016 // all vector operands, including NEON and SVE.
5017 operand vReg()
5018 %{
5019 constraint(ALLOC_IN_RC(dynamic));
5020 match(VecA);
5021 match(VecD);
5022 match(VecX);
5023
5024 op_cost(0);
5025 format %{ %}
5026 interface(REG_INTER);
5027 %}
5028
5029 operand vReg_V10()
5030 %{
5031 constraint(ALLOC_IN_RC(v10_veca_reg));
5032 match(vReg);
5033
5034 op_cost(0);
5035 format %{ %}
5036 interface(REG_INTER);
5037 %}
5038
5039 operand vReg_V11()
5040 %{
5041 constraint(ALLOC_IN_RC(v11_veca_reg));
5042 match(vReg);
5043
5044 op_cost(0);
5045 format %{ %}
5046 interface(REG_INTER);
5047 %}
5048
5049 operand vReg_V12()
5050 %{
5051 constraint(ALLOC_IN_RC(v12_veca_reg));
5052 match(vReg);
5053
5054 op_cost(0);
5055 format %{ %}
5056 interface(REG_INTER);
5057 %}
5058
5059 operand vReg_V13()
5060 %{
5061 constraint(ALLOC_IN_RC(v13_veca_reg));
5062 match(vReg);
5063
5064 op_cost(0);
5065 format %{ %}
5066 interface(REG_INTER);
5067 %}
5068
5069 operand vReg_V17()
5070 %{
5071 constraint(ALLOC_IN_RC(v17_veca_reg));
5072 match(vReg);
5073
5074 op_cost(0);
5075 format %{ %}
5076 interface(REG_INTER);
5077 %}
5078
5079 operand vReg_V18()
5080 %{
5081 constraint(ALLOC_IN_RC(v18_veca_reg));
5082 match(vReg);
5083
5084 op_cost(0);
5085 format %{ %}
5086 interface(REG_INTER);
5087 %}
5088
5089 operand vReg_V23()
5090 %{
5091 constraint(ALLOC_IN_RC(v23_veca_reg));
5092 match(vReg);
5093
5094 op_cost(0);
5095 format %{ %}
5096 interface(REG_INTER);
5097 %}
5098
5099 operand vReg_V24()
5100 %{
5101 constraint(ALLOC_IN_RC(v24_veca_reg));
5102 match(vReg);
5103
5104 op_cost(0);
5105 format %{ %}
5106 interface(REG_INTER);
5107 %}
5108
5109 operand vecA()
5110 %{
5111 constraint(ALLOC_IN_RC(vectora_reg));
5112 match(VecA);
5113
5114 op_cost(0);
5115 format %{ %}
5116 interface(REG_INTER);
5117 %}
5118
5119 operand vecD()
5120 %{
5121 constraint(ALLOC_IN_RC(vectord_reg));
5122 match(VecD);
5123
5124 op_cost(0);
5125 format %{ %}
5126 interface(REG_INTER);
5127 %}
5128
5129 operand vecX()
5130 %{
5131 constraint(ALLOC_IN_RC(vectorx_reg));
5132 match(VecX);
5133
5134 op_cost(0);
5135 format %{ %}
5136 interface(REG_INTER);
5137 %}
5138
5139 operand vRegD_V0()
5140 %{
5141 constraint(ALLOC_IN_RC(v0_reg));
5142 match(RegD);
5143 op_cost(0);
5144 format %{ %}
5145 interface(REG_INTER);
5146 %}
5147
5148 operand vRegD_V1()
5149 %{
5150 constraint(ALLOC_IN_RC(v1_reg));
5151 match(RegD);
5152 op_cost(0);
5153 format %{ %}
5154 interface(REG_INTER);
5155 %}
5156
5157 operand vRegD_V2()
5158 %{
5159 constraint(ALLOC_IN_RC(v2_reg));
5160 match(RegD);
5161 op_cost(0);
5162 format %{ %}
5163 interface(REG_INTER);
5164 %}
5165
5166 operand vRegD_V3()
5167 %{
5168 constraint(ALLOC_IN_RC(v3_reg));
5169 match(RegD);
5170 op_cost(0);
5171 format %{ %}
5172 interface(REG_INTER);
5173 %}
5174
5175 operand vRegD_V4()
5176 %{
5177 constraint(ALLOC_IN_RC(v4_reg));
5178 match(RegD);
5179 op_cost(0);
5180 format %{ %}
5181 interface(REG_INTER);
5182 %}
5183
5184 operand vRegD_V5()
5185 %{
5186 constraint(ALLOC_IN_RC(v5_reg));
5187 match(RegD);
5188 op_cost(0);
5189 format %{ %}
5190 interface(REG_INTER);
5191 %}
5192
5193 operand vRegD_V6()
5194 %{
5195 constraint(ALLOC_IN_RC(v6_reg));
5196 match(RegD);
5197 op_cost(0);
5198 format %{ %}
5199 interface(REG_INTER);
5200 %}
5201
5202 operand vRegD_V7()
5203 %{
5204 constraint(ALLOC_IN_RC(v7_reg));
5205 match(RegD);
5206 op_cost(0);
5207 format %{ %}
5208 interface(REG_INTER);
5209 %}
5210
5211 operand vRegD_V12()
5212 %{
5213 constraint(ALLOC_IN_RC(v12_reg));
5214 match(RegD);
5215 op_cost(0);
5216 format %{ %}
5217 interface(REG_INTER);
5218 %}
5219
5220 operand vRegD_V13()
5221 %{
5222 constraint(ALLOC_IN_RC(v13_reg));
5223 match(RegD);
5224 op_cost(0);
5225 format %{ %}
5226 interface(REG_INTER);
5227 %}
5228
5229 operand pReg()
5230 %{
5231 constraint(ALLOC_IN_RC(pr_reg));
5232 match(RegVectMask);
5233 match(pRegGov);
5234 op_cost(0);
5235 format %{ %}
5236 interface(REG_INTER);
5237 %}
5238
5239 operand pRegGov()
5240 %{
5241 constraint(ALLOC_IN_RC(gov_pr));
5242 match(RegVectMask);
5243 match(pReg);
5244 op_cost(0);
5245 format %{ %}
5246 interface(REG_INTER);
5247 %}
5248
5249 operand pRegGov_P0()
5250 %{
5251 constraint(ALLOC_IN_RC(p0_reg));
5252 match(RegVectMask);
5253 op_cost(0);
5254 format %{ %}
5255 interface(REG_INTER);
5256 %}
5257
5258 operand pRegGov_P1()
5259 %{
5260 constraint(ALLOC_IN_RC(p1_reg));
5261 match(RegVectMask);
5262 op_cost(0);
5263 format %{ %}
5264 interface(REG_INTER);
5265 %}
5266
5267 // Flags register, used as output of signed compare instructions
5268
5269 // note that on AArch64 we also use this register as the output for
5270 // for floating point compare instructions (CmpF CmpD). this ensures
5271 // that ordered inequality tests use GT, GE, LT or LE none of which
5272 // pass through cases where the result is unordered i.e. one or both
5273 // inputs to the compare is a NaN. this means that the ideal code can
5274 // replace e.g. a GT with an LE and not end up capturing the NaN case
5275 // (where the comparison should always fail). EQ and NE tests are
5276 // always generated in ideal code so that unordered folds into the NE
5277 // case, matching the behaviour of AArch64 NE.
5278 //
5279 // This differs from x86 where the outputs of FP compares use a
5280 // special FP flags registers and where compares based on this
5281 // register are distinguished into ordered inequalities (cmpOpUCF) and
5282 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5283 // to explicitly handle the unordered case in branches. x86 also has
5284 // to include extra CMoveX rules to accept a cmpOpUCF input.
5285
5286 operand rFlagsReg()
5287 %{
5288 constraint(ALLOC_IN_RC(int_flags));
5289 match(RegFlags);
5290
5291 op_cost(0);
5292 format %{ "RFLAGS" %}
5293 interface(REG_INTER);
5294 %}
5295
5296 // Flags register, used as output of unsigned compare instructions
5297 operand rFlagsRegU()
5298 %{
5299 constraint(ALLOC_IN_RC(int_flags));
5300 match(RegFlags);
5301
5302 op_cost(0);
5303 format %{ "RFLAGSU" %}
5304 interface(REG_INTER);
5305 %}
5306
5307 // Special Registers
5308
5309 // Method Register
5310 operand inline_cache_RegP(iRegP reg)
5311 %{
5312 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5313 match(reg);
5314 match(iRegPNoSp);
5315 op_cost(0);
5316 format %{ %}
5317 interface(REG_INTER);
5318 %}
5319
5320 // Thread Register
5321 operand thread_RegP(iRegP reg)
5322 %{
5323 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5324 match(reg);
5325 op_cost(0);
5326 format %{ %}
5327 interface(REG_INTER);
5328 %}
5329
5330 //----------Memory Operands----------------------------------------------------
5331
5332 operand indirect(iRegP reg)
5333 %{
5334 constraint(ALLOC_IN_RC(ptr_reg));
5335 match(reg);
5336 op_cost(0);
5337 format %{ "[$reg]" %}
5338 interface(MEMORY_INTER) %{
5339 base($reg);
5340 index(0xffffffff);
5341 scale(0x0);
5342 disp(0x0);
5343 %}
5344 %}
5345
5346 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5347 %{
5348 constraint(ALLOC_IN_RC(ptr_reg));
5349 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5350 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5351 op_cost(0);
5352 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5353 interface(MEMORY_INTER) %{
5354 base($reg);
5355 index($ireg);
5356 scale($scale);
5357 disp(0x0);
5358 %}
5359 %}
5360
5361 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5362 %{
5363 constraint(ALLOC_IN_RC(ptr_reg));
5364 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5365 match(AddP reg (LShiftL lreg scale));
5366 op_cost(0);
5367 format %{ "$reg, $lreg lsl($scale)" %}
5368 interface(MEMORY_INTER) %{
5369 base($reg);
5370 index($lreg);
5371 scale($scale);
5372 disp(0x0);
5373 %}
5374 %}
5375
5376 operand indIndexI2L(iRegP reg, iRegI ireg)
5377 %{
5378 constraint(ALLOC_IN_RC(ptr_reg));
5379 match(AddP reg (ConvI2L ireg));
5380 op_cost(0);
5381 format %{ "$reg, $ireg, 0, I2L" %}
5382 interface(MEMORY_INTER) %{
5383 base($reg);
5384 index($ireg);
5385 scale(0x0);
5386 disp(0x0);
5387 %}
5388 %}
5389
5390 operand indIndex(iRegP reg, iRegL lreg)
5391 %{
5392 constraint(ALLOC_IN_RC(ptr_reg));
5393 match(AddP reg lreg);
5394 op_cost(0);
5395 format %{ "$reg, $lreg" %}
5396 interface(MEMORY_INTER) %{
5397 base($reg);
5398 index($lreg);
5399 scale(0x0);
5400 disp(0x0);
5401 %}
5402 %}
5403
5404 operand indOffI1(iRegP reg, immIOffset1 off)
5405 %{
5406 constraint(ALLOC_IN_RC(ptr_reg));
5407 match(AddP reg off);
5408 op_cost(0);
5409 format %{ "[$reg, $off]" %}
5410 interface(MEMORY_INTER) %{
5411 base($reg);
5412 index(0xffffffff);
5413 scale(0x0);
5414 disp($off);
5415 %}
5416 %}
5417
5418 operand indOffI2(iRegP reg, immIOffset2 off)
5419 %{
5420 constraint(ALLOC_IN_RC(ptr_reg));
5421 match(AddP reg off);
5422 op_cost(0);
5423 format %{ "[$reg, $off]" %}
5424 interface(MEMORY_INTER) %{
5425 base($reg);
5426 index(0xffffffff);
5427 scale(0x0);
5428 disp($off);
5429 %}
5430 %}
5431
5432 operand indOffI4(iRegP reg, immIOffset4 off)
5433 %{
5434 constraint(ALLOC_IN_RC(ptr_reg));
5435 match(AddP reg off);
5436 op_cost(0);
5437 format %{ "[$reg, $off]" %}
5438 interface(MEMORY_INTER) %{
5439 base($reg);
5440 index(0xffffffff);
5441 scale(0x0);
5442 disp($off);
5443 %}
5444 %}
5445
5446 operand indOffI8(iRegP reg, immIOffset8 off)
5447 %{
5448 constraint(ALLOC_IN_RC(ptr_reg));
5449 match(AddP reg off);
5450 op_cost(0);
5451 format %{ "[$reg, $off]" %}
5452 interface(MEMORY_INTER) %{
5453 base($reg);
5454 index(0xffffffff);
5455 scale(0x0);
5456 disp($off);
5457 %}
5458 %}
5459
5460 operand indOffI16(iRegP reg, immIOffset16 off)
5461 %{
5462 constraint(ALLOC_IN_RC(ptr_reg));
5463 match(AddP reg off);
5464 op_cost(0);
5465 format %{ "[$reg, $off]" %}
5466 interface(MEMORY_INTER) %{
5467 base($reg);
5468 index(0xffffffff);
5469 scale(0x0);
5470 disp($off);
5471 %}
5472 %}
5473
5474 operand indOffL1(iRegP reg, immLoffset1 off)
5475 %{
5476 constraint(ALLOC_IN_RC(ptr_reg));
5477 match(AddP reg off);
5478 op_cost(0);
5479 format %{ "[$reg, $off]" %}
5480 interface(MEMORY_INTER) %{
5481 base($reg);
5482 index(0xffffffff);
5483 scale(0x0);
5484 disp($off);
5485 %}
5486 %}
5487
5488 operand indOffL2(iRegP reg, immLoffset2 off)
5489 %{
5490 constraint(ALLOC_IN_RC(ptr_reg));
5491 match(AddP reg off);
5492 op_cost(0);
5493 format %{ "[$reg, $off]" %}
5494 interface(MEMORY_INTER) %{
5495 base($reg);
5496 index(0xffffffff);
5497 scale(0x0);
5498 disp($off);
5499 %}
5500 %}
5501
5502 operand indOffL4(iRegP reg, immLoffset4 off)
5503 %{
5504 constraint(ALLOC_IN_RC(ptr_reg));
5505 match(AddP reg off);
5506 op_cost(0);
5507 format %{ "[$reg, $off]" %}
5508 interface(MEMORY_INTER) %{
5509 base($reg);
5510 index(0xffffffff);
5511 scale(0x0);
5512 disp($off);
5513 %}
5514 %}
5515
5516 operand indOffL8(iRegP reg, immLoffset8 off)
5517 %{
5518 constraint(ALLOC_IN_RC(ptr_reg));
5519 match(AddP reg off);
5520 op_cost(0);
5521 format %{ "[$reg, $off]" %}
5522 interface(MEMORY_INTER) %{
5523 base($reg);
5524 index(0xffffffff);
5525 scale(0x0);
5526 disp($off);
5527 %}
5528 %}
5529
5530 operand indOffL16(iRegP reg, immLoffset16 off)
5531 %{
5532 constraint(ALLOC_IN_RC(ptr_reg));
5533 match(AddP reg off);
5534 op_cost(0);
5535 format %{ "[$reg, $off]" %}
5536 interface(MEMORY_INTER) %{
5537 base($reg);
5538 index(0xffffffff);
5539 scale(0x0);
5540 disp($off);
5541 %}
5542 %}
5543
5544 operand indirectX2P(iRegL reg)
5545 %{
5546 constraint(ALLOC_IN_RC(ptr_reg));
5547 match(CastX2P reg);
5548 op_cost(0);
5549 format %{ "[$reg]\t# long -> ptr" %}
5550 interface(MEMORY_INTER) %{
5551 base($reg);
5552 index(0xffffffff);
5553 scale(0x0);
5554 disp(0x0);
5555 %}
5556 %}
5557
5558 operand indOffX2P(iRegL reg, immLOffset off)
5559 %{
5560 constraint(ALLOC_IN_RC(ptr_reg));
5561 match(AddP (CastX2P reg) off);
5562 op_cost(0);
5563 format %{ "[$reg, $off]\t# long -> ptr" %}
5564 interface(MEMORY_INTER) %{
5565 base($reg);
5566 index(0xffffffff);
5567 scale(0x0);
5568 disp($off);
5569 %}
5570 %}
5571
5572 operand indirectN(iRegN reg)
5573 %{
5574 predicate(CompressedOops::shift() == 0);
5575 constraint(ALLOC_IN_RC(ptr_reg));
5576 match(DecodeN reg);
5577 op_cost(0);
5578 format %{ "[$reg]\t# narrow" %}
5579 interface(MEMORY_INTER) %{
5580 base($reg);
5581 index(0xffffffff);
5582 scale(0x0);
5583 disp(0x0);
5584 %}
5585 %}
5586
5587 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5588 %{
5589 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5590 constraint(ALLOC_IN_RC(ptr_reg));
5591 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5592 op_cost(0);
5593 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5594 interface(MEMORY_INTER) %{
5595 base($reg);
5596 index($ireg);
5597 scale($scale);
5598 disp(0x0);
5599 %}
5600 %}
5601
5602 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5603 %{
5604 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5605 constraint(ALLOC_IN_RC(ptr_reg));
5606 match(AddP (DecodeN reg) (LShiftL lreg scale));
5607 op_cost(0);
5608 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5609 interface(MEMORY_INTER) %{
5610 base($reg);
5611 index($lreg);
5612 scale($scale);
5613 disp(0x0);
5614 %}
5615 %}
5616
5617 operand indIndexI2LN(iRegN reg, iRegI ireg)
5618 %{
5619 predicate(CompressedOops::shift() == 0);
5620 constraint(ALLOC_IN_RC(ptr_reg));
5621 match(AddP (DecodeN reg) (ConvI2L ireg));
5622 op_cost(0);
5623 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5624 interface(MEMORY_INTER) %{
5625 base($reg);
5626 index($ireg);
5627 scale(0x0);
5628 disp(0x0);
5629 %}
5630 %}
5631
5632 operand indIndexN(iRegN reg, iRegL lreg)
5633 %{
5634 predicate(CompressedOops::shift() == 0);
5635 constraint(ALLOC_IN_RC(ptr_reg));
5636 match(AddP (DecodeN reg) lreg);
5637 op_cost(0);
5638 format %{ "$reg, $lreg\t# narrow" %}
5639 interface(MEMORY_INTER) %{
5640 base($reg);
5641 index($lreg);
5642 scale(0x0);
5643 disp(0x0);
5644 %}
5645 %}
5646
5647 operand indOffIN(iRegN reg, immIOffset off)
5648 %{
5649 predicate(CompressedOops::shift() == 0);
5650 constraint(ALLOC_IN_RC(ptr_reg));
5651 match(AddP (DecodeN reg) off);
5652 op_cost(0);
5653 format %{ "[$reg, $off]\t# narrow" %}
5654 interface(MEMORY_INTER) %{
5655 base($reg);
5656 index(0xffffffff);
5657 scale(0x0);
5658 disp($off);
5659 %}
5660 %}
5661
5662 operand indOffLN(iRegN reg, immLOffset off)
5663 %{
5664 predicate(CompressedOops::shift() == 0);
5665 constraint(ALLOC_IN_RC(ptr_reg));
5666 match(AddP (DecodeN reg) off);
5667 op_cost(0);
5668 format %{ "[$reg, $off]\t# narrow" %}
5669 interface(MEMORY_INTER) %{
5670 base($reg);
5671 index(0xffffffff);
5672 scale(0x0);
5673 disp($off);
5674 %}
5675 %}
5676
5677
5678 //----------Special Memory Operands--------------------------------------------
5679 // Stack Slot Operand - This operand is used for loading and storing temporary
5680 // values on the stack where a match requires a value to
5681 // flow through memory.
5682 operand stackSlotP(sRegP reg)
5683 %{
5684 constraint(ALLOC_IN_RC(stack_slots));
5685 op_cost(100);
5686 // No match rule because this operand is only generated in matching
5687 // match(RegP);
5688 format %{ "[$reg]" %}
5689 interface(MEMORY_INTER) %{
5690 base(0x1e); // RSP
5691 index(0x0); // No Index
5692 scale(0x0); // No Scale
5693 disp($reg); // Stack Offset
5694 %}
5695 %}
5696
5697 operand stackSlotI(sRegI reg)
5698 %{
5699 constraint(ALLOC_IN_RC(stack_slots));
5700 // No match rule because this operand is only generated in matching
5701 // match(RegI);
5702 format %{ "[$reg]" %}
5703 interface(MEMORY_INTER) %{
5704 base(0x1e); // RSP
5705 index(0x0); // No Index
5706 scale(0x0); // No Scale
5707 disp($reg); // Stack Offset
5708 %}
5709 %}
5710
5711 operand stackSlotF(sRegF reg)
5712 %{
5713 constraint(ALLOC_IN_RC(stack_slots));
5714 // No match rule because this operand is only generated in matching
5715 // match(RegF);
5716 format %{ "[$reg]" %}
5717 interface(MEMORY_INTER) %{
5718 base(0x1e); // RSP
5719 index(0x0); // No Index
5720 scale(0x0); // No Scale
5721 disp($reg); // Stack Offset
5722 %}
5723 %}
5724
5725 operand stackSlotD(sRegD reg)
5726 %{
5727 constraint(ALLOC_IN_RC(stack_slots));
5728 // No match rule because this operand is only generated in matching
5729 // match(RegD);
5730 format %{ "[$reg]" %}
5731 interface(MEMORY_INTER) %{
5732 base(0x1e); // RSP
5733 index(0x0); // No Index
5734 scale(0x0); // No Scale
5735 disp($reg); // Stack Offset
5736 %}
5737 %}
5738
5739 operand stackSlotL(sRegL reg)
5740 %{
5741 constraint(ALLOC_IN_RC(stack_slots));
5742 // No match rule because this operand is only generated in matching
5743 // match(RegL);
5744 format %{ "[$reg]" %}
5745 interface(MEMORY_INTER) %{
5746 base(0x1e); // RSP
5747 index(0x0); // No Index
5748 scale(0x0); // No Scale
5749 disp($reg); // Stack Offset
5750 %}
5751 %}
5752
5753 // Operands for expressing Control Flow
5754 // NOTE: Label is a predefined operand which should not be redefined in
5755 // the AD file. It is generically handled within the ADLC.
5756
5757 //----------Conditional Branch Operands----------------------------------------
5758 // Comparison Op - This is the operation of the comparison, and is limited to
5759 // the following set of codes:
5760 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5761 //
5762 // Other attributes of the comparison, such as unsignedness, are specified
5763 // by the comparison instruction that sets a condition code flags register.
5764 // That result is represented by a flags operand whose subtype is appropriate
5765 // to the unsignedness (etc.) of the comparison.
5766 //
5767 // Later, the instruction which matches both the Comparison Op (a Bool) and
5768 // the flags (produced by the Cmp) specifies the coding of the comparison op
5769 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5770
5771 // used for signed integral comparisons and fp comparisons
5772
5773 operand cmpOp()
5774 %{
5775 match(Bool);
5776
5777 format %{ "" %}
5778 interface(COND_INTER) %{
5779 equal(0x0, "eq");
5780 not_equal(0x1, "ne");
5781 less(0xb, "lt");
5782 greater_equal(0xa, "ge");
5783 less_equal(0xd, "le");
5784 greater(0xc, "gt");
5785 overflow(0x6, "vs");
5786 no_overflow(0x7, "vc");
5787 %}
5788 %}
5789
5790 // used for unsigned integral comparisons
5791
5792 operand cmpOpU()
5793 %{
5794 match(Bool);
5795
5796 format %{ "" %}
5797 interface(COND_INTER) %{
5798 equal(0x0, "eq");
5799 not_equal(0x1, "ne");
5800 less(0x3, "lo");
5801 greater_equal(0x2, "hs");
5802 less_equal(0x9, "ls");
5803 greater(0x8, "hi");
5804 overflow(0x6, "vs");
5805 no_overflow(0x7, "vc");
5806 %}
5807 %}
5808
5809 // used for certain integral comparisons which can be
5810 // converted to cbxx or tbxx instructions
5811
5812 operand cmpOpEqNe()
5813 %{
5814 match(Bool);
5815 op_cost(0);
5816 predicate(n->as_Bool()->_test._test == BoolTest::ne
5817 || n->as_Bool()->_test._test == BoolTest::eq);
5818
5819 format %{ "" %}
5820 interface(COND_INTER) %{
5821 equal(0x0, "eq");
5822 not_equal(0x1, "ne");
5823 less(0xb, "lt");
5824 greater_equal(0xa, "ge");
5825 less_equal(0xd, "le");
5826 greater(0xc, "gt");
5827 overflow(0x6, "vs");
5828 no_overflow(0x7, "vc");
5829 %}
5830 %}
5831
5832 // used for certain integral comparisons which can be
5833 // converted to cbxx or tbxx instructions
5834
5835 operand cmpOpLtGe()
5836 %{
5837 match(Bool);
5838 op_cost(0);
5839
5840 predicate(n->as_Bool()->_test._test == BoolTest::lt
5841 || n->as_Bool()->_test._test == BoolTest::ge);
5842
5843 format %{ "" %}
5844 interface(COND_INTER) %{
5845 equal(0x0, "eq");
5846 not_equal(0x1, "ne");
5847 less(0xb, "lt");
5848 greater_equal(0xa, "ge");
5849 less_equal(0xd, "le");
5850 greater(0xc, "gt");
5851 overflow(0x6, "vs");
5852 no_overflow(0x7, "vc");
5853 %}
5854 %}
5855
5856 // used for certain unsigned integral comparisons which can be
5857 // converted to cbxx or tbxx instructions
5858
5859 operand cmpOpUEqNeLeGt()
5860 %{
5861 match(Bool);
5862 op_cost(0);
5863
5864 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5865 n->as_Bool()->_test._test == BoolTest::ne ||
5866 n->as_Bool()->_test._test == BoolTest::le ||
5867 n->as_Bool()->_test._test == BoolTest::gt);
5868
5869 format %{ "" %}
5870 interface(COND_INTER) %{
5871 equal(0x0, "eq");
5872 not_equal(0x1, "ne");
5873 less(0x3, "lo");
5874 greater_equal(0x2, "hs");
5875 less_equal(0x9, "ls");
5876 greater(0x8, "hi");
5877 overflow(0x6, "vs");
5878 no_overflow(0x7, "vc");
5879 %}
5880 %}
5881
5882 // Special operand allowing long args to int ops to be truncated for free
5883
5884 operand iRegL2I(iRegL reg) %{
5885
5886 op_cost(0);
5887
5888 match(ConvL2I reg);
5889
5890 format %{ "l2i($reg)" %}
5891
5892 interface(REG_INTER)
5893 %}
5894
5895 operand iRegL2P(iRegL reg) %{
5896
5897 op_cost(0);
5898
5899 match(CastX2P reg);
5900
5901 format %{ "l2p($reg)" %}
5902
5903 interface(REG_INTER)
5904 %}
5905
5906 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5907 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5908 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5909 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5910
5911 //----------OPERAND CLASSES----------------------------------------------------
5912 // Operand Classes are groups of operands that are used as to simplify
5913 // instruction definitions by not requiring the AD writer to specify
5914 // separate instructions for every form of operand when the
5915 // instruction accepts multiple operand types with the same basic
5916 // encoding and format. The classic case of this is memory operands.
5917
5918 // memory is used to define read/write location for load/store
5919 // instruction defs. we can turn a memory op into an Address
5920
5921 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5922 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5923
5924 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5925 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5926
5927 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5928 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5929
5930 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5931 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5932
5933 // All of the memory operands. For the pipeline description.
5934 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5935 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5936 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5937
5938
5939 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5940 // operations. it allows the src to be either an iRegI or a (ConvL2I
5941 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5942 // can be elided because the 32-bit instruction will just employ the
5943 // lower 32 bits anyway.
5944 //
5945 // n.b. this does not elide all L2I conversions. if the truncated
5946 // value is consumed by more than one operation then the ConvL2I
5947 // cannot be bundled into the consuming nodes so an l2i gets planted
5948 // (actually a movw $dst $src) and the downstream instructions consume
5949 // the result of the l2i as an iRegI input. That's a shame since the
5950 // movw is actually redundant but its not too costly.
5951
5952 opclass iRegIorL2I(iRegI, iRegL2I);
5953 opclass iRegPorL2P(iRegP, iRegL2P);
5954
5955 //----------PIPELINE-----------------------------------------------------------
5956 // Rules which define the behavior of the target architectures pipeline.
5957
5958 // For specific pipelines, eg A53, define the stages of that pipeline
5959 //pipe_desc(ISS, EX1, EX2, WR);
5960 #define ISS S0
5961 #define EX1 S1
5962 #define EX2 S2
5963 #define WR S3
5964
5965 // Integer ALU reg operation
5966 pipeline %{
5967
5968 attributes %{
5969 // ARM instructions are of fixed length
5970 fixed_size_instructions; // Fixed size instructions TODO does
5971 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5972 // ARM instructions come in 32-bit word units
5973 instruction_unit_size = 4; // An instruction is 4 bytes long
5974 instruction_fetch_unit_size = 64; // The processor fetches one line
5975 instruction_fetch_units = 1; // of 64 bytes
5976 %}
5977
5978 // We don't use an actual pipeline model so don't care about resources
5979 // or description. we do use pipeline classes to introduce fixed
5980 // latencies
5981
5982 //----------RESOURCES----------------------------------------------------------
5983 // Resources are the functional units available to the machine
5984
5985 resources( INS0, INS1, INS01 = INS0 | INS1,
5986 ALU0, ALU1, ALU = ALU0 | ALU1,
5987 MAC,
5988 DIV,
5989 BRANCH,
5990 LDST,
5991 NEON_FP);
5992
5993 //----------PIPELINE DESCRIPTION-----------------------------------------------
5994 // Pipeline Description specifies the stages in the machine's pipeline
5995
5996 // Define the pipeline as a generic 6 stage pipeline
5997 pipe_desc(S0, S1, S2, S3, S4, S5);
5998
5999 //----------PIPELINE CLASSES---------------------------------------------------
6000 // Pipeline Classes describe the stages in which input and output are
6001 // referenced by the hardware pipeline.
6002
6003 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
6004 %{
6005 single_instruction;
6006 src1 : S1(read);
6007 src2 : S2(read);
6008 dst : S5(write);
6009 INS01 : ISS;
6010 NEON_FP : S5;
6011 %}
6012
6013 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
6014 %{
6015 single_instruction;
6016 src1 : S1(read);
6017 src2 : S2(read);
6018 dst : S5(write);
6019 INS01 : ISS;
6020 NEON_FP : S5;
6021 %}
6022
6023 pipe_class fp_uop_s(vRegF dst, vRegF src)
6024 %{
6025 single_instruction;
6026 src : S1(read);
6027 dst : S5(write);
6028 INS01 : ISS;
6029 NEON_FP : S5;
6030 %}
6031
6032 pipe_class fp_uop_d(vRegD dst, vRegD src)
6033 %{
6034 single_instruction;
6035 src : S1(read);
6036 dst : S5(write);
6037 INS01 : ISS;
6038 NEON_FP : S5;
6039 %}
6040
6041 pipe_class fp_d2f(vRegF dst, vRegD src)
6042 %{
6043 single_instruction;
6044 src : S1(read);
6045 dst : S5(write);
6046 INS01 : ISS;
6047 NEON_FP : S5;
6048 %}
6049
6050 pipe_class fp_f2d(vRegD dst, vRegF src)
6051 %{
6052 single_instruction;
6053 src : S1(read);
6054 dst : S5(write);
6055 INS01 : ISS;
6056 NEON_FP : S5;
6057 %}
6058
6059 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
6060 %{
6061 single_instruction;
6062 src : S1(read);
6063 dst : S5(write);
6064 INS01 : ISS;
6065 NEON_FP : S5;
6066 %}
6067
6068 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
6069 %{
6070 single_instruction;
6071 src : S1(read);
6072 dst : S5(write);
6073 INS01 : ISS;
6074 NEON_FP : S5;
6075 %}
6076
6077 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
6078 %{
6079 single_instruction;
6080 src : S1(read);
6081 dst : S5(write);
6082 INS01 : ISS;
6083 NEON_FP : S5;
6084 %}
6085
6086 pipe_class fp_l2f(vRegF dst, iRegL src)
6087 %{
6088 single_instruction;
6089 src : S1(read);
6090 dst : S5(write);
6091 INS01 : ISS;
6092 NEON_FP : S5;
6093 %}
6094
6095 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
6096 %{
6097 single_instruction;
6098 src : S1(read);
6099 dst : S5(write);
6100 INS01 : ISS;
6101 NEON_FP : S5;
6102 %}
6103
6104 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
6105 %{
6106 single_instruction;
6107 src : S1(read);
6108 dst : S5(write);
6109 INS01 : ISS;
6110 NEON_FP : S5;
6111 %}
6112
6113 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
6114 %{
6115 single_instruction;
6116 src : S1(read);
6117 dst : S5(write);
6118 INS01 : ISS;
6119 NEON_FP : S5;
6120 %}
6121
6122 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
6123 %{
6124 single_instruction;
6125 src : S1(read);
6126 dst : S5(write);
6127 INS01 : ISS;
6128 NEON_FP : S5;
6129 %}
6130
6131 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
6132 %{
6133 single_instruction;
6134 src1 : S1(read);
6135 src2 : S2(read);
6136 dst : S5(write);
6137 INS0 : ISS;
6138 NEON_FP : S5;
6139 %}
6140
6141 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
6142 %{
6143 single_instruction;
6144 src1 : S1(read);
6145 src2 : S2(read);
6146 dst : S5(write);
6147 INS0 : ISS;
6148 NEON_FP : S5;
6149 %}
6150
6151 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
6152 %{
6153 single_instruction;
6154 cr : S1(read);
6155 src1 : S1(read);
6156 src2 : S1(read);
6157 dst : S3(write);
6158 INS01 : ISS;
6159 NEON_FP : S3;
6160 %}
6161
6162 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
6163 %{
6164 single_instruction;
6165 cr : S1(read);
6166 src1 : S1(read);
6167 src2 : S1(read);
6168 dst : S3(write);
6169 INS01 : ISS;
6170 NEON_FP : S3;
6171 %}
6172
6173 pipe_class fp_imm_s(vRegF dst)
6174 %{
6175 single_instruction;
6176 dst : S3(write);
6177 INS01 : ISS;
6178 NEON_FP : S3;
6179 %}
6180
6181 pipe_class fp_imm_d(vRegD dst)
6182 %{
6183 single_instruction;
6184 dst : S3(write);
6185 INS01 : ISS;
6186 NEON_FP : S3;
6187 %}
6188
6189 pipe_class fp_load_constant_s(vRegF dst)
6190 %{
6191 single_instruction;
6192 dst : S4(write);
6193 INS01 : ISS;
6194 NEON_FP : S4;
6195 %}
6196
6197 pipe_class fp_load_constant_d(vRegD dst)
6198 %{
6199 single_instruction;
6200 dst : S4(write);
6201 INS01 : ISS;
6202 NEON_FP : S4;
6203 %}
6204
6205 //------- Integer ALU operations --------------------------
6206
6207 // Integer ALU reg-reg operation
6208 // Operands needed in EX1, result generated in EX2
6209 // Eg. ADD x0, x1, x2
6210 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6211 %{
6212 single_instruction;
6213 dst : EX2(write);
6214 src1 : EX1(read);
6215 src2 : EX1(read);
6216 INS01 : ISS; // Dual issue as instruction 0 or 1
6217 ALU : EX2;
6218 %}
6219
6220 // Integer ALU reg-reg operation with constant shift
6221 // Shifted register must be available in LATE_ISS instead of EX1
6222 // Eg. ADD x0, x1, x2, LSL #2
6223 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6224 %{
6225 single_instruction;
6226 dst : EX2(write);
6227 src1 : EX1(read);
6228 src2 : ISS(read);
6229 INS01 : ISS;
6230 ALU : EX2;
6231 %}
6232
6233 // Integer ALU reg operation with constant shift
6234 // Eg. LSL x0, x1, #shift
6235 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6236 %{
6237 single_instruction;
6238 dst : EX2(write);
6239 src1 : ISS(read);
6240 INS01 : ISS;
6241 ALU : EX2;
6242 %}
6243
6244 // Integer ALU reg-reg operation with variable shift
6245 // Both operands must be available in LATE_ISS instead of EX1
6246 // Result is available in EX1 instead of EX2
6247 // Eg. LSLV x0, x1, x2
6248 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6249 %{
6250 single_instruction;
6251 dst : EX1(write);
6252 src1 : ISS(read);
6253 src2 : ISS(read);
6254 INS01 : ISS;
6255 ALU : EX1;
6256 %}
6257
6258 // Integer ALU reg-reg operation with extract
6259 // As for _vshift above, but result generated in EX2
6260 // Eg. EXTR x0, x1, x2, #N
6261 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6262 %{
6263 single_instruction;
6264 dst : EX2(write);
6265 src1 : ISS(read);
6266 src2 : ISS(read);
6267 INS1 : ISS; // Can only dual issue as Instruction 1
6268 ALU : EX1;
6269 %}
6270
6271 // Integer ALU reg operation
6272 // Eg. NEG x0, x1
6273 pipe_class ialu_reg(iRegI dst, iRegI src)
6274 %{
6275 single_instruction;
6276 dst : EX2(write);
6277 src : EX1(read);
6278 INS01 : ISS;
6279 ALU : EX2;
6280 %}
6281
6282 // Integer ALU reg mmediate operation
6283 // Eg. ADD x0, x1, #N
6284 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6285 %{
6286 single_instruction;
6287 dst : EX2(write);
6288 src1 : EX1(read);
6289 INS01 : ISS;
6290 ALU : EX2;
6291 %}
6292
6293 // Integer ALU immediate operation (no source operands)
6294 // Eg. MOV x0, #N
6295 pipe_class ialu_imm(iRegI dst)
6296 %{
6297 single_instruction;
6298 dst : EX1(write);
6299 INS01 : ISS;
6300 ALU : EX1;
6301 %}
6302
6303 //------- Compare operation -------------------------------
6304
6305 // Compare reg-reg
6306 // Eg. CMP x0, x1
6307 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6308 %{
6309 single_instruction;
6310 // fixed_latency(16);
6311 cr : EX2(write);
6312 op1 : EX1(read);
6313 op2 : EX1(read);
6314 INS01 : ISS;
6315 ALU : EX2;
6316 %}
6317
6318 // Compare reg-reg
6319 // Eg. CMP x0, #N
6320 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6321 %{
6322 single_instruction;
6323 // fixed_latency(16);
6324 cr : EX2(write);
6325 op1 : EX1(read);
6326 INS01 : ISS;
6327 ALU : EX2;
6328 %}
6329
6330 //------- Conditional instructions ------------------------
6331
6332 // Conditional no operands
6333 // Eg. CSINC x0, zr, zr, <cond>
6334 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6335 %{
6336 single_instruction;
6337 cr : EX1(read);
6338 dst : EX2(write);
6339 INS01 : ISS;
6340 ALU : EX2;
6341 %}
6342
6343 // Conditional 2 operand
6344 // EG. CSEL X0, X1, X2, <cond>
6345 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6346 %{
6347 single_instruction;
6348 cr : EX1(read);
6349 src1 : EX1(read);
6350 src2 : EX1(read);
6351 dst : EX2(write);
6352 INS01 : ISS;
6353 ALU : EX2;
6354 %}
6355
6356 // Conditional 2 operand
6357 // EG. CSEL X0, X1, X2, <cond>
6358 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6359 %{
6360 single_instruction;
6361 cr : EX1(read);
6362 src : EX1(read);
6363 dst : EX2(write);
6364 INS01 : ISS;
6365 ALU : EX2;
6366 %}
6367
6368 //------- Multiply pipeline operations --------------------
6369
6370 // Multiply reg-reg
6371 // Eg. MUL w0, w1, w2
6372 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6373 %{
6374 single_instruction;
6375 dst : WR(write);
6376 src1 : ISS(read);
6377 src2 : ISS(read);
6378 INS01 : ISS;
6379 MAC : WR;
6380 %}
6381
6382 // Multiply accumulate
6383 // Eg. MADD w0, w1, w2, w3
6384 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6385 %{
6386 single_instruction;
6387 dst : WR(write);
6388 src1 : ISS(read);
6389 src2 : ISS(read);
6390 src3 : ISS(read);
6391 INS01 : ISS;
6392 MAC : WR;
6393 %}
6394
6395 // Eg. MUL w0, w1, w2
6396 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6397 %{
6398 single_instruction;
6399 fixed_latency(3); // Maximum latency for 64 bit mul
6400 dst : WR(write);
6401 src1 : ISS(read);
6402 src2 : ISS(read);
6403 INS01 : ISS;
6404 MAC : WR;
6405 %}
6406
6407 // Multiply accumulate
6408 // Eg. MADD w0, w1, w2, w3
6409 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6410 %{
6411 single_instruction;
6412 fixed_latency(3); // Maximum latency for 64 bit mul
6413 dst : WR(write);
6414 src1 : ISS(read);
6415 src2 : ISS(read);
6416 src3 : ISS(read);
6417 INS01 : ISS;
6418 MAC : WR;
6419 %}
6420
6421 //------- Divide pipeline operations --------------------
6422
6423 // Eg. SDIV w0, w1, w2
6424 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6425 %{
6426 single_instruction;
6427 fixed_latency(8); // Maximum latency for 32 bit divide
6428 dst : WR(write);
6429 src1 : ISS(read);
6430 src2 : ISS(read);
6431 INS0 : ISS; // Can only dual issue as instruction 0
6432 DIV : WR;
6433 %}
6434
6435 // Eg. SDIV x0, x1, x2
6436 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6437 %{
6438 single_instruction;
6439 fixed_latency(16); // Maximum latency for 64 bit divide
6440 dst : WR(write);
6441 src1 : ISS(read);
6442 src2 : ISS(read);
6443 INS0 : ISS; // Can only dual issue as instruction 0
6444 DIV : WR;
6445 %}
6446
6447 //------- Load pipeline operations ------------------------
6448
6449 // Load - prefetch
6450 // Eg. PFRM <mem>
6451 pipe_class iload_prefetch(memory mem)
6452 %{
6453 single_instruction;
6454 mem : ISS(read);
6455 INS01 : ISS;
6456 LDST : WR;
6457 %}
6458
6459 // Load - reg, mem
6460 // Eg. LDR x0, <mem>
6461 pipe_class iload_reg_mem(iRegI dst, memory mem)
6462 %{
6463 single_instruction;
6464 dst : WR(write);
6465 mem : ISS(read);
6466 INS01 : ISS;
6467 LDST : WR;
6468 %}
6469
6470 // Load - reg, reg
6471 // Eg. LDR x0, [sp, x1]
6472 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6473 %{
6474 single_instruction;
6475 dst : WR(write);
6476 src : ISS(read);
6477 INS01 : ISS;
6478 LDST : WR;
6479 %}
6480
6481 //------- Store pipeline operations -----------------------
6482
6483 // Store - zr, mem
6484 // Eg. STR zr, <mem>
6485 pipe_class istore_mem(memory mem)
6486 %{
6487 single_instruction;
6488 mem : ISS(read);
6489 INS01 : ISS;
6490 LDST : WR;
6491 %}
6492
6493 // Store - reg, mem
6494 // Eg. STR x0, <mem>
6495 pipe_class istore_reg_mem(iRegI src, memory mem)
6496 %{
6497 single_instruction;
6498 mem : ISS(read);
6499 src : EX2(read);
6500 INS01 : ISS;
6501 LDST : WR;
6502 %}
6503
6504 // Store - reg, reg
6505 // Eg. STR x0, [sp, x1]
6506 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6507 %{
6508 single_instruction;
6509 dst : ISS(read);
6510 src : EX2(read);
6511 INS01 : ISS;
6512 LDST : WR;
6513 %}
6514
6515 //------- Store pipeline operations -----------------------
6516
6517 // Branch
6518 pipe_class pipe_branch()
6519 %{
6520 single_instruction;
6521 INS01 : ISS;
6522 BRANCH : EX1;
6523 %}
6524
6525 // Conditional branch
6526 pipe_class pipe_branch_cond(rFlagsReg cr)
6527 %{
6528 single_instruction;
6529 cr : EX1(read);
6530 INS01 : ISS;
6531 BRANCH : EX1;
6532 %}
6533
6534 // Compare & Branch
6535 // EG. CBZ/CBNZ
6536 pipe_class pipe_cmp_branch(iRegI op1)
6537 %{
6538 single_instruction;
6539 op1 : EX1(read);
6540 INS01 : ISS;
6541 BRANCH : EX1;
6542 %}
6543
6544 //------- Synchronisation operations ----------------------
6545
6546 // Any operation requiring serialization.
6547 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6548 pipe_class pipe_serial()
6549 %{
6550 single_instruction;
6551 force_serialization;
6552 fixed_latency(16);
6553 INS01 : ISS(2); // Cannot dual issue with any other instruction
6554 LDST : WR;
6555 %}
6556
6557 // Generic big/slow expanded idiom - also serialized
6558 pipe_class pipe_slow()
6559 %{
6560 instruction_count(10);
6561 multiple_bundles;
6562 force_serialization;
6563 fixed_latency(16);
6564 INS01 : ISS(2); // Cannot dual issue with any other instruction
6565 LDST : WR;
6566 %}
6567
6568 // Empty pipeline class
6569 pipe_class pipe_class_empty()
6570 %{
6571 single_instruction;
6572 fixed_latency(0);
6573 %}
6574
6575 // Default pipeline class.
6576 pipe_class pipe_class_default()
6577 %{
6578 single_instruction;
6579 fixed_latency(2);
6580 %}
6581
6582 // Pipeline class for compares.
6583 pipe_class pipe_class_compare()
6584 %{
6585 single_instruction;
6586 fixed_latency(16);
6587 %}
6588
6589 // Pipeline class for memory operations.
6590 pipe_class pipe_class_memory()
6591 %{
6592 single_instruction;
6593 fixed_latency(16);
6594 %}
6595
6596 // Pipeline class for call.
6597 pipe_class pipe_class_call()
6598 %{
6599 single_instruction;
6600 fixed_latency(100);
6601 %}
6602
6603 // Define the class for the Nop node.
6604 define %{
6605 MachNop = pipe_class_empty;
6606 %}
6607
6608 %}
6609 //----------INSTRUCTIONS-------------------------------------------------------
6610 //
6611 // match -- States which machine-independent subtree may be replaced
6612 // by this instruction.
6613 // ins_cost -- The estimated cost of this instruction is used by instruction
6614 // selection to identify a minimum cost tree of machine
6615 // instructions that matches a tree of machine-independent
6616 // instructions.
6617 // format -- A string providing the disassembly for this instruction.
6618 // The value of an instruction's operand may be inserted
6619 // by referring to it with a '$' prefix.
6620 // opcode -- Three instruction opcodes may be provided. These are referred
6621 // to within an encode class as $primary, $secondary, and $tertiary
6622 // rrspectively. The primary opcode is commonly used to
6623 // indicate the type of machine instruction, while secondary
6624 // and tertiary are often used for prefix options or addressing
6625 // modes.
6626 // ins_encode -- A list of encode classes with parameters. The encode class
6627 // name must have been defined in an 'enc_class' specification
6628 // in the encode section of the architecture description.
6629
6630 // ============================================================================
6631 // Memory (Load/Store) Instructions
6632
6633 // Load Instructions
6634
6635 // Load Byte (8 bit signed)
6636 instruct loadB(iRegINoSp dst, memory1 mem)
6637 %{
6638 match(Set dst (LoadB mem));
6639 predicate(!needs_acquiring_load(n));
6640
6641 ins_cost(4 * INSN_COST);
6642 format %{ "ldrsbw $dst, $mem\t# byte" %}
6643
6644 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6645
6646 ins_pipe(iload_reg_mem);
6647 %}
6648
6649 // Load Byte (8 bit signed) into long
6650 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6651 %{
6652 match(Set dst (ConvI2L (LoadB mem)));
6653 predicate(!needs_acquiring_load(n->in(1)));
6654
6655 ins_cost(4 * INSN_COST);
6656 format %{ "ldrsb $dst, $mem\t# byte" %}
6657
6658 ins_encode(aarch64_enc_ldrsb(dst, mem));
6659
6660 ins_pipe(iload_reg_mem);
6661 %}
6662
6663 // Load Byte (8 bit unsigned)
6664 instruct loadUB(iRegINoSp dst, memory1 mem)
6665 %{
6666 match(Set dst (LoadUB mem));
6667 predicate(!needs_acquiring_load(n));
6668
6669 ins_cost(4 * INSN_COST);
6670 format %{ "ldrbw $dst, $mem\t# byte" %}
6671
6672 ins_encode(aarch64_enc_ldrb(dst, mem));
6673
6674 ins_pipe(iload_reg_mem);
6675 %}
6676
6677 // Load Byte (8 bit unsigned) into long
6678 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6679 %{
6680 match(Set dst (ConvI2L (LoadUB mem)));
6681 predicate(!needs_acquiring_load(n->in(1)));
6682
6683 ins_cost(4 * INSN_COST);
6684 format %{ "ldrb $dst, $mem\t# byte" %}
6685
6686 ins_encode(aarch64_enc_ldrb(dst, mem));
6687
6688 ins_pipe(iload_reg_mem);
6689 %}
6690
6691 // Load Short (16 bit signed)
6692 instruct loadS(iRegINoSp dst, memory2 mem)
6693 %{
6694 match(Set dst (LoadS mem));
6695 predicate(!needs_acquiring_load(n));
6696
6697 ins_cost(4 * INSN_COST);
6698 format %{ "ldrshw $dst, $mem\t# short" %}
6699
6700 ins_encode(aarch64_enc_ldrshw(dst, mem));
6701
6702 ins_pipe(iload_reg_mem);
6703 %}
6704
6705 // Load Short (16 bit signed) into long
6706 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6707 %{
6708 match(Set dst (ConvI2L (LoadS mem)));
6709 predicate(!needs_acquiring_load(n->in(1)));
6710
6711 ins_cost(4 * INSN_COST);
6712 format %{ "ldrsh $dst, $mem\t# short" %}
6713
6714 ins_encode(aarch64_enc_ldrsh(dst, mem));
6715
6716 ins_pipe(iload_reg_mem);
6717 %}
6718
6719 // Load Char (16 bit unsigned)
6720 instruct loadUS(iRegINoSp dst, memory2 mem)
6721 %{
6722 match(Set dst (LoadUS mem));
6723 predicate(!needs_acquiring_load(n));
6724
6725 ins_cost(4 * INSN_COST);
6726 format %{ "ldrh $dst, $mem\t# short" %}
6727
6728 ins_encode(aarch64_enc_ldrh(dst, mem));
6729
6730 ins_pipe(iload_reg_mem);
6731 %}
6732
6733 // Load Short/Char (16 bit unsigned) into long
6734 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6735 %{
6736 match(Set dst (ConvI2L (LoadUS mem)));
6737 predicate(!needs_acquiring_load(n->in(1)));
6738
6739 ins_cost(4 * INSN_COST);
6740 format %{ "ldrh $dst, $mem\t# short" %}
6741
6742 ins_encode(aarch64_enc_ldrh(dst, mem));
6743
6744 ins_pipe(iload_reg_mem);
6745 %}
6746
6747 // Load Integer (32 bit signed)
6748 instruct loadI(iRegINoSp dst, memory4 mem)
6749 %{
6750 match(Set dst (LoadI mem));
6751 predicate(!needs_acquiring_load(n));
6752
6753 ins_cost(4 * INSN_COST);
6754 format %{ "ldrw $dst, $mem\t# int" %}
6755
6756 ins_encode(aarch64_enc_ldrw(dst, mem));
6757
6758 ins_pipe(iload_reg_mem);
6759 %}
6760
6761 // Load Integer (32 bit signed) into long
6762 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6763 %{
6764 match(Set dst (ConvI2L (LoadI mem)));
6765 predicate(!needs_acquiring_load(n->in(1)));
6766
6767 ins_cost(4 * INSN_COST);
6768 format %{ "ldrsw $dst, $mem\t# int" %}
6769
6770 ins_encode(aarch64_enc_ldrsw(dst, mem));
6771
6772 ins_pipe(iload_reg_mem);
6773 %}
6774
6775 // Load Integer (32 bit unsigned) into long
6776 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6777 %{
6778 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6779 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6780
6781 ins_cost(4 * INSN_COST);
6782 format %{ "ldrw $dst, $mem\t# int" %}
6783
6784 ins_encode(aarch64_enc_ldrw(dst, mem));
6785
6786 ins_pipe(iload_reg_mem);
6787 %}
6788
6789 // Load Long (64 bit signed)
6790 instruct loadL(iRegLNoSp dst, memory8 mem)
6791 %{
6792 match(Set dst (LoadL mem));
6793 predicate(!needs_acquiring_load(n));
6794
6795 ins_cost(4 * INSN_COST);
6796 format %{ "ldr $dst, $mem\t# int" %}
6797
6798 ins_encode(aarch64_enc_ldr(dst, mem));
6799
6800 ins_pipe(iload_reg_mem);
6801 %}
6802
6803 // Load Range
6804 instruct loadRange(iRegINoSp dst, memory4 mem)
6805 %{
6806 match(Set dst (LoadRange mem));
6807
6808 ins_cost(4 * INSN_COST);
6809 format %{ "ldrw $dst, $mem\t# range" %}
6810
6811 ins_encode(aarch64_enc_ldrw(dst, mem));
6812
6813 ins_pipe(iload_reg_mem);
6814 %}
6815
6816 // Load Pointer
6817 instruct loadP(iRegPNoSp dst, memory8 mem)
6818 %{
6819 match(Set dst (LoadP mem));
6820 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6821
6822 ins_cost(4 * INSN_COST);
6823 format %{ "ldr $dst, $mem\t# ptr" %}
6824
6825 ins_encode(aarch64_enc_ldr(dst, mem));
6826
6827 ins_pipe(iload_reg_mem);
6828 %}
6829
6830 // Load Compressed Pointer
6831 instruct loadN(iRegNNoSp dst, memory4 mem)
6832 %{
6833 match(Set dst (LoadN mem));
6834 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6835
6836 ins_cost(4 * INSN_COST);
6837 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6838
6839 ins_encode(aarch64_enc_ldrw(dst, mem));
6840
6841 ins_pipe(iload_reg_mem);
6842 %}
6843
6844 // Load Klass Pointer
6845 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6846 %{
6847 match(Set dst (LoadKlass mem));
6848 predicate(!needs_acquiring_load(n));
6849
6850 ins_cost(4 * INSN_COST);
6851 format %{ "ldr $dst, $mem\t# class" %}
6852
6853 ins_encode(aarch64_enc_ldr(dst, mem));
6854
6855 ins_pipe(iload_reg_mem);
6856 %}
6857
6858 // Load Narrow Klass Pointer
6859 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6860 %{
6861 match(Set dst (LoadNKlass mem));
6862 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6863
6864 ins_cost(4 * INSN_COST);
6865 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6866
6867 ins_encode(aarch64_enc_ldrw(dst, mem));
6868
6869 ins_pipe(iload_reg_mem);
6870 %}
6871
6872 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem)
6873 %{
6874 match(Set dst (LoadNKlass mem));
6875 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6876
6877 ins_cost(4 * INSN_COST);
6878 format %{
6879 "ldrw $dst, $mem\t# compressed class ptr, shifted\n\t"
6880 "lsrw $dst, $dst, markWord::klass_shift_at_offset"
6881 %}
6882 ins_encode %{
6883 // inlined aarch64_enc_ldrw
6884 loadStore(masm, &MacroAssembler::ldrw, $dst$$Register, $mem->opcode(),
6885 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
6886 __ lsrw($dst$$Register, $dst$$Register, markWord::klass_shift_at_offset);
6887 %}
6888 ins_pipe(iload_reg_mem);
6889 %}
6890
6891 // Load Float
6892 instruct loadF(vRegF dst, memory4 mem)
6893 %{
6894 match(Set dst (LoadF mem));
6895 predicate(!needs_acquiring_load(n));
6896
6897 ins_cost(4 * INSN_COST);
6898 format %{ "ldrs $dst, $mem\t# float" %}
6899
6900 ins_encode( aarch64_enc_ldrs(dst, mem) );
6901
6902 ins_pipe(pipe_class_memory);
6903 %}
6904
6905 // Load Double
6906 instruct loadD(vRegD dst, memory8 mem)
6907 %{
6908 match(Set dst (LoadD mem));
6909 predicate(!needs_acquiring_load(n));
6910
6911 ins_cost(4 * INSN_COST);
6912 format %{ "ldrd $dst, $mem\t# double" %}
6913
6914 ins_encode( aarch64_enc_ldrd(dst, mem) );
6915
6916 ins_pipe(pipe_class_memory);
6917 %}
6918
6919
6920 // Load Int Constant
6921 instruct loadConI(iRegINoSp dst, immI src)
6922 %{
6923 match(Set dst src);
6924
6925 ins_cost(INSN_COST);
6926 format %{ "mov $dst, $src\t# int" %}
6927
6928 ins_encode( aarch64_enc_movw_imm(dst, src) );
6929
6930 ins_pipe(ialu_imm);
6931 %}
6932
6933 // Load Long Constant
6934 instruct loadConL(iRegLNoSp dst, immL src)
6935 %{
6936 match(Set dst src);
6937
6938 ins_cost(INSN_COST);
6939 format %{ "mov $dst, $src\t# long" %}
6940
6941 ins_encode( aarch64_enc_mov_imm(dst, src) );
6942
6943 ins_pipe(ialu_imm);
6944 %}
6945
6946 // Load Pointer Constant
6947
6948 instruct loadConP(iRegPNoSp dst, immP con)
6949 %{
6950 match(Set dst con);
6951
6952 ins_cost(INSN_COST * 4);
6953 format %{
6954 "mov $dst, $con\t# ptr\n\t"
6955 %}
6956
6957 ins_encode(aarch64_enc_mov_p(dst, con));
6958
6959 ins_pipe(ialu_imm);
6960 %}
6961
6962 // Load Null Pointer Constant
6963
6964 instruct loadConP0(iRegPNoSp dst, immP0 con)
6965 %{
6966 match(Set dst con);
6967
6968 ins_cost(INSN_COST);
6969 format %{ "mov $dst, $con\t# nullptr ptr" %}
6970
6971 ins_encode(aarch64_enc_mov_p0(dst, con));
6972
6973 ins_pipe(ialu_imm);
6974 %}
6975
6976 // Load Pointer Constant One
6977
6978 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6979 %{
6980 match(Set dst con);
6981
6982 ins_cost(INSN_COST);
6983 format %{ "mov $dst, $con\t# nullptr ptr" %}
6984
6985 ins_encode(aarch64_enc_mov_p1(dst, con));
6986
6987 ins_pipe(ialu_imm);
6988 %}
6989
6990 instruct loadAOTRCAddress(iRegPNoSp dst, immAOTRuntimeConstantsAddress con)
6991 %{
6992 match(Set dst con);
6993
6994 ins_cost(INSN_COST);
6995 format %{ "adr $dst, $con\t# AOT Runtime Constants Address" %}
6996
6997 ins_encode %{
6998 __ load_aotrc_address($dst$$Register, (address)$con$$constant);
6999 %}
7000
7001 ins_pipe(ialu_imm);
7002 %}
7003
7004 // Load Narrow Pointer Constant
7005
7006 instruct loadConN(iRegNNoSp dst, immN con)
7007 %{
7008 match(Set dst con);
7009
7010 ins_cost(INSN_COST * 4);
7011 format %{ "mov $dst, $con\t# compressed ptr" %}
7012
7013 ins_encode(aarch64_enc_mov_n(dst, con));
7014
7015 ins_pipe(ialu_imm);
7016 %}
7017
7018 // Load Narrow Null Pointer Constant
7019
7020 instruct loadConN0(iRegNNoSp dst, immN0 con)
7021 %{
7022 match(Set dst con);
7023
7024 ins_cost(INSN_COST);
7025 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
7026
7027 ins_encode(aarch64_enc_mov_n0(dst, con));
7028
7029 ins_pipe(ialu_imm);
7030 %}
7031
7032 // Load Narrow Klass Constant
7033
7034 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
7035 %{
7036 match(Set dst con);
7037
7038 ins_cost(INSN_COST);
7039 format %{ "mov $dst, $con\t# compressed klass ptr" %}
7040
7041 ins_encode(aarch64_enc_mov_nk(dst, con));
7042
7043 ins_pipe(ialu_imm);
7044 %}
7045
7046 // Load Packed Float Constant
7047
7048 instruct loadConF_packed(vRegF dst, immFPacked con) %{
7049 match(Set dst con);
7050 ins_cost(INSN_COST * 4);
7051 format %{ "fmovs $dst, $con"%}
7052 ins_encode %{
7053 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
7054 %}
7055
7056 ins_pipe(fp_imm_s);
7057 %}
7058
7059 // Load Float Constant
7060
7061 instruct loadConF(vRegF dst, immF con) %{
7062 match(Set dst con);
7063
7064 ins_cost(INSN_COST * 4);
7065
7066 format %{
7067 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7068 %}
7069
7070 ins_encode %{
7071 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
7072 %}
7073
7074 ins_pipe(fp_load_constant_s);
7075 %}
7076
7077 // Load Packed Double Constant
7078
7079 instruct loadConD_packed(vRegD dst, immDPacked con) %{
7080 match(Set dst con);
7081 ins_cost(INSN_COST);
7082 format %{ "fmovd $dst, $con"%}
7083 ins_encode %{
7084 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
7085 %}
7086
7087 ins_pipe(fp_imm_d);
7088 %}
7089
7090 // Load Double Constant
7091
7092 instruct loadConD(vRegD dst, immD con) %{
7093 match(Set dst con);
7094
7095 ins_cost(INSN_COST * 5);
7096 format %{
7097 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
7098 %}
7099
7100 ins_encode %{
7101 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
7102 %}
7103
7104 ins_pipe(fp_load_constant_d);
7105 %}
7106
7107 // Load Half Float Constant
7108 instruct loadConH(vRegF dst, immH con) %{
7109 match(Set dst con);
7110 format %{ "mov rscratch1, $con\n\t"
7111 "fmov $dst, rscratch1"
7112 %}
7113 ins_encode %{
7114 __ movw(rscratch1, (uint32_t)$con$$constant);
7115 __ fmovs($dst$$FloatRegister, rscratch1);
7116 %}
7117 ins_pipe(pipe_class_default);
7118 %}
7119
7120 // Store Instructions
7121
7122 // Store Byte
7123 instruct storeB(iRegIorL2I src, memory1 mem)
7124 %{
7125 match(Set mem (StoreB mem src));
7126 predicate(!needs_releasing_store(n));
7127
7128 ins_cost(INSN_COST);
7129 format %{ "strb $src, $mem\t# byte" %}
7130
7131 ins_encode(aarch64_enc_strb(src, mem));
7132
7133 ins_pipe(istore_reg_mem);
7134 %}
7135
7136
7137 instruct storeimmB0(immI0 zero, memory1 mem)
7138 %{
7139 match(Set mem (StoreB mem zero));
7140 predicate(!needs_releasing_store(n));
7141
7142 ins_cost(INSN_COST);
7143 format %{ "strb rscractch2, $mem\t# byte" %}
7144
7145 ins_encode(aarch64_enc_strb0(mem));
7146
7147 ins_pipe(istore_mem);
7148 %}
7149
7150 // Store Char/Short
7151 instruct storeC(iRegIorL2I src, memory2 mem)
7152 %{
7153 match(Set mem (StoreC mem src));
7154 predicate(!needs_releasing_store(n));
7155
7156 ins_cost(INSN_COST);
7157 format %{ "strh $src, $mem\t# short" %}
7158
7159 ins_encode(aarch64_enc_strh(src, mem));
7160
7161 ins_pipe(istore_reg_mem);
7162 %}
7163
7164 instruct storeimmC0(immI0 zero, memory2 mem)
7165 %{
7166 match(Set mem (StoreC mem zero));
7167 predicate(!needs_releasing_store(n));
7168
7169 ins_cost(INSN_COST);
7170 format %{ "strh zr, $mem\t# short" %}
7171
7172 ins_encode(aarch64_enc_strh0(mem));
7173
7174 ins_pipe(istore_mem);
7175 %}
7176
7177 // Store Integer
7178
7179 instruct storeI(iRegIorL2I src, memory4 mem)
7180 %{
7181 match(Set mem(StoreI mem src));
7182 predicate(!needs_releasing_store(n));
7183
7184 ins_cost(INSN_COST);
7185 format %{ "strw $src, $mem\t# int" %}
7186
7187 ins_encode(aarch64_enc_strw(src, mem));
7188
7189 ins_pipe(istore_reg_mem);
7190 %}
7191
7192 instruct storeimmI0(immI0 zero, memory4 mem)
7193 %{
7194 match(Set mem(StoreI mem zero));
7195 predicate(!needs_releasing_store(n));
7196
7197 ins_cost(INSN_COST);
7198 format %{ "strw zr, $mem\t# int" %}
7199
7200 ins_encode(aarch64_enc_strw0(mem));
7201
7202 ins_pipe(istore_mem);
7203 %}
7204
7205 // Store Long (64 bit signed)
7206 instruct storeL(iRegL src, memory8 mem)
7207 %{
7208 match(Set mem (StoreL mem src));
7209 predicate(!needs_releasing_store(n));
7210
7211 ins_cost(INSN_COST);
7212 format %{ "str $src, $mem\t# int" %}
7213
7214 ins_encode(aarch64_enc_str(src, mem));
7215
7216 ins_pipe(istore_reg_mem);
7217 %}
7218
7219 // Store Long (64 bit signed)
7220 instruct storeimmL0(immL0 zero, memory8 mem)
7221 %{
7222 match(Set mem (StoreL mem zero));
7223 predicate(!needs_releasing_store(n));
7224
7225 ins_cost(INSN_COST);
7226 format %{ "str zr, $mem\t# int" %}
7227
7228 ins_encode(aarch64_enc_str0(mem));
7229
7230 ins_pipe(istore_mem);
7231 %}
7232
7233 // Store Pointer
7234 instruct storeP(iRegP src, memory8 mem)
7235 %{
7236 match(Set mem (StoreP mem src));
7237 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7238
7239 ins_cost(INSN_COST);
7240 format %{ "str $src, $mem\t# ptr" %}
7241
7242 ins_encode(aarch64_enc_str(src, mem));
7243
7244 ins_pipe(istore_reg_mem);
7245 %}
7246
7247 // Store Pointer
7248 instruct storeimmP0(immP0 zero, memory8 mem)
7249 %{
7250 match(Set mem (StoreP mem zero));
7251 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7252
7253 ins_cost(INSN_COST);
7254 format %{ "str zr, $mem\t# ptr" %}
7255
7256 ins_encode(aarch64_enc_str0(mem));
7257
7258 ins_pipe(istore_mem);
7259 %}
7260
7261 // Store Compressed Pointer
7262 instruct storeN(iRegN src, memory4 mem)
7263 %{
7264 match(Set mem (StoreN mem src));
7265 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7266
7267 ins_cost(INSN_COST);
7268 format %{ "strw $src, $mem\t# compressed ptr" %}
7269
7270 ins_encode(aarch64_enc_strw(src, mem));
7271
7272 ins_pipe(istore_reg_mem);
7273 %}
7274
7275 instruct storeImmN0(immN0 zero, memory4 mem)
7276 %{
7277 match(Set mem (StoreN mem zero));
7278 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7279
7280 ins_cost(INSN_COST);
7281 format %{ "strw zr, $mem\t# compressed ptr" %}
7282
7283 ins_encode(aarch64_enc_strw0(mem));
7284
7285 ins_pipe(istore_mem);
7286 %}
7287
7288 // Store Float
7289 instruct storeF(vRegF src, memory4 mem)
7290 %{
7291 match(Set mem (StoreF mem src));
7292 predicate(!needs_releasing_store(n));
7293
7294 ins_cost(INSN_COST);
7295 format %{ "strs $src, $mem\t# float" %}
7296
7297 ins_encode( aarch64_enc_strs(src, mem) );
7298
7299 ins_pipe(pipe_class_memory);
7300 %}
7301
7302 // TODO
7303 // implement storeImmF0 and storeFImmPacked
7304
7305 // Store Double
7306 instruct storeD(vRegD src, memory8 mem)
7307 %{
7308 match(Set mem (StoreD mem src));
7309 predicate(!needs_releasing_store(n));
7310
7311 ins_cost(INSN_COST);
7312 format %{ "strd $src, $mem\t# double" %}
7313
7314 ins_encode( aarch64_enc_strd(src, mem) );
7315
7316 ins_pipe(pipe_class_memory);
7317 %}
7318
7319 // Store Compressed Klass Pointer
7320 instruct storeNKlass(iRegN src, memory4 mem)
7321 %{
7322 predicate(!needs_releasing_store(n));
7323 match(Set mem (StoreNKlass mem src));
7324
7325 ins_cost(INSN_COST);
7326 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7327
7328 ins_encode(aarch64_enc_strw(src, mem));
7329
7330 ins_pipe(istore_reg_mem);
7331 %}
7332
7333 // TODO
7334 // implement storeImmD0 and storeDImmPacked
7335
7336 // prefetch instructions
7337 // Must be safe to execute with invalid address (cannot fault).
7338
7339 instruct prefetchalloc( memory8 mem ) %{
7340 match(PrefetchAllocation mem);
7341
7342 ins_cost(INSN_COST);
7343 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7344
7345 ins_encode( aarch64_enc_prefetchw(mem) );
7346
7347 ins_pipe(iload_prefetch);
7348 %}
7349
7350 // ---------------- volatile loads and stores ----------------
7351
7352 // Load Byte (8 bit signed)
7353 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7354 %{
7355 match(Set dst (LoadB mem));
7356
7357 ins_cost(VOLATILE_REF_COST);
7358 format %{ "ldarsb $dst, $mem\t# byte" %}
7359
7360 ins_encode(aarch64_enc_ldarsb(dst, mem));
7361
7362 ins_pipe(pipe_serial);
7363 %}
7364
7365 // Load Byte (8 bit signed) into long
7366 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7367 %{
7368 match(Set dst (ConvI2L (LoadB mem)));
7369
7370 ins_cost(VOLATILE_REF_COST);
7371 format %{ "ldarsb $dst, $mem\t# byte" %}
7372
7373 ins_encode(aarch64_enc_ldarsb(dst, mem));
7374
7375 ins_pipe(pipe_serial);
7376 %}
7377
7378 // Load Byte (8 bit unsigned)
7379 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7380 %{
7381 match(Set dst (LoadUB mem));
7382
7383 ins_cost(VOLATILE_REF_COST);
7384 format %{ "ldarb $dst, $mem\t# byte" %}
7385
7386 ins_encode(aarch64_enc_ldarb(dst, mem));
7387
7388 ins_pipe(pipe_serial);
7389 %}
7390
7391 // Load Byte (8 bit unsigned) into long
7392 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7393 %{
7394 match(Set dst (ConvI2L (LoadUB mem)));
7395
7396 ins_cost(VOLATILE_REF_COST);
7397 format %{ "ldarb $dst, $mem\t# byte" %}
7398
7399 ins_encode(aarch64_enc_ldarb(dst, mem));
7400
7401 ins_pipe(pipe_serial);
7402 %}
7403
7404 // Load Short (16 bit signed)
7405 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7406 %{
7407 match(Set dst (LoadS mem));
7408
7409 ins_cost(VOLATILE_REF_COST);
7410 format %{ "ldarshw $dst, $mem\t# short" %}
7411
7412 ins_encode(aarch64_enc_ldarshw(dst, mem));
7413
7414 ins_pipe(pipe_serial);
7415 %}
7416
7417 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7418 %{
7419 match(Set dst (LoadUS mem));
7420
7421 ins_cost(VOLATILE_REF_COST);
7422 format %{ "ldarhw $dst, $mem\t# short" %}
7423
7424 ins_encode(aarch64_enc_ldarhw(dst, mem));
7425
7426 ins_pipe(pipe_serial);
7427 %}
7428
7429 // Load Short/Char (16 bit unsigned) into long
7430 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7431 %{
7432 match(Set dst (ConvI2L (LoadUS mem)));
7433
7434 ins_cost(VOLATILE_REF_COST);
7435 format %{ "ldarh $dst, $mem\t# short" %}
7436
7437 ins_encode(aarch64_enc_ldarh(dst, mem));
7438
7439 ins_pipe(pipe_serial);
7440 %}
7441
7442 // Load Short/Char (16 bit signed) into long
7443 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7444 %{
7445 match(Set dst (ConvI2L (LoadS mem)));
7446
7447 ins_cost(VOLATILE_REF_COST);
7448 format %{ "ldarh $dst, $mem\t# short" %}
7449
7450 ins_encode(aarch64_enc_ldarsh(dst, mem));
7451
7452 ins_pipe(pipe_serial);
7453 %}
7454
7455 // Load Integer (32 bit signed)
7456 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7457 %{
7458 match(Set dst (LoadI mem));
7459
7460 ins_cost(VOLATILE_REF_COST);
7461 format %{ "ldarw $dst, $mem\t# int" %}
7462
7463 ins_encode(aarch64_enc_ldarw(dst, mem));
7464
7465 ins_pipe(pipe_serial);
7466 %}
7467
7468 // Load Integer (32 bit unsigned) into long
7469 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7470 %{
7471 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7472
7473 ins_cost(VOLATILE_REF_COST);
7474 format %{ "ldarw $dst, $mem\t# int" %}
7475
7476 ins_encode(aarch64_enc_ldarw(dst, mem));
7477
7478 ins_pipe(pipe_serial);
7479 %}
7480
7481 // Load Long (64 bit signed)
7482 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7483 %{
7484 match(Set dst (LoadL mem));
7485
7486 ins_cost(VOLATILE_REF_COST);
7487 format %{ "ldar $dst, $mem\t# int" %}
7488
7489 ins_encode(aarch64_enc_ldar(dst, mem));
7490
7491 ins_pipe(pipe_serial);
7492 %}
7493
7494 // Load Pointer
7495 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7496 %{
7497 match(Set dst (LoadP mem));
7498 predicate(n->as_Load()->barrier_data() == 0);
7499
7500 ins_cost(VOLATILE_REF_COST);
7501 format %{ "ldar $dst, $mem\t# ptr" %}
7502
7503 ins_encode(aarch64_enc_ldar(dst, mem));
7504
7505 ins_pipe(pipe_serial);
7506 %}
7507
7508 // Load Compressed Pointer
7509 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7510 %{
7511 match(Set dst (LoadN mem));
7512 predicate(n->as_Load()->barrier_data() == 0);
7513
7514 ins_cost(VOLATILE_REF_COST);
7515 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7516
7517 ins_encode(aarch64_enc_ldarw(dst, mem));
7518
7519 ins_pipe(pipe_serial);
7520 %}
7521
7522 // Load Float
7523 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7524 %{
7525 match(Set dst (LoadF mem));
7526
7527 ins_cost(VOLATILE_REF_COST);
7528 format %{ "ldars $dst, $mem\t# float" %}
7529
7530 ins_encode( aarch64_enc_fldars(dst, mem) );
7531
7532 ins_pipe(pipe_serial);
7533 %}
7534
7535 // Load Double
7536 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7537 %{
7538 match(Set dst (LoadD mem));
7539
7540 ins_cost(VOLATILE_REF_COST);
7541 format %{ "ldard $dst, $mem\t# double" %}
7542
7543 ins_encode( aarch64_enc_fldard(dst, mem) );
7544
7545 ins_pipe(pipe_serial);
7546 %}
7547
7548 // Store Byte
7549 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7550 %{
7551 match(Set mem (StoreB mem src));
7552
7553 ins_cost(VOLATILE_REF_COST);
7554 format %{ "stlrb $src, $mem\t# byte" %}
7555
7556 ins_encode(aarch64_enc_stlrb(src, mem));
7557
7558 ins_pipe(pipe_class_memory);
7559 %}
7560
7561 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7562 %{
7563 match(Set mem (StoreB mem zero));
7564
7565 ins_cost(VOLATILE_REF_COST);
7566 format %{ "stlrb zr, $mem\t# byte" %}
7567
7568 ins_encode(aarch64_enc_stlrb0(mem));
7569
7570 ins_pipe(pipe_class_memory);
7571 %}
7572
7573 // Store Char/Short
7574 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7575 %{
7576 match(Set mem (StoreC mem src));
7577
7578 ins_cost(VOLATILE_REF_COST);
7579 format %{ "stlrh $src, $mem\t# short" %}
7580
7581 ins_encode(aarch64_enc_stlrh(src, mem));
7582
7583 ins_pipe(pipe_class_memory);
7584 %}
7585
7586 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7587 %{
7588 match(Set mem (StoreC mem zero));
7589
7590 ins_cost(VOLATILE_REF_COST);
7591 format %{ "stlrh zr, $mem\t# short" %}
7592
7593 ins_encode(aarch64_enc_stlrh0(mem));
7594
7595 ins_pipe(pipe_class_memory);
7596 %}
7597
7598 // Store Integer
7599
7600 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7601 %{
7602 match(Set mem(StoreI mem src));
7603
7604 ins_cost(VOLATILE_REF_COST);
7605 format %{ "stlrw $src, $mem\t# int" %}
7606
7607 ins_encode(aarch64_enc_stlrw(src, mem));
7608
7609 ins_pipe(pipe_class_memory);
7610 %}
7611
7612 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7613 %{
7614 match(Set mem(StoreI mem zero));
7615
7616 ins_cost(VOLATILE_REF_COST);
7617 format %{ "stlrw zr, $mem\t# int" %}
7618
7619 ins_encode(aarch64_enc_stlrw0(mem));
7620
7621 ins_pipe(pipe_class_memory);
7622 %}
7623
7624 // Store Long (64 bit signed)
7625 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7626 %{
7627 match(Set mem (StoreL mem src));
7628
7629 ins_cost(VOLATILE_REF_COST);
7630 format %{ "stlr $src, $mem\t# int" %}
7631
7632 ins_encode(aarch64_enc_stlr(src, mem));
7633
7634 ins_pipe(pipe_class_memory);
7635 %}
7636
7637 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7638 %{
7639 match(Set mem (StoreL mem zero));
7640
7641 ins_cost(VOLATILE_REF_COST);
7642 format %{ "stlr zr, $mem\t# int" %}
7643
7644 ins_encode(aarch64_enc_stlr0(mem));
7645
7646 ins_pipe(pipe_class_memory);
7647 %}
7648
7649 // Store Pointer
7650 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7651 %{
7652 match(Set mem (StoreP mem src));
7653 predicate(n->as_Store()->barrier_data() == 0);
7654
7655 ins_cost(VOLATILE_REF_COST);
7656 format %{ "stlr $src, $mem\t# ptr" %}
7657
7658 ins_encode(aarch64_enc_stlr(src, mem));
7659
7660 ins_pipe(pipe_class_memory);
7661 %}
7662
7663 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7664 %{
7665 match(Set mem (StoreP mem zero));
7666 predicate(n->as_Store()->barrier_data() == 0);
7667
7668 ins_cost(VOLATILE_REF_COST);
7669 format %{ "stlr zr, $mem\t# ptr" %}
7670
7671 ins_encode(aarch64_enc_stlr0(mem));
7672
7673 ins_pipe(pipe_class_memory);
7674 %}
7675
7676 // Store Compressed Pointer
7677 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7678 %{
7679 match(Set mem (StoreN mem src));
7680 predicate(n->as_Store()->barrier_data() == 0);
7681
7682 ins_cost(VOLATILE_REF_COST);
7683 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7684
7685 ins_encode(aarch64_enc_stlrw(src, mem));
7686
7687 ins_pipe(pipe_class_memory);
7688 %}
7689
7690 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7691 %{
7692 match(Set mem (StoreN mem zero));
7693 predicate(n->as_Store()->barrier_data() == 0);
7694
7695 ins_cost(VOLATILE_REF_COST);
7696 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7697
7698 ins_encode(aarch64_enc_stlrw0(mem));
7699
7700 ins_pipe(pipe_class_memory);
7701 %}
7702
7703 // Store Float
7704 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7705 %{
7706 match(Set mem (StoreF mem src));
7707
7708 ins_cost(VOLATILE_REF_COST);
7709 format %{ "stlrs $src, $mem\t# float" %}
7710
7711 ins_encode( aarch64_enc_fstlrs(src, mem) );
7712
7713 ins_pipe(pipe_class_memory);
7714 %}
7715
7716 // TODO
7717 // implement storeImmF0 and storeFImmPacked
7718
7719 // Store Double
7720 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7721 %{
7722 match(Set mem (StoreD mem src));
7723
7724 ins_cost(VOLATILE_REF_COST);
7725 format %{ "stlrd $src, $mem\t# double" %}
7726
7727 ins_encode( aarch64_enc_fstlrd(src, mem) );
7728
7729 ins_pipe(pipe_class_memory);
7730 %}
7731
7732 // ---------------- end of volatile loads and stores ----------------
7733
7734 instruct cacheWB(indirect addr)
7735 %{
7736 predicate(VM_Version::supports_data_cache_line_flush());
7737 match(CacheWB addr);
7738
7739 ins_cost(100);
7740 format %{"cache wb $addr" %}
7741 ins_encode %{
7742 assert($addr->index_position() < 0, "should be");
7743 assert($addr$$disp == 0, "should be");
7744 __ cache_wb(Address($addr$$base$$Register, 0));
7745 %}
7746 ins_pipe(pipe_slow); // XXX
7747 %}
7748
7749 instruct cacheWBPreSync()
7750 %{
7751 predicate(VM_Version::supports_data_cache_line_flush());
7752 match(CacheWBPreSync);
7753
7754 ins_cost(100);
7755 format %{"cache wb presync" %}
7756 ins_encode %{
7757 __ cache_wbsync(true);
7758 %}
7759 ins_pipe(pipe_slow); // XXX
7760 %}
7761
7762 instruct cacheWBPostSync()
7763 %{
7764 predicate(VM_Version::supports_data_cache_line_flush());
7765 match(CacheWBPostSync);
7766
7767 ins_cost(100);
7768 format %{"cache wb postsync" %}
7769 ins_encode %{
7770 __ cache_wbsync(false);
7771 %}
7772 ins_pipe(pipe_slow); // XXX
7773 %}
7774
7775 // ============================================================================
7776 // BSWAP Instructions
7777
7778 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7779 match(Set dst (ReverseBytesI src));
7780
7781 ins_cost(INSN_COST);
7782 format %{ "revw $dst, $src" %}
7783
7784 ins_encode %{
7785 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7786 %}
7787
7788 ins_pipe(ialu_reg);
7789 %}
7790
7791 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7792 match(Set dst (ReverseBytesL src));
7793
7794 ins_cost(INSN_COST);
7795 format %{ "rev $dst, $src" %}
7796
7797 ins_encode %{
7798 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7799 %}
7800
7801 ins_pipe(ialu_reg);
7802 %}
7803
7804 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7805 match(Set dst (ReverseBytesUS src));
7806
7807 ins_cost(INSN_COST);
7808 format %{ "rev16w $dst, $src" %}
7809
7810 ins_encode %{
7811 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7812 %}
7813
7814 ins_pipe(ialu_reg);
7815 %}
7816
7817 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7818 match(Set dst (ReverseBytesS src));
7819
7820 ins_cost(INSN_COST);
7821 format %{ "rev16w $dst, $src\n\t"
7822 "sbfmw $dst, $dst, #0, #15" %}
7823
7824 ins_encode %{
7825 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7826 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7827 %}
7828
7829 ins_pipe(ialu_reg);
7830 %}
7831
7832 // ============================================================================
7833 // Zero Count Instructions
7834
7835 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7836 match(Set dst (CountLeadingZerosI src));
7837
7838 ins_cost(INSN_COST);
7839 format %{ "clzw $dst, $src" %}
7840 ins_encode %{
7841 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7842 %}
7843
7844 ins_pipe(ialu_reg);
7845 %}
7846
7847 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7848 match(Set dst (CountLeadingZerosL src));
7849
7850 ins_cost(INSN_COST);
7851 format %{ "clz $dst, $src" %}
7852 ins_encode %{
7853 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7854 %}
7855
7856 ins_pipe(ialu_reg);
7857 %}
7858
7859 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7860 match(Set dst (CountTrailingZerosI src));
7861
7862 ins_cost(INSN_COST * 2);
7863 format %{ "rbitw $dst, $src\n\t"
7864 "clzw $dst, $dst" %}
7865 ins_encode %{
7866 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7867 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7868 %}
7869
7870 ins_pipe(ialu_reg);
7871 %}
7872
7873 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7874 match(Set dst (CountTrailingZerosL src));
7875
7876 ins_cost(INSN_COST * 2);
7877 format %{ "rbit $dst, $src\n\t"
7878 "clz $dst, $dst" %}
7879 ins_encode %{
7880 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7881 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7882 %}
7883
7884 ins_pipe(ialu_reg);
7885 %}
7886
7887 //---------- Population Count Instructions -------------------------------------
7888 //
7889
7890 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7891 match(Set dst (PopCountI src));
7892 effect(TEMP tmp);
7893 ins_cost(INSN_COST * 13);
7894
7895 format %{ "fmovs $tmp, $src\t# vector (1S)\n\t"
7896 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7897 "addv $tmp, $tmp\t# vector (8B)\n\t"
7898 "mov $dst, $tmp\t# vector (1D)" %}
7899 ins_encode %{
7900 __ fmovs($tmp$$FloatRegister, $src$$Register);
7901 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7902 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7903 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7904 %}
7905
7906 ins_pipe(pipe_class_default);
7907 %}
7908
7909 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7910 match(Set dst (PopCountI (LoadI mem)));
7911 effect(TEMP tmp);
7912 ins_cost(INSN_COST * 13);
7913
7914 format %{ "ldrs $tmp, $mem\n\t"
7915 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7916 "addv $tmp, $tmp\t# vector (8B)\n\t"
7917 "mov $dst, $tmp\t# vector (1D)" %}
7918 ins_encode %{
7919 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7920 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7921 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7922 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7923 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7924 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7925 %}
7926
7927 ins_pipe(pipe_class_default);
7928 %}
7929
7930 // Note: Long.bitCount(long) returns an int.
7931 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7932 match(Set dst (PopCountL src));
7933 effect(TEMP tmp);
7934 ins_cost(INSN_COST * 13);
7935
7936 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7937 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7938 "addv $tmp, $tmp\t# vector (8B)\n\t"
7939 "mov $dst, $tmp\t# vector (1D)" %}
7940 ins_encode %{
7941 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7942 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7943 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7944 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7945 %}
7946
7947 ins_pipe(pipe_class_default);
7948 %}
7949
7950 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7951 match(Set dst (PopCountL (LoadL mem)));
7952 effect(TEMP tmp);
7953 ins_cost(INSN_COST * 13);
7954
7955 format %{ "ldrd $tmp, $mem\n\t"
7956 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7957 "addv $tmp, $tmp\t# vector (8B)\n\t"
7958 "mov $dst, $tmp\t# vector (1D)" %}
7959 ins_encode %{
7960 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7961 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7962 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7963 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7964 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7965 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7966 %}
7967
7968 ins_pipe(pipe_class_default);
7969 %}
7970
7971 // ============================================================================
7972 // VerifyVectorAlignment Instruction
7973
7974 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7975 match(Set addr (VerifyVectorAlignment addr mask));
7976 effect(KILL cr);
7977 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7978 ins_encode %{
7979 Label Lskip;
7980 // check if masked bits of addr are zero
7981 __ tst($addr$$Register, $mask$$constant);
7982 __ br(Assembler::EQ, Lskip);
7983 __ stop("verify_vector_alignment found a misaligned vector memory access");
7984 __ bind(Lskip);
7985 %}
7986 ins_pipe(pipe_slow);
7987 %}
7988
7989 // ============================================================================
7990 // MemBar Instruction
7991
7992 instruct load_fence() %{
7993 match(LoadFence);
7994 ins_cost(VOLATILE_REF_COST);
7995
7996 format %{ "load_fence" %}
7997
7998 ins_encode %{
7999 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8000 %}
8001 ins_pipe(pipe_serial);
8002 %}
8003
8004 instruct unnecessary_membar_acquire() %{
8005 predicate(unnecessary_acquire(n));
8006 match(MemBarAcquire);
8007 ins_cost(0);
8008
8009 format %{ "membar_acquire (elided)" %}
8010
8011 ins_encode %{
8012 __ block_comment("membar_acquire (elided)");
8013 %}
8014
8015 ins_pipe(pipe_class_empty);
8016 %}
8017
8018 instruct membar_acquire() %{
8019 match(MemBarAcquire);
8020 ins_cost(VOLATILE_REF_COST);
8021
8022 format %{ "membar_acquire\n\t"
8023 "dmb ishld" %}
8024
8025 ins_encode %{
8026 __ block_comment("membar_acquire");
8027 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
8028 %}
8029
8030 ins_pipe(pipe_serial);
8031 %}
8032
8033
8034 instruct membar_acquire_lock() %{
8035 match(MemBarAcquireLock);
8036 ins_cost(VOLATILE_REF_COST);
8037
8038 format %{ "membar_acquire_lock (elided)" %}
8039
8040 ins_encode %{
8041 __ block_comment("membar_acquire_lock (elided)");
8042 %}
8043
8044 ins_pipe(pipe_serial);
8045 %}
8046
8047 instruct store_fence() %{
8048 match(StoreFence);
8049 ins_cost(VOLATILE_REF_COST);
8050
8051 format %{ "store_fence" %}
8052
8053 ins_encode %{
8054 __ membar(Assembler::LoadStore|Assembler::StoreStore);
8055 %}
8056 ins_pipe(pipe_serial);
8057 %}
8058
8059 instruct unnecessary_membar_release() %{
8060 predicate(unnecessary_release(n));
8061 match(MemBarRelease);
8062 ins_cost(0);
8063
8064 format %{ "membar_release (elided)" %}
8065
8066 ins_encode %{
8067 __ block_comment("membar_release (elided)");
8068 %}
8069 ins_pipe(pipe_serial);
8070 %}
8071
8072 instruct membar_release() %{
8073 match(MemBarRelease);
8074 ins_cost(VOLATILE_REF_COST);
8075
8076 format %{ "membar_release\n\t"
8077 "dmb ishst\n\tdmb ishld" %}
8078
8079 ins_encode %{
8080 __ block_comment("membar_release");
8081 // These will be merged if AlwaysMergeDMB is enabled.
8082 __ membar(Assembler::StoreStore);
8083 __ membar(Assembler::LoadStore);
8084 %}
8085 ins_pipe(pipe_serial);
8086 %}
8087
8088 instruct membar_storestore() %{
8089 match(MemBarStoreStore);
8090 match(StoreStoreFence);
8091 ins_cost(VOLATILE_REF_COST);
8092
8093 format %{ "MEMBAR-store-store" %}
8094
8095 ins_encode %{
8096 __ membar(Assembler::StoreStore);
8097 %}
8098 ins_pipe(pipe_serial);
8099 %}
8100
8101 instruct membar_release_lock() %{
8102 match(MemBarReleaseLock);
8103 ins_cost(VOLATILE_REF_COST);
8104
8105 format %{ "membar_release_lock (elided)" %}
8106
8107 ins_encode %{
8108 __ block_comment("membar_release_lock (elided)");
8109 %}
8110
8111 ins_pipe(pipe_serial);
8112 %}
8113
8114 instruct unnecessary_membar_volatile() %{
8115 predicate(unnecessary_volatile(n));
8116 match(MemBarVolatile);
8117 ins_cost(0);
8118
8119 format %{ "membar_volatile (elided)" %}
8120
8121 ins_encode %{
8122 __ block_comment("membar_volatile (elided)");
8123 %}
8124
8125 ins_pipe(pipe_serial);
8126 %}
8127
8128 instruct membar_volatile() %{
8129 match(MemBarVolatile);
8130 ins_cost(VOLATILE_REF_COST*100);
8131
8132 format %{ "membar_volatile\n\t"
8133 "dmb ish"%}
8134
8135 ins_encode %{
8136 __ block_comment("membar_volatile");
8137 __ membar(Assembler::StoreLoad);
8138 %}
8139
8140 ins_pipe(pipe_serial);
8141 %}
8142
8143 // ============================================================================
8144 // Cast/Convert Instructions
8145
8146 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8147 match(Set dst (CastX2P src));
8148
8149 ins_cost(INSN_COST);
8150 format %{ "mov $dst, $src\t# long -> ptr" %}
8151
8152 ins_encode %{
8153 if ($dst$$reg != $src$$reg) {
8154 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8155 }
8156 %}
8157
8158 ins_pipe(ialu_reg);
8159 %}
8160
8161 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8162 match(Set dst (CastP2X src));
8163
8164 ins_cost(INSN_COST);
8165 format %{ "mov $dst, $src\t# ptr -> long" %}
8166
8167 ins_encode %{
8168 if ($dst$$reg != $src$$reg) {
8169 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8170 }
8171 %}
8172
8173 ins_pipe(ialu_reg);
8174 %}
8175
8176 // Convert oop into int for vectors alignment masking
8177 instruct convP2I(iRegINoSp dst, iRegP src) %{
8178 match(Set dst (ConvL2I (CastP2X src)));
8179
8180 ins_cost(INSN_COST);
8181 format %{ "movw $dst, $src\t# ptr -> int" %}
8182 ins_encode %{
8183 __ movw($dst$$Register, $src$$Register);
8184 %}
8185
8186 ins_pipe(ialu_reg);
8187 %}
8188
8189 // Convert compressed oop into int for vectors alignment masking
8190 // in case of 32bit oops (heap < 4Gb).
8191 instruct convN2I(iRegINoSp dst, iRegN src)
8192 %{
8193 predicate(CompressedOops::shift() == 0);
8194 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
8195
8196 ins_cost(INSN_COST);
8197 format %{ "mov dst, $src\t# compressed ptr -> int" %}
8198 ins_encode %{
8199 __ movw($dst$$Register, $src$$Register);
8200 %}
8201
8202 ins_pipe(ialu_reg);
8203 %}
8204
8205
8206 // Convert oop pointer into compressed form
8207 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8208 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
8209 match(Set dst (EncodeP src));
8210 effect(KILL cr);
8211 ins_cost(INSN_COST * 3);
8212 format %{ "encode_heap_oop $dst, $src" %}
8213 ins_encode %{
8214 Register s = $src$$Register;
8215 Register d = $dst$$Register;
8216 __ encode_heap_oop(d, s);
8217 %}
8218 ins_pipe(ialu_reg);
8219 %}
8220
8221 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
8222 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
8223 match(Set dst (EncodeP src));
8224 ins_cost(INSN_COST * 3);
8225 format %{ "encode_heap_oop_not_null $dst, $src" %}
8226 ins_encode %{
8227 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8228 %}
8229 ins_pipe(ialu_reg);
8230 %}
8231
8232 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8233 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8234 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8235 match(Set dst (DecodeN src));
8236 ins_cost(INSN_COST * 3);
8237 format %{ "decode_heap_oop $dst, $src" %}
8238 ins_encode %{
8239 Register s = $src$$Register;
8240 Register d = $dst$$Register;
8241 __ decode_heap_oop(d, s);
8242 %}
8243 ins_pipe(ialu_reg);
8244 %}
8245
8246 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8247 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8248 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8249 match(Set dst (DecodeN src));
8250 ins_cost(INSN_COST * 3);
8251 format %{ "decode_heap_oop_not_null $dst, $src" %}
8252 ins_encode %{
8253 Register s = $src$$Register;
8254 Register d = $dst$$Register;
8255 __ decode_heap_oop_not_null(d, s);
8256 %}
8257 ins_pipe(ialu_reg);
8258 %}
8259
8260 // n.b. AArch64 implementations of encode_klass_not_null and
8261 // decode_klass_not_null do not modify the flags register so, unlike
8262 // Intel, we don't kill CR as a side effect here
8263
8264 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8265 match(Set dst (EncodePKlass src));
8266
8267 ins_cost(INSN_COST * 3);
8268 format %{ "encode_klass_not_null $dst,$src" %}
8269
8270 ins_encode %{
8271 Register src_reg = as_Register($src$$reg);
8272 Register dst_reg = as_Register($dst$$reg);
8273 __ encode_klass_not_null(dst_reg, src_reg);
8274 %}
8275
8276 ins_pipe(ialu_reg);
8277 %}
8278
8279 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8280 match(Set dst (DecodeNKlass src));
8281
8282 ins_cost(INSN_COST * 3);
8283 format %{ "decode_klass_not_null $dst,$src" %}
8284
8285 ins_encode %{
8286 Register src_reg = as_Register($src$$reg);
8287 Register dst_reg = as_Register($dst$$reg);
8288 if (dst_reg != src_reg) {
8289 __ decode_klass_not_null(dst_reg, src_reg);
8290 } else {
8291 __ decode_klass_not_null(dst_reg);
8292 }
8293 %}
8294
8295 ins_pipe(ialu_reg);
8296 %}
8297
8298 instruct checkCastPP(iRegPNoSp dst)
8299 %{
8300 match(Set dst (CheckCastPP dst));
8301
8302 size(0);
8303 format %{ "# checkcastPP of $dst" %}
8304 ins_encode(/* empty encoding */);
8305 ins_pipe(pipe_class_empty);
8306 %}
8307
8308 instruct castPP(iRegPNoSp dst)
8309 %{
8310 match(Set dst (CastPP dst));
8311
8312 size(0);
8313 format %{ "# castPP of $dst" %}
8314 ins_encode(/* empty encoding */);
8315 ins_pipe(pipe_class_empty);
8316 %}
8317
8318 instruct castII(iRegI dst)
8319 %{
8320 predicate(VerifyConstraintCasts == 0);
8321 match(Set dst (CastII dst));
8322
8323 size(0);
8324 format %{ "# castII of $dst" %}
8325 ins_encode(/* empty encoding */);
8326 ins_cost(0);
8327 ins_pipe(pipe_class_empty);
8328 %}
8329
8330 instruct castII_checked(iRegI dst, rFlagsReg cr)
8331 %{
8332 predicate(VerifyConstraintCasts > 0);
8333 match(Set dst (CastII dst));
8334 effect(KILL cr);
8335
8336 format %{ "# castII_checked of $dst" %}
8337 ins_encode %{
8338 __ verify_int_in_range(_idx, bottom_type()->is_int(), $dst$$Register, rscratch1);
8339 %}
8340 ins_pipe(pipe_slow);
8341 %}
8342
8343 instruct castLL(iRegL dst)
8344 %{
8345 predicate(VerifyConstraintCasts == 0);
8346 match(Set dst (CastLL dst));
8347
8348 size(0);
8349 format %{ "# castLL of $dst" %}
8350 ins_encode(/* empty encoding */);
8351 ins_cost(0);
8352 ins_pipe(pipe_class_empty);
8353 %}
8354
8355 instruct castLL_checked(iRegL dst, rFlagsReg cr)
8356 %{
8357 predicate(VerifyConstraintCasts > 0);
8358 match(Set dst (CastLL dst));
8359 effect(KILL cr);
8360
8361 format %{ "# castLL_checked of $dst" %}
8362 ins_encode %{
8363 __ verify_long_in_range(_idx, bottom_type()->is_long(), $dst$$Register, rscratch1);
8364 %}
8365 ins_pipe(pipe_slow);
8366 %}
8367
8368 instruct castHH(vRegF dst)
8369 %{
8370 match(Set dst (CastHH dst));
8371 size(0);
8372 format %{ "# castHH of $dst" %}
8373 ins_encode(/* empty encoding */);
8374 ins_cost(0);
8375 ins_pipe(pipe_class_empty);
8376 %}
8377
8378 instruct castFF(vRegF dst)
8379 %{
8380 match(Set dst (CastFF dst));
8381
8382 size(0);
8383 format %{ "# castFF of $dst" %}
8384 ins_encode(/* empty encoding */);
8385 ins_cost(0);
8386 ins_pipe(pipe_class_empty);
8387 %}
8388
8389 instruct castDD(vRegD dst)
8390 %{
8391 match(Set dst (CastDD dst));
8392
8393 size(0);
8394 format %{ "# castDD of $dst" %}
8395 ins_encode(/* empty encoding */);
8396 ins_cost(0);
8397 ins_pipe(pipe_class_empty);
8398 %}
8399
8400 instruct castVV(vReg dst)
8401 %{
8402 match(Set dst (CastVV dst));
8403
8404 size(0);
8405 format %{ "# castVV of $dst" %}
8406 ins_encode(/* empty encoding */);
8407 ins_cost(0);
8408 ins_pipe(pipe_class_empty);
8409 %}
8410
8411 instruct castVVMask(pRegGov dst)
8412 %{
8413 match(Set dst (CastVV dst));
8414
8415 size(0);
8416 format %{ "# castVV of $dst" %}
8417 ins_encode(/* empty encoding */);
8418 ins_cost(0);
8419 ins_pipe(pipe_class_empty);
8420 %}
8421
8422 // ============================================================================
8423 // Atomic operation instructions
8424 //
8425
8426 // standard CompareAndSwapX when we are using barriers
8427 // these have higher priority than the rules selected by a predicate
8428
8429 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8430 // can't match them
8431
8432 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8433
8434 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8435 ins_cost(2 * VOLATILE_REF_COST);
8436
8437 effect(KILL cr);
8438
8439 format %{
8440 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8441 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8442 %}
8443
8444 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8445 aarch64_enc_cset_eq(res));
8446
8447 ins_pipe(pipe_slow);
8448 %}
8449
8450 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8451
8452 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8453 ins_cost(2 * VOLATILE_REF_COST);
8454
8455 effect(KILL cr);
8456
8457 format %{
8458 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8459 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8460 %}
8461
8462 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8463 aarch64_enc_cset_eq(res));
8464
8465 ins_pipe(pipe_slow);
8466 %}
8467
8468 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8469
8470 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8471 ins_cost(2 * VOLATILE_REF_COST);
8472
8473 effect(KILL cr);
8474
8475 format %{
8476 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8477 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8478 %}
8479
8480 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8481 aarch64_enc_cset_eq(res));
8482
8483 ins_pipe(pipe_slow);
8484 %}
8485
8486 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8487
8488 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8489 ins_cost(2 * VOLATILE_REF_COST);
8490
8491 effect(KILL cr);
8492
8493 format %{
8494 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8495 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8496 %}
8497
8498 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8499 aarch64_enc_cset_eq(res));
8500
8501 ins_pipe(pipe_slow);
8502 %}
8503
8504 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8505
8506 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8507 predicate(n->as_LoadStore()->barrier_data() == 0);
8508 ins_cost(2 * VOLATILE_REF_COST);
8509
8510 effect(KILL cr);
8511
8512 format %{
8513 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8514 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8515 %}
8516
8517 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8518 aarch64_enc_cset_eq(res));
8519
8520 ins_pipe(pipe_slow);
8521 %}
8522
8523 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8524
8525 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8526 predicate(n->as_LoadStore()->barrier_data() == 0);
8527 ins_cost(2 * VOLATILE_REF_COST);
8528
8529 effect(KILL cr);
8530
8531 format %{
8532 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8533 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8534 %}
8535
8536 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8537 aarch64_enc_cset_eq(res));
8538
8539 ins_pipe(pipe_slow);
8540 %}
8541
8542 // alternative CompareAndSwapX when we are eliding barriers
8543
8544 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8545
8546 predicate(needs_acquiring_load_exclusive(n));
8547 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8548 ins_cost(VOLATILE_REF_COST);
8549
8550 effect(KILL cr);
8551
8552 format %{
8553 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8554 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8555 %}
8556
8557 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8558 aarch64_enc_cset_eq(res));
8559
8560 ins_pipe(pipe_slow);
8561 %}
8562
8563 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8564
8565 predicate(needs_acquiring_load_exclusive(n));
8566 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8567 ins_cost(VOLATILE_REF_COST);
8568
8569 effect(KILL cr);
8570
8571 format %{
8572 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8573 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8574 %}
8575
8576 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8577 aarch64_enc_cset_eq(res));
8578
8579 ins_pipe(pipe_slow);
8580 %}
8581
8582 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8583
8584 predicate(needs_acquiring_load_exclusive(n));
8585 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8586 ins_cost(VOLATILE_REF_COST);
8587
8588 effect(KILL cr);
8589
8590 format %{
8591 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8592 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8593 %}
8594
8595 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8596 aarch64_enc_cset_eq(res));
8597
8598 ins_pipe(pipe_slow);
8599 %}
8600
8601 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8602
8603 predicate(needs_acquiring_load_exclusive(n));
8604 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8605 ins_cost(VOLATILE_REF_COST);
8606
8607 effect(KILL cr);
8608
8609 format %{
8610 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8611 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8612 %}
8613
8614 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8615 aarch64_enc_cset_eq(res));
8616
8617 ins_pipe(pipe_slow);
8618 %}
8619
8620 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8621
8622 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8623 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8624 ins_cost(VOLATILE_REF_COST);
8625
8626 effect(KILL cr);
8627
8628 format %{
8629 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8630 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8631 %}
8632
8633 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8634 aarch64_enc_cset_eq(res));
8635
8636 ins_pipe(pipe_slow);
8637 %}
8638
8639 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8640
8641 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8642 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8643 ins_cost(VOLATILE_REF_COST);
8644
8645 effect(KILL cr);
8646
8647 format %{
8648 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8649 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8650 %}
8651
8652 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8653 aarch64_enc_cset_eq(res));
8654
8655 ins_pipe(pipe_slow);
8656 %}
8657
8658
8659 // ---------------------------------------------------------------------
8660
8661 // BEGIN This section of the file is automatically generated. Do not edit --------------
8662
8663 // Sundry CAS operations. Note that release is always true,
8664 // regardless of the memory ordering of the CAS. This is because we
8665 // need the volatile case to be sequentially consistent but there is
8666 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8667 // can't check the type of memory ordering here, so we always emit a
8668 // STLXR.
8669
8670 // This section is generated from cas.m4
8671
8672
8673 // This pattern is generated automatically from cas.m4.
8674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8675 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8676 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8677 ins_cost(2 * VOLATILE_REF_COST);
8678 effect(TEMP_DEF res, KILL cr);
8679 format %{
8680 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8681 %}
8682 ins_encode %{
8683 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8684 Assembler::byte, /*acquire*/ false, /*release*/ true,
8685 /*weak*/ false, $res$$Register);
8686 __ sxtbw($res$$Register, $res$$Register);
8687 %}
8688 ins_pipe(pipe_slow);
8689 %}
8690
8691 // This pattern is generated automatically from cas.m4.
8692 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8693 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8694 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8695 ins_cost(2 * VOLATILE_REF_COST);
8696 effect(TEMP_DEF res, KILL cr);
8697 format %{
8698 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8699 %}
8700 ins_encode %{
8701 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8702 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8703 /*weak*/ false, $res$$Register);
8704 __ sxthw($res$$Register, $res$$Register);
8705 %}
8706 ins_pipe(pipe_slow);
8707 %}
8708
8709 // This pattern is generated automatically from cas.m4.
8710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8711 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8712 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8713 ins_cost(2 * VOLATILE_REF_COST);
8714 effect(TEMP_DEF res, KILL cr);
8715 format %{
8716 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8717 %}
8718 ins_encode %{
8719 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8720 Assembler::word, /*acquire*/ false, /*release*/ true,
8721 /*weak*/ false, $res$$Register);
8722 %}
8723 ins_pipe(pipe_slow);
8724 %}
8725
8726 // This pattern is generated automatically from cas.m4.
8727 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8728 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8729 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8730 ins_cost(2 * VOLATILE_REF_COST);
8731 effect(TEMP_DEF res, KILL cr);
8732 format %{
8733 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8734 %}
8735 ins_encode %{
8736 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8737 Assembler::xword, /*acquire*/ false, /*release*/ true,
8738 /*weak*/ false, $res$$Register);
8739 %}
8740 ins_pipe(pipe_slow);
8741 %}
8742
8743 // This pattern is generated automatically from cas.m4.
8744 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8745 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8746 predicate(n->as_LoadStore()->barrier_data() == 0);
8747 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8748 ins_cost(2 * VOLATILE_REF_COST);
8749 effect(TEMP_DEF res, KILL cr);
8750 format %{
8751 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8752 %}
8753 ins_encode %{
8754 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8755 Assembler::word, /*acquire*/ false, /*release*/ true,
8756 /*weak*/ false, $res$$Register);
8757 %}
8758 ins_pipe(pipe_slow);
8759 %}
8760
8761 // This pattern is generated automatically from cas.m4.
8762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8763 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8764 predicate(n->as_LoadStore()->barrier_data() == 0);
8765 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8766 ins_cost(2 * VOLATILE_REF_COST);
8767 effect(TEMP_DEF res, KILL cr);
8768 format %{
8769 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8770 %}
8771 ins_encode %{
8772 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8773 Assembler::xword, /*acquire*/ false, /*release*/ true,
8774 /*weak*/ false, $res$$Register);
8775 %}
8776 ins_pipe(pipe_slow);
8777 %}
8778
8779 // This pattern is generated automatically from cas.m4.
8780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8781 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8782 predicate(needs_acquiring_load_exclusive(n));
8783 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8784 ins_cost(VOLATILE_REF_COST);
8785 effect(TEMP_DEF res, KILL cr);
8786 format %{
8787 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8788 %}
8789 ins_encode %{
8790 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8791 Assembler::byte, /*acquire*/ true, /*release*/ true,
8792 /*weak*/ false, $res$$Register);
8793 __ sxtbw($res$$Register, $res$$Register);
8794 %}
8795 ins_pipe(pipe_slow);
8796 %}
8797
8798 // This pattern is generated automatically from cas.m4.
8799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8800 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8801 predicate(needs_acquiring_load_exclusive(n));
8802 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8803 ins_cost(VOLATILE_REF_COST);
8804 effect(TEMP_DEF res, KILL cr);
8805 format %{
8806 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8807 %}
8808 ins_encode %{
8809 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8810 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8811 /*weak*/ false, $res$$Register);
8812 __ sxthw($res$$Register, $res$$Register);
8813 %}
8814 ins_pipe(pipe_slow);
8815 %}
8816
8817 // This pattern is generated automatically from cas.m4.
8818 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8819 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8820 predicate(needs_acquiring_load_exclusive(n));
8821 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8822 ins_cost(VOLATILE_REF_COST);
8823 effect(TEMP_DEF res, KILL cr);
8824 format %{
8825 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8826 %}
8827 ins_encode %{
8828 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8829 Assembler::word, /*acquire*/ true, /*release*/ true,
8830 /*weak*/ false, $res$$Register);
8831 %}
8832 ins_pipe(pipe_slow);
8833 %}
8834
8835 // This pattern is generated automatically from cas.m4.
8836 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8837 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8838 predicate(needs_acquiring_load_exclusive(n));
8839 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8840 ins_cost(VOLATILE_REF_COST);
8841 effect(TEMP_DEF res, KILL cr);
8842 format %{
8843 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8844 %}
8845 ins_encode %{
8846 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8847 Assembler::xword, /*acquire*/ true, /*release*/ true,
8848 /*weak*/ false, $res$$Register);
8849 %}
8850 ins_pipe(pipe_slow);
8851 %}
8852
8853 // This pattern is generated automatically from cas.m4.
8854 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8855 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8856 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8857 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8858 ins_cost(VOLATILE_REF_COST);
8859 effect(TEMP_DEF res, KILL cr);
8860 format %{
8861 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8862 %}
8863 ins_encode %{
8864 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8865 Assembler::word, /*acquire*/ true, /*release*/ true,
8866 /*weak*/ false, $res$$Register);
8867 %}
8868 ins_pipe(pipe_slow);
8869 %}
8870
8871 // This pattern is generated automatically from cas.m4.
8872 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8873 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8874 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8875 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8876 ins_cost(VOLATILE_REF_COST);
8877 effect(TEMP_DEF res, KILL cr);
8878 format %{
8879 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8880 %}
8881 ins_encode %{
8882 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8883 Assembler::xword, /*acquire*/ true, /*release*/ true,
8884 /*weak*/ false, $res$$Register);
8885 %}
8886 ins_pipe(pipe_slow);
8887 %}
8888
8889 // This pattern is generated automatically from cas.m4.
8890 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8891 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8892 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8893 ins_cost(2 * VOLATILE_REF_COST);
8894 effect(KILL cr);
8895 format %{
8896 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8897 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8898 %}
8899 ins_encode %{
8900 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8901 Assembler::byte, /*acquire*/ false, /*release*/ true,
8902 /*weak*/ true, noreg);
8903 __ csetw($res$$Register, Assembler::EQ);
8904 %}
8905 ins_pipe(pipe_slow);
8906 %}
8907
8908 // This pattern is generated automatically from cas.m4.
8909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8910 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8911 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8912 ins_cost(2 * VOLATILE_REF_COST);
8913 effect(KILL cr);
8914 format %{
8915 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8916 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8917 %}
8918 ins_encode %{
8919 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8920 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8921 /*weak*/ true, noreg);
8922 __ csetw($res$$Register, Assembler::EQ);
8923 %}
8924 ins_pipe(pipe_slow);
8925 %}
8926
8927 // This pattern is generated automatically from cas.m4.
8928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8929 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8930 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8931 ins_cost(2 * VOLATILE_REF_COST);
8932 effect(KILL cr);
8933 format %{
8934 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8935 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8936 %}
8937 ins_encode %{
8938 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8939 Assembler::word, /*acquire*/ false, /*release*/ true,
8940 /*weak*/ true, noreg);
8941 __ csetw($res$$Register, Assembler::EQ);
8942 %}
8943 ins_pipe(pipe_slow);
8944 %}
8945
8946 // This pattern is generated automatically from cas.m4.
8947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8948 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8949 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8950 ins_cost(2 * VOLATILE_REF_COST);
8951 effect(KILL cr);
8952 format %{
8953 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8954 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8955 %}
8956 ins_encode %{
8957 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8958 Assembler::xword, /*acquire*/ false, /*release*/ true,
8959 /*weak*/ true, noreg);
8960 __ csetw($res$$Register, Assembler::EQ);
8961 %}
8962 ins_pipe(pipe_slow);
8963 %}
8964
8965 // This pattern is generated automatically from cas.m4.
8966 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8967 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8968 predicate(n->as_LoadStore()->barrier_data() == 0);
8969 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8970 ins_cost(2 * VOLATILE_REF_COST);
8971 effect(KILL cr);
8972 format %{
8973 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8974 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8975 %}
8976 ins_encode %{
8977 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8978 Assembler::word, /*acquire*/ false, /*release*/ true,
8979 /*weak*/ true, noreg);
8980 __ csetw($res$$Register, Assembler::EQ);
8981 %}
8982 ins_pipe(pipe_slow);
8983 %}
8984
8985 // This pattern is generated automatically from cas.m4.
8986 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8987 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8988 predicate(n->as_LoadStore()->barrier_data() == 0);
8989 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8990 ins_cost(2 * VOLATILE_REF_COST);
8991 effect(KILL cr);
8992 format %{
8993 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8994 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8995 %}
8996 ins_encode %{
8997 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8998 Assembler::xword, /*acquire*/ false, /*release*/ true,
8999 /*weak*/ true, noreg);
9000 __ csetw($res$$Register, Assembler::EQ);
9001 %}
9002 ins_pipe(pipe_slow);
9003 %}
9004
9005 // This pattern is generated automatically from cas.m4.
9006 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9007 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9008 predicate(needs_acquiring_load_exclusive(n));
9009 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
9010 ins_cost(VOLATILE_REF_COST);
9011 effect(KILL cr);
9012 format %{
9013 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
9014 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9015 %}
9016 ins_encode %{
9017 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9018 Assembler::byte, /*acquire*/ true, /*release*/ true,
9019 /*weak*/ true, noreg);
9020 __ csetw($res$$Register, Assembler::EQ);
9021 %}
9022 ins_pipe(pipe_slow);
9023 %}
9024
9025 // This pattern is generated automatically from cas.m4.
9026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9027 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9028 predicate(needs_acquiring_load_exclusive(n));
9029 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
9030 ins_cost(VOLATILE_REF_COST);
9031 effect(KILL cr);
9032 format %{
9033 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
9034 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9035 %}
9036 ins_encode %{
9037 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9038 Assembler::halfword, /*acquire*/ true, /*release*/ true,
9039 /*weak*/ true, noreg);
9040 __ csetw($res$$Register, Assembler::EQ);
9041 %}
9042 ins_pipe(pipe_slow);
9043 %}
9044
9045 // This pattern is generated automatically from cas.m4.
9046 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9047 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
9048 predicate(needs_acquiring_load_exclusive(n));
9049 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
9050 ins_cost(VOLATILE_REF_COST);
9051 effect(KILL cr);
9052 format %{
9053 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
9054 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9055 %}
9056 ins_encode %{
9057 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9058 Assembler::word, /*acquire*/ true, /*release*/ true,
9059 /*weak*/ true, noreg);
9060 __ csetw($res$$Register, Assembler::EQ);
9061 %}
9062 ins_pipe(pipe_slow);
9063 %}
9064
9065 // This pattern is generated automatically from cas.m4.
9066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9067 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
9068 predicate(needs_acquiring_load_exclusive(n));
9069 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
9070 ins_cost(VOLATILE_REF_COST);
9071 effect(KILL cr);
9072 format %{
9073 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
9074 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9075 %}
9076 ins_encode %{
9077 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9078 Assembler::xword, /*acquire*/ true, /*release*/ true,
9079 /*weak*/ true, noreg);
9080 __ csetw($res$$Register, Assembler::EQ);
9081 %}
9082 ins_pipe(pipe_slow);
9083 %}
9084
9085 // This pattern is generated automatically from cas.m4.
9086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9087 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
9088 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9089 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
9090 ins_cost(VOLATILE_REF_COST);
9091 effect(KILL cr);
9092 format %{
9093 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
9094 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9095 %}
9096 ins_encode %{
9097 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9098 Assembler::word, /*acquire*/ true, /*release*/ true,
9099 /*weak*/ true, noreg);
9100 __ csetw($res$$Register, Assembler::EQ);
9101 %}
9102 ins_pipe(pipe_slow);
9103 %}
9104
9105 // This pattern is generated automatically from cas.m4.
9106 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
9107 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
9108 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9109 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
9110 ins_cost(VOLATILE_REF_COST);
9111 effect(KILL cr);
9112 format %{
9113 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
9114 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
9115 %}
9116 ins_encode %{
9117 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
9118 Assembler::xword, /*acquire*/ true, /*release*/ true,
9119 /*weak*/ true, noreg);
9120 __ csetw($res$$Register, Assembler::EQ);
9121 %}
9122 ins_pipe(pipe_slow);
9123 %}
9124
9125 // END This section of the file is automatically generated. Do not edit --------------
9126 // ---------------------------------------------------------------------
9127
9128 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
9129 match(Set prev (GetAndSetI mem newv));
9130 ins_cost(2 * VOLATILE_REF_COST);
9131 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9132 ins_encode %{
9133 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9134 %}
9135 ins_pipe(pipe_serial);
9136 %}
9137
9138 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
9139 match(Set prev (GetAndSetL mem newv));
9140 ins_cost(2 * VOLATILE_REF_COST);
9141 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9142 ins_encode %{
9143 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9144 %}
9145 ins_pipe(pipe_serial);
9146 %}
9147
9148 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
9149 predicate(n->as_LoadStore()->barrier_data() == 0);
9150 match(Set prev (GetAndSetN mem newv));
9151 ins_cost(2 * VOLATILE_REF_COST);
9152 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
9153 ins_encode %{
9154 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9155 %}
9156 ins_pipe(pipe_serial);
9157 %}
9158
9159 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
9160 predicate(n->as_LoadStore()->barrier_data() == 0);
9161 match(Set prev (GetAndSetP mem newv));
9162 ins_cost(2 * VOLATILE_REF_COST);
9163 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
9164 ins_encode %{
9165 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
9166 %}
9167 ins_pipe(pipe_serial);
9168 %}
9169
9170 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
9171 predicate(needs_acquiring_load_exclusive(n));
9172 match(Set prev (GetAndSetI mem newv));
9173 ins_cost(VOLATILE_REF_COST);
9174 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9175 ins_encode %{
9176 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9177 %}
9178 ins_pipe(pipe_serial);
9179 %}
9180
9181 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
9182 predicate(needs_acquiring_load_exclusive(n));
9183 match(Set prev (GetAndSetL mem newv));
9184 ins_cost(VOLATILE_REF_COST);
9185 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9186 ins_encode %{
9187 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9188 %}
9189 ins_pipe(pipe_serial);
9190 %}
9191
9192 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
9193 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
9194 match(Set prev (GetAndSetN mem newv));
9195 ins_cost(VOLATILE_REF_COST);
9196 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
9197 ins_encode %{
9198 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
9199 %}
9200 ins_pipe(pipe_serial);
9201 %}
9202
9203 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
9204 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
9205 match(Set prev (GetAndSetP mem newv));
9206 ins_cost(VOLATILE_REF_COST);
9207 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
9208 ins_encode %{
9209 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
9210 %}
9211 ins_pipe(pipe_serial);
9212 %}
9213
9214
9215 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
9216 match(Set newval (GetAndAddL mem incr));
9217 ins_cost(2 * VOLATILE_REF_COST + 1);
9218 format %{ "get_and_addL $newval, [$mem], $incr" %}
9219 ins_encode %{
9220 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
9221 %}
9222 ins_pipe(pipe_serial);
9223 %}
9224
9225 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
9226 predicate(n->as_LoadStore()->result_not_used());
9227 match(Set dummy (GetAndAddL mem incr));
9228 ins_cost(2 * VOLATILE_REF_COST);
9229 format %{ "get_and_addL [$mem], $incr" %}
9230 ins_encode %{
9231 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
9232 %}
9233 ins_pipe(pipe_serial);
9234 %}
9235
9236 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9237 match(Set newval (GetAndAddL mem incr));
9238 ins_cost(2 * VOLATILE_REF_COST + 1);
9239 format %{ "get_and_addL $newval, [$mem], $incr" %}
9240 ins_encode %{
9241 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
9242 %}
9243 ins_pipe(pipe_serial);
9244 %}
9245
9246 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
9247 predicate(n->as_LoadStore()->result_not_used());
9248 match(Set dummy (GetAndAddL mem incr));
9249 ins_cost(2 * VOLATILE_REF_COST);
9250 format %{ "get_and_addL [$mem], $incr" %}
9251 ins_encode %{
9252 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
9253 %}
9254 ins_pipe(pipe_serial);
9255 %}
9256
9257 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9258 match(Set newval (GetAndAddI mem incr));
9259 ins_cost(2 * VOLATILE_REF_COST + 1);
9260 format %{ "get_and_addI $newval, [$mem], $incr" %}
9261 ins_encode %{
9262 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9263 %}
9264 ins_pipe(pipe_serial);
9265 %}
9266
9267 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9268 predicate(n->as_LoadStore()->result_not_used());
9269 match(Set dummy (GetAndAddI mem incr));
9270 ins_cost(2 * VOLATILE_REF_COST);
9271 format %{ "get_and_addI [$mem], $incr" %}
9272 ins_encode %{
9273 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9274 %}
9275 ins_pipe(pipe_serial);
9276 %}
9277
9278 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9279 match(Set newval (GetAndAddI mem incr));
9280 ins_cost(2 * VOLATILE_REF_COST + 1);
9281 format %{ "get_and_addI $newval, [$mem], $incr" %}
9282 ins_encode %{
9283 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9284 %}
9285 ins_pipe(pipe_serial);
9286 %}
9287
9288 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9289 predicate(n->as_LoadStore()->result_not_used());
9290 match(Set dummy (GetAndAddI mem incr));
9291 ins_cost(2 * VOLATILE_REF_COST);
9292 format %{ "get_and_addI [$mem], $incr" %}
9293 ins_encode %{
9294 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9295 %}
9296 ins_pipe(pipe_serial);
9297 %}
9298
9299 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9300 predicate(needs_acquiring_load_exclusive(n));
9301 match(Set newval (GetAndAddL mem incr));
9302 ins_cost(VOLATILE_REF_COST + 1);
9303 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9304 ins_encode %{
9305 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9306 %}
9307 ins_pipe(pipe_serial);
9308 %}
9309
9310 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9311 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9312 match(Set dummy (GetAndAddL mem incr));
9313 ins_cost(VOLATILE_REF_COST);
9314 format %{ "get_and_addL_acq [$mem], $incr" %}
9315 ins_encode %{
9316 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9317 %}
9318 ins_pipe(pipe_serial);
9319 %}
9320
9321 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9322 predicate(needs_acquiring_load_exclusive(n));
9323 match(Set newval (GetAndAddL mem incr));
9324 ins_cost(VOLATILE_REF_COST + 1);
9325 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9326 ins_encode %{
9327 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9328 %}
9329 ins_pipe(pipe_serial);
9330 %}
9331
9332 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9333 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9334 match(Set dummy (GetAndAddL mem incr));
9335 ins_cost(VOLATILE_REF_COST);
9336 format %{ "get_and_addL_acq [$mem], $incr" %}
9337 ins_encode %{
9338 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9339 %}
9340 ins_pipe(pipe_serial);
9341 %}
9342
9343 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9344 predicate(needs_acquiring_load_exclusive(n));
9345 match(Set newval (GetAndAddI mem incr));
9346 ins_cost(VOLATILE_REF_COST + 1);
9347 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9348 ins_encode %{
9349 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9350 %}
9351 ins_pipe(pipe_serial);
9352 %}
9353
9354 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9355 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9356 match(Set dummy (GetAndAddI mem incr));
9357 ins_cost(VOLATILE_REF_COST);
9358 format %{ "get_and_addI_acq [$mem], $incr" %}
9359 ins_encode %{
9360 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9361 %}
9362 ins_pipe(pipe_serial);
9363 %}
9364
9365 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9366 predicate(needs_acquiring_load_exclusive(n));
9367 match(Set newval (GetAndAddI mem incr));
9368 ins_cost(VOLATILE_REF_COST + 1);
9369 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9370 ins_encode %{
9371 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9372 %}
9373 ins_pipe(pipe_serial);
9374 %}
9375
9376 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9377 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9378 match(Set dummy (GetAndAddI mem incr));
9379 ins_cost(VOLATILE_REF_COST);
9380 format %{ "get_and_addI_acq [$mem], $incr" %}
9381 ins_encode %{
9382 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9383 %}
9384 ins_pipe(pipe_serial);
9385 %}
9386
9387 // Manifest a CmpU result in an integer register.
9388 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9389 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9390 %{
9391 match(Set dst (CmpU3 src1 src2));
9392 effect(KILL flags);
9393
9394 ins_cost(INSN_COST * 3);
9395 format %{
9396 "cmpw $src1, $src2\n\t"
9397 "csetw $dst, ne\n\t"
9398 "cnegw $dst, lo\t# CmpU3(reg)"
9399 %}
9400 ins_encode %{
9401 __ cmpw($src1$$Register, $src2$$Register);
9402 __ csetw($dst$$Register, Assembler::NE);
9403 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9404 %}
9405
9406 ins_pipe(pipe_class_default);
9407 %}
9408
9409 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9410 %{
9411 match(Set dst (CmpU3 src1 src2));
9412 effect(KILL flags);
9413
9414 ins_cost(INSN_COST * 3);
9415 format %{
9416 "subsw zr, $src1, $src2\n\t"
9417 "csetw $dst, ne\n\t"
9418 "cnegw $dst, lo\t# CmpU3(imm)"
9419 %}
9420 ins_encode %{
9421 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9422 __ csetw($dst$$Register, Assembler::NE);
9423 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9424 %}
9425
9426 ins_pipe(pipe_class_default);
9427 %}
9428
9429 // Manifest a CmpUL result in an integer register.
9430 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9431 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9432 %{
9433 match(Set dst (CmpUL3 src1 src2));
9434 effect(KILL flags);
9435
9436 ins_cost(INSN_COST * 3);
9437 format %{
9438 "cmp $src1, $src2\n\t"
9439 "csetw $dst, ne\n\t"
9440 "cnegw $dst, lo\t# CmpUL3(reg)"
9441 %}
9442 ins_encode %{
9443 __ cmp($src1$$Register, $src2$$Register);
9444 __ csetw($dst$$Register, Assembler::NE);
9445 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9446 %}
9447
9448 ins_pipe(pipe_class_default);
9449 %}
9450
9451 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9452 %{
9453 match(Set dst (CmpUL3 src1 src2));
9454 effect(KILL flags);
9455
9456 ins_cost(INSN_COST * 3);
9457 format %{
9458 "subs zr, $src1, $src2\n\t"
9459 "csetw $dst, ne\n\t"
9460 "cnegw $dst, lo\t# CmpUL3(imm)"
9461 %}
9462 ins_encode %{
9463 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9464 __ csetw($dst$$Register, Assembler::NE);
9465 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9466 %}
9467
9468 ins_pipe(pipe_class_default);
9469 %}
9470
9471 // Manifest a CmpL result in an integer register.
9472 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9473 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9474 %{
9475 match(Set dst (CmpL3 src1 src2));
9476 effect(KILL flags);
9477
9478 ins_cost(INSN_COST * 3);
9479 format %{
9480 "cmp $src1, $src2\n\t"
9481 "csetw $dst, ne\n\t"
9482 "cnegw $dst, lt\t# CmpL3(reg)"
9483 %}
9484 ins_encode %{
9485 __ cmp($src1$$Register, $src2$$Register);
9486 __ csetw($dst$$Register, Assembler::NE);
9487 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9488 %}
9489
9490 ins_pipe(pipe_class_default);
9491 %}
9492
9493 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9494 %{
9495 match(Set dst (CmpL3 src1 src2));
9496 effect(KILL flags);
9497
9498 ins_cost(INSN_COST * 3);
9499 format %{
9500 "subs zr, $src1, $src2\n\t"
9501 "csetw $dst, ne\n\t"
9502 "cnegw $dst, lt\t# CmpL3(imm)"
9503 %}
9504 ins_encode %{
9505 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9506 __ csetw($dst$$Register, Assembler::NE);
9507 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9508 %}
9509
9510 ins_pipe(pipe_class_default);
9511 %}
9512
9513 // ============================================================================
9514 // Conditional Move Instructions
9515
9516 // n.b. we have identical rules for both a signed compare op (cmpOp)
9517 // and an unsigned compare op (cmpOpU). it would be nice if we could
9518 // define an op class which merged both inputs and use it to type the
9519 // argument to a single rule. unfortunatelyt his fails because the
9520 // opclass does not live up to the COND_INTER interface of its
9521 // component operands. When the generic code tries to negate the
9522 // operand it ends up running the generci Machoper::negate method
9523 // which throws a ShouldNotHappen. So, we have to provide two flavours
9524 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9525
9526 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9527 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9528
9529 ins_cost(INSN_COST * 2);
9530 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9531
9532 ins_encode %{
9533 __ cselw(as_Register($dst$$reg),
9534 as_Register($src2$$reg),
9535 as_Register($src1$$reg),
9536 (Assembler::Condition)$cmp$$cmpcode);
9537 %}
9538
9539 ins_pipe(icond_reg_reg);
9540 %}
9541
9542 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9543 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9544
9545 ins_cost(INSN_COST * 2);
9546 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9547
9548 ins_encode %{
9549 __ cselw(as_Register($dst$$reg),
9550 as_Register($src2$$reg),
9551 as_Register($src1$$reg),
9552 (Assembler::Condition)$cmp$$cmpcode);
9553 %}
9554
9555 ins_pipe(icond_reg_reg);
9556 %}
9557
9558 // special cases where one arg is zero
9559
9560 // n.b. this is selected in preference to the rule above because it
9561 // avoids loading constant 0 into a source register
9562
9563 // TODO
9564 // we ought only to be able to cull one of these variants as the ideal
9565 // transforms ought always to order the zero consistently (to left/right?)
9566
9567 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9568 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9569
9570 ins_cost(INSN_COST * 2);
9571 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9572
9573 ins_encode %{
9574 __ cselw(as_Register($dst$$reg),
9575 as_Register($src$$reg),
9576 zr,
9577 (Assembler::Condition)$cmp$$cmpcode);
9578 %}
9579
9580 ins_pipe(icond_reg);
9581 %}
9582
9583 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9584 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9585
9586 ins_cost(INSN_COST * 2);
9587 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9588
9589 ins_encode %{
9590 __ cselw(as_Register($dst$$reg),
9591 as_Register($src$$reg),
9592 zr,
9593 (Assembler::Condition)$cmp$$cmpcode);
9594 %}
9595
9596 ins_pipe(icond_reg);
9597 %}
9598
9599 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9600 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9601
9602 ins_cost(INSN_COST * 2);
9603 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9604
9605 ins_encode %{
9606 __ cselw(as_Register($dst$$reg),
9607 zr,
9608 as_Register($src$$reg),
9609 (Assembler::Condition)$cmp$$cmpcode);
9610 %}
9611
9612 ins_pipe(icond_reg);
9613 %}
9614
9615 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9616 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9617
9618 ins_cost(INSN_COST * 2);
9619 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9620
9621 ins_encode %{
9622 __ cselw(as_Register($dst$$reg),
9623 zr,
9624 as_Register($src$$reg),
9625 (Assembler::Condition)$cmp$$cmpcode);
9626 %}
9627
9628 ins_pipe(icond_reg);
9629 %}
9630
9631 // special case for creating a boolean 0 or 1
9632
9633 // n.b. this is selected in preference to the rule above because it
9634 // avoids loading constants 0 and 1 into a source register
9635
9636 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9637 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9638
9639 ins_cost(INSN_COST * 2);
9640 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9641
9642 ins_encode %{
9643 // equivalently
9644 // cset(as_Register($dst$$reg),
9645 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9646 __ csincw(as_Register($dst$$reg),
9647 zr,
9648 zr,
9649 (Assembler::Condition)$cmp$$cmpcode);
9650 %}
9651
9652 ins_pipe(icond_none);
9653 %}
9654
9655 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9656 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9657
9658 ins_cost(INSN_COST * 2);
9659 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9660
9661 ins_encode %{
9662 // equivalently
9663 // cset(as_Register($dst$$reg),
9664 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9665 __ csincw(as_Register($dst$$reg),
9666 zr,
9667 zr,
9668 (Assembler::Condition)$cmp$$cmpcode);
9669 %}
9670
9671 ins_pipe(icond_none);
9672 %}
9673
9674 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9675 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9676
9677 ins_cost(INSN_COST * 2);
9678 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9679
9680 ins_encode %{
9681 __ csel(as_Register($dst$$reg),
9682 as_Register($src2$$reg),
9683 as_Register($src1$$reg),
9684 (Assembler::Condition)$cmp$$cmpcode);
9685 %}
9686
9687 ins_pipe(icond_reg_reg);
9688 %}
9689
9690 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9691 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9692
9693 ins_cost(INSN_COST * 2);
9694 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9695
9696 ins_encode %{
9697 __ csel(as_Register($dst$$reg),
9698 as_Register($src2$$reg),
9699 as_Register($src1$$reg),
9700 (Assembler::Condition)$cmp$$cmpcode);
9701 %}
9702
9703 ins_pipe(icond_reg_reg);
9704 %}
9705
9706 // special cases where one arg is zero
9707
9708 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9709 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9710
9711 ins_cost(INSN_COST * 2);
9712 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9713
9714 ins_encode %{
9715 __ csel(as_Register($dst$$reg),
9716 zr,
9717 as_Register($src$$reg),
9718 (Assembler::Condition)$cmp$$cmpcode);
9719 %}
9720
9721 ins_pipe(icond_reg);
9722 %}
9723
9724 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9725 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9726
9727 ins_cost(INSN_COST * 2);
9728 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9729
9730 ins_encode %{
9731 __ csel(as_Register($dst$$reg),
9732 zr,
9733 as_Register($src$$reg),
9734 (Assembler::Condition)$cmp$$cmpcode);
9735 %}
9736
9737 ins_pipe(icond_reg);
9738 %}
9739
9740 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9741 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9742
9743 ins_cost(INSN_COST * 2);
9744 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9745
9746 ins_encode %{
9747 __ csel(as_Register($dst$$reg),
9748 as_Register($src$$reg),
9749 zr,
9750 (Assembler::Condition)$cmp$$cmpcode);
9751 %}
9752
9753 ins_pipe(icond_reg);
9754 %}
9755
9756 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9757 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9758
9759 ins_cost(INSN_COST * 2);
9760 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9761
9762 ins_encode %{
9763 __ csel(as_Register($dst$$reg),
9764 as_Register($src$$reg),
9765 zr,
9766 (Assembler::Condition)$cmp$$cmpcode);
9767 %}
9768
9769 ins_pipe(icond_reg);
9770 %}
9771
9772 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9773 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9774
9775 ins_cost(INSN_COST * 2);
9776 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9777
9778 ins_encode %{
9779 __ csel(as_Register($dst$$reg),
9780 as_Register($src2$$reg),
9781 as_Register($src1$$reg),
9782 (Assembler::Condition)$cmp$$cmpcode);
9783 %}
9784
9785 ins_pipe(icond_reg_reg);
9786 %}
9787
9788 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9789 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9790
9791 ins_cost(INSN_COST * 2);
9792 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9793
9794 ins_encode %{
9795 __ csel(as_Register($dst$$reg),
9796 as_Register($src2$$reg),
9797 as_Register($src1$$reg),
9798 (Assembler::Condition)$cmp$$cmpcode);
9799 %}
9800
9801 ins_pipe(icond_reg_reg);
9802 %}
9803
9804 // special cases where one arg is zero
9805
9806 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9807 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9808
9809 ins_cost(INSN_COST * 2);
9810 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9811
9812 ins_encode %{
9813 __ csel(as_Register($dst$$reg),
9814 zr,
9815 as_Register($src$$reg),
9816 (Assembler::Condition)$cmp$$cmpcode);
9817 %}
9818
9819 ins_pipe(icond_reg);
9820 %}
9821
9822 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9823 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9824
9825 ins_cost(INSN_COST * 2);
9826 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9827
9828 ins_encode %{
9829 __ csel(as_Register($dst$$reg),
9830 zr,
9831 as_Register($src$$reg),
9832 (Assembler::Condition)$cmp$$cmpcode);
9833 %}
9834
9835 ins_pipe(icond_reg);
9836 %}
9837
9838 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9839 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9840
9841 ins_cost(INSN_COST * 2);
9842 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9843
9844 ins_encode %{
9845 __ csel(as_Register($dst$$reg),
9846 as_Register($src$$reg),
9847 zr,
9848 (Assembler::Condition)$cmp$$cmpcode);
9849 %}
9850
9851 ins_pipe(icond_reg);
9852 %}
9853
9854 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9855 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9856
9857 ins_cost(INSN_COST * 2);
9858 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9859
9860 ins_encode %{
9861 __ csel(as_Register($dst$$reg),
9862 as_Register($src$$reg),
9863 zr,
9864 (Assembler::Condition)$cmp$$cmpcode);
9865 %}
9866
9867 ins_pipe(icond_reg);
9868 %}
9869
9870 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9871 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9872
9873 ins_cost(INSN_COST * 2);
9874 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9875
9876 ins_encode %{
9877 __ cselw(as_Register($dst$$reg),
9878 as_Register($src2$$reg),
9879 as_Register($src1$$reg),
9880 (Assembler::Condition)$cmp$$cmpcode);
9881 %}
9882
9883 ins_pipe(icond_reg_reg);
9884 %}
9885
9886 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9887 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9888
9889 ins_cost(INSN_COST * 2);
9890 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9891
9892 ins_encode %{
9893 __ cselw(as_Register($dst$$reg),
9894 as_Register($src2$$reg),
9895 as_Register($src1$$reg),
9896 (Assembler::Condition)$cmp$$cmpcode);
9897 %}
9898
9899 ins_pipe(icond_reg_reg);
9900 %}
9901
9902 // special cases where one arg is zero
9903
9904 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9905 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9906
9907 ins_cost(INSN_COST * 2);
9908 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9909
9910 ins_encode %{
9911 __ cselw(as_Register($dst$$reg),
9912 zr,
9913 as_Register($src$$reg),
9914 (Assembler::Condition)$cmp$$cmpcode);
9915 %}
9916
9917 ins_pipe(icond_reg);
9918 %}
9919
9920 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9921 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9922
9923 ins_cost(INSN_COST * 2);
9924 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9925
9926 ins_encode %{
9927 __ cselw(as_Register($dst$$reg),
9928 zr,
9929 as_Register($src$$reg),
9930 (Assembler::Condition)$cmp$$cmpcode);
9931 %}
9932
9933 ins_pipe(icond_reg);
9934 %}
9935
9936 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9937 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9938
9939 ins_cost(INSN_COST * 2);
9940 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9941
9942 ins_encode %{
9943 __ cselw(as_Register($dst$$reg),
9944 as_Register($src$$reg),
9945 zr,
9946 (Assembler::Condition)$cmp$$cmpcode);
9947 %}
9948
9949 ins_pipe(icond_reg);
9950 %}
9951
9952 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9953 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9954
9955 ins_cost(INSN_COST * 2);
9956 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9957
9958 ins_encode %{
9959 __ cselw(as_Register($dst$$reg),
9960 as_Register($src$$reg),
9961 zr,
9962 (Assembler::Condition)$cmp$$cmpcode);
9963 %}
9964
9965 ins_pipe(icond_reg);
9966 %}
9967
9968 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9969 %{
9970 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9971
9972 ins_cost(INSN_COST * 3);
9973
9974 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9975 ins_encode %{
9976 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9977 __ fcsels(as_FloatRegister($dst$$reg),
9978 as_FloatRegister($src2$$reg),
9979 as_FloatRegister($src1$$reg),
9980 cond);
9981 %}
9982
9983 ins_pipe(fp_cond_reg_reg_s);
9984 %}
9985
9986 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9987 %{
9988 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9989
9990 ins_cost(INSN_COST * 3);
9991
9992 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9993 ins_encode %{
9994 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9995 __ fcsels(as_FloatRegister($dst$$reg),
9996 as_FloatRegister($src2$$reg),
9997 as_FloatRegister($src1$$reg),
9998 cond);
9999 %}
10000
10001 ins_pipe(fp_cond_reg_reg_s);
10002 %}
10003
10004 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
10005 %{
10006 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10007
10008 ins_cost(INSN_COST * 3);
10009
10010 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
10011 ins_encode %{
10012 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10013 __ fcseld(as_FloatRegister($dst$$reg),
10014 as_FloatRegister($src2$$reg),
10015 as_FloatRegister($src1$$reg),
10016 cond);
10017 %}
10018
10019 ins_pipe(fp_cond_reg_reg_d);
10020 %}
10021
10022 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
10023 %{
10024 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
10025
10026 ins_cost(INSN_COST * 3);
10027
10028 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
10029 ins_encode %{
10030 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
10031 __ fcseld(as_FloatRegister($dst$$reg),
10032 as_FloatRegister($src2$$reg),
10033 as_FloatRegister($src1$$reg),
10034 cond);
10035 %}
10036
10037 ins_pipe(fp_cond_reg_reg_d);
10038 %}
10039
10040 // ============================================================================
10041 // Arithmetic Instructions
10042 //
10043
10044 // Integer Addition
10045
10046 // TODO
10047 // these currently employ operations which do not set CR and hence are
10048 // not flagged as killing CR but we would like to isolate the cases
10049 // where we want to set flags from those where we don't. need to work
10050 // out how to do that.
10051
10052 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10053 match(Set dst (AddI src1 src2));
10054
10055 ins_cost(INSN_COST);
10056 format %{ "addw $dst, $src1, $src2" %}
10057
10058 ins_encode %{
10059 __ addw(as_Register($dst$$reg),
10060 as_Register($src1$$reg),
10061 as_Register($src2$$reg));
10062 %}
10063
10064 ins_pipe(ialu_reg_reg);
10065 %}
10066
10067 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10068 match(Set dst (AddI src1 src2));
10069
10070 ins_cost(INSN_COST);
10071 format %{ "addw $dst, $src1, $src2" %}
10072
10073 // use opcode to indicate that this is an add not a sub
10074 opcode(0x0);
10075
10076 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10077
10078 ins_pipe(ialu_reg_imm);
10079 %}
10080
10081 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
10082 match(Set dst (AddI (ConvL2I src1) src2));
10083
10084 ins_cost(INSN_COST);
10085 format %{ "addw $dst, $src1, $src2" %}
10086
10087 // use opcode to indicate that this is an add not a sub
10088 opcode(0x0);
10089
10090 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10091
10092 ins_pipe(ialu_reg_imm);
10093 %}
10094
10095 // Pointer Addition
10096 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
10097 match(Set dst (AddP src1 src2));
10098
10099 ins_cost(INSN_COST);
10100 format %{ "add $dst, $src1, $src2\t# ptr" %}
10101
10102 ins_encode %{
10103 __ add(as_Register($dst$$reg),
10104 as_Register($src1$$reg),
10105 as_Register($src2$$reg));
10106 %}
10107
10108 ins_pipe(ialu_reg_reg);
10109 %}
10110
10111 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
10112 match(Set dst (AddP src1 (ConvI2L src2)));
10113
10114 ins_cost(1.9 * INSN_COST);
10115 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
10116
10117 ins_encode %{
10118 __ add(as_Register($dst$$reg),
10119 as_Register($src1$$reg),
10120 as_Register($src2$$reg), ext::sxtw);
10121 %}
10122
10123 ins_pipe(ialu_reg_reg);
10124 %}
10125
10126 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
10127 match(Set dst (AddP src1 (LShiftL src2 scale)));
10128
10129 ins_cost(1.9 * INSN_COST);
10130 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
10131
10132 ins_encode %{
10133 __ lea(as_Register($dst$$reg),
10134 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10135 Address::lsl($scale$$constant)));
10136 %}
10137
10138 ins_pipe(ialu_reg_reg_shift);
10139 %}
10140
10141 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
10142 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
10143
10144 ins_cost(1.9 * INSN_COST);
10145 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
10146
10147 ins_encode %{
10148 __ lea(as_Register($dst$$reg),
10149 Address(as_Register($src1$$reg), as_Register($src2$$reg),
10150 Address::sxtw($scale$$constant)));
10151 %}
10152
10153 ins_pipe(ialu_reg_reg_shift);
10154 %}
10155
10156 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
10157 match(Set dst (LShiftL (ConvI2L src) scale));
10158
10159 ins_cost(INSN_COST);
10160 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
10161
10162 ins_encode %{
10163 __ sbfiz(as_Register($dst$$reg),
10164 as_Register($src$$reg),
10165 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
10166 %}
10167
10168 ins_pipe(ialu_reg_shift);
10169 %}
10170
10171 // Pointer Immediate Addition
10172 // n.b. this needs to be more expensive than using an indirect memory
10173 // operand
10174 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
10175 match(Set dst (AddP src1 src2));
10176
10177 ins_cost(INSN_COST);
10178 format %{ "add $dst, $src1, $src2\t# ptr" %}
10179
10180 // use opcode to indicate that this is an add not a sub
10181 opcode(0x0);
10182
10183 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10184
10185 ins_pipe(ialu_reg_imm);
10186 %}
10187
10188 // Long Addition
10189 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10190
10191 match(Set dst (AddL src1 src2));
10192
10193 ins_cost(INSN_COST);
10194 format %{ "add $dst, $src1, $src2" %}
10195
10196 ins_encode %{
10197 __ add(as_Register($dst$$reg),
10198 as_Register($src1$$reg),
10199 as_Register($src2$$reg));
10200 %}
10201
10202 ins_pipe(ialu_reg_reg);
10203 %}
10204
10205 // No constant pool entries requiredLong Immediate Addition.
10206 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10207 match(Set dst (AddL src1 src2));
10208
10209 ins_cost(INSN_COST);
10210 format %{ "add $dst, $src1, $src2" %}
10211
10212 // use opcode to indicate that this is an add not a sub
10213 opcode(0x0);
10214
10215 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10216
10217 ins_pipe(ialu_reg_imm);
10218 %}
10219
10220 // Integer Subtraction
10221 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10222 match(Set dst (SubI src1 src2));
10223
10224 ins_cost(INSN_COST);
10225 format %{ "subw $dst, $src1, $src2" %}
10226
10227 ins_encode %{
10228 __ subw(as_Register($dst$$reg),
10229 as_Register($src1$$reg),
10230 as_Register($src2$$reg));
10231 %}
10232
10233 ins_pipe(ialu_reg_reg);
10234 %}
10235
10236 // Immediate Subtraction
10237 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
10238 match(Set dst (SubI src1 src2));
10239
10240 ins_cost(INSN_COST);
10241 format %{ "subw $dst, $src1, $src2" %}
10242
10243 // use opcode to indicate that this is a sub not an add
10244 opcode(0x1);
10245
10246 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
10247
10248 ins_pipe(ialu_reg_imm);
10249 %}
10250
10251 // Long Subtraction
10252 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10253
10254 match(Set dst (SubL src1 src2));
10255
10256 ins_cost(INSN_COST);
10257 format %{ "sub $dst, $src1, $src2" %}
10258
10259 ins_encode %{
10260 __ sub(as_Register($dst$$reg),
10261 as_Register($src1$$reg),
10262 as_Register($src2$$reg));
10263 %}
10264
10265 ins_pipe(ialu_reg_reg);
10266 %}
10267
10268 // No constant pool entries requiredLong Immediate Subtraction.
10269 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10270 match(Set dst (SubL src1 src2));
10271
10272 ins_cost(INSN_COST);
10273 format %{ "sub$dst, $src1, $src2" %}
10274
10275 // use opcode to indicate that this is a sub not an add
10276 opcode(0x1);
10277
10278 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10279
10280 ins_pipe(ialu_reg_imm);
10281 %}
10282
10283 // Integer Negation (special case for sub)
10284
10285 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10286 match(Set dst (SubI zero src));
10287
10288 ins_cost(INSN_COST);
10289 format %{ "negw $dst, $src\t# int" %}
10290
10291 ins_encode %{
10292 __ negw(as_Register($dst$$reg),
10293 as_Register($src$$reg));
10294 %}
10295
10296 ins_pipe(ialu_reg);
10297 %}
10298
10299 // Long Negation
10300
10301 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10302 match(Set dst (SubL zero src));
10303
10304 ins_cost(INSN_COST);
10305 format %{ "neg $dst, $src\t# long" %}
10306
10307 ins_encode %{
10308 __ neg(as_Register($dst$$reg),
10309 as_Register($src$$reg));
10310 %}
10311
10312 ins_pipe(ialu_reg);
10313 %}
10314
10315 // Integer Multiply
10316
10317 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10318 match(Set dst (MulI src1 src2));
10319
10320 ins_cost(INSN_COST * 3);
10321 format %{ "mulw $dst, $src1, $src2" %}
10322
10323 ins_encode %{
10324 __ mulw(as_Register($dst$$reg),
10325 as_Register($src1$$reg),
10326 as_Register($src2$$reg));
10327 %}
10328
10329 ins_pipe(imul_reg_reg);
10330 %}
10331
10332 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10333 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10334
10335 ins_cost(INSN_COST * 3);
10336 format %{ "smull $dst, $src1, $src2" %}
10337
10338 ins_encode %{
10339 __ smull(as_Register($dst$$reg),
10340 as_Register($src1$$reg),
10341 as_Register($src2$$reg));
10342 %}
10343
10344 ins_pipe(imul_reg_reg);
10345 %}
10346
10347 // Long Multiply
10348
10349 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10350 match(Set dst (MulL src1 src2));
10351
10352 ins_cost(INSN_COST * 5);
10353 format %{ "mul $dst, $src1, $src2" %}
10354
10355 ins_encode %{
10356 __ mul(as_Register($dst$$reg),
10357 as_Register($src1$$reg),
10358 as_Register($src2$$reg));
10359 %}
10360
10361 ins_pipe(lmul_reg_reg);
10362 %}
10363
10364 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10365 %{
10366 match(Set dst (MulHiL src1 src2));
10367
10368 ins_cost(INSN_COST * 7);
10369 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10370
10371 ins_encode %{
10372 __ smulh(as_Register($dst$$reg),
10373 as_Register($src1$$reg),
10374 as_Register($src2$$reg));
10375 %}
10376
10377 ins_pipe(lmul_reg_reg);
10378 %}
10379
10380 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10381 %{
10382 match(Set dst (UMulHiL src1 src2));
10383
10384 ins_cost(INSN_COST * 7);
10385 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10386
10387 ins_encode %{
10388 __ umulh(as_Register($dst$$reg),
10389 as_Register($src1$$reg),
10390 as_Register($src2$$reg));
10391 %}
10392
10393 ins_pipe(lmul_reg_reg);
10394 %}
10395
10396 // Combined Integer Multiply & Add/Sub
10397
10398 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10399 match(Set dst (AddI src3 (MulI src1 src2)));
10400
10401 ins_cost(INSN_COST * 3);
10402 format %{ "madd $dst, $src1, $src2, $src3" %}
10403
10404 ins_encode %{
10405 __ maddw(as_Register($dst$$reg),
10406 as_Register($src1$$reg),
10407 as_Register($src2$$reg),
10408 as_Register($src3$$reg));
10409 %}
10410
10411 ins_pipe(imac_reg_reg);
10412 %}
10413
10414 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10415 match(Set dst (SubI src3 (MulI src1 src2)));
10416
10417 ins_cost(INSN_COST * 3);
10418 format %{ "msub $dst, $src1, $src2, $src3" %}
10419
10420 ins_encode %{
10421 __ msubw(as_Register($dst$$reg),
10422 as_Register($src1$$reg),
10423 as_Register($src2$$reg),
10424 as_Register($src3$$reg));
10425 %}
10426
10427 ins_pipe(imac_reg_reg);
10428 %}
10429
10430 // Combined Integer Multiply & Neg
10431
10432 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10433 match(Set dst (MulI (SubI zero src1) src2));
10434
10435 ins_cost(INSN_COST * 3);
10436 format %{ "mneg $dst, $src1, $src2" %}
10437
10438 ins_encode %{
10439 __ mnegw(as_Register($dst$$reg),
10440 as_Register($src1$$reg),
10441 as_Register($src2$$reg));
10442 %}
10443
10444 ins_pipe(imac_reg_reg);
10445 %}
10446
10447 // Combined Long Multiply & Add/Sub
10448
10449 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10450 match(Set dst (AddL src3 (MulL src1 src2)));
10451
10452 ins_cost(INSN_COST * 5);
10453 format %{ "madd $dst, $src1, $src2, $src3" %}
10454
10455 ins_encode %{
10456 __ madd(as_Register($dst$$reg),
10457 as_Register($src1$$reg),
10458 as_Register($src2$$reg),
10459 as_Register($src3$$reg));
10460 %}
10461
10462 ins_pipe(lmac_reg_reg);
10463 %}
10464
10465 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10466 match(Set dst (SubL src3 (MulL src1 src2)));
10467
10468 ins_cost(INSN_COST * 5);
10469 format %{ "msub $dst, $src1, $src2, $src3" %}
10470
10471 ins_encode %{
10472 __ msub(as_Register($dst$$reg),
10473 as_Register($src1$$reg),
10474 as_Register($src2$$reg),
10475 as_Register($src3$$reg));
10476 %}
10477
10478 ins_pipe(lmac_reg_reg);
10479 %}
10480
10481 // Combined Long Multiply & Neg
10482
10483 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10484 match(Set dst (MulL (SubL zero src1) src2));
10485
10486 ins_cost(INSN_COST * 5);
10487 format %{ "mneg $dst, $src1, $src2" %}
10488
10489 ins_encode %{
10490 __ mneg(as_Register($dst$$reg),
10491 as_Register($src1$$reg),
10492 as_Register($src2$$reg));
10493 %}
10494
10495 ins_pipe(lmac_reg_reg);
10496 %}
10497
10498 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10499
10500 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10501 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10502
10503 ins_cost(INSN_COST * 3);
10504 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10505
10506 ins_encode %{
10507 __ smaddl(as_Register($dst$$reg),
10508 as_Register($src1$$reg),
10509 as_Register($src2$$reg),
10510 as_Register($src3$$reg));
10511 %}
10512
10513 ins_pipe(imac_reg_reg);
10514 %}
10515
10516 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10517 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10518
10519 ins_cost(INSN_COST * 3);
10520 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10521
10522 ins_encode %{
10523 __ smsubl(as_Register($dst$$reg),
10524 as_Register($src1$$reg),
10525 as_Register($src2$$reg),
10526 as_Register($src3$$reg));
10527 %}
10528
10529 ins_pipe(imac_reg_reg);
10530 %}
10531
10532 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10533 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10534
10535 ins_cost(INSN_COST * 3);
10536 format %{ "smnegl $dst, $src1, $src2" %}
10537
10538 ins_encode %{
10539 __ smnegl(as_Register($dst$$reg),
10540 as_Register($src1$$reg),
10541 as_Register($src2$$reg));
10542 %}
10543
10544 ins_pipe(imac_reg_reg);
10545 %}
10546
10547 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10548
10549 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10550 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10551
10552 ins_cost(INSN_COST * 5);
10553 format %{ "mulw rscratch1, $src1, $src2\n\t"
10554 "maddw $dst, $src3, $src4, rscratch1" %}
10555
10556 ins_encode %{
10557 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10558 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10559
10560 ins_pipe(imac_reg_reg);
10561 %}
10562
10563 // Integer Divide
10564
10565 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10566 match(Set dst (DivI src1 src2));
10567
10568 ins_cost(INSN_COST * 19);
10569 format %{ "sdivw $dst, $src1, $src2" %}
10570
10571 ins_encode(aarch64_enc_divw(dst, src1, src2));
10572 ins_pipe(idiv_reg_reg);
10573 %}
10574
10575 // Long Divide
10576
10577 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10578 match(Set dst (DivL src1 src2));
10579
10580 ins_cost(INSN_COST * 35);
10581 format %{ "sdiv $dst, $src1, $src2" %}
10582
10583 ins_encode(aarch64_enc_div(dst, src1, src2));
10584 ins_pipe(ldiv_reg_reg);
10585 %}
10586
10587 // Integer Remainder
10588
10589 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10590 match(Set dst (ModI src1 src2));
10591
10592 ins_cost(INSN_COST * 22);
10593 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10594 "msubw $dst, rscratch1, $src2, $src1" %}
10595
10596 ins_encode(aarch64_enc_modw(dst, src1, src2));
10597 ins_pipe(idiv_reg_reg);
10598 %}
10599
10600 // Long Remainder
10601
10602 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10603 match(Set dst (ModL src1 src2));
10604
10605 ins_cost(INSN_COST * 38);
10606 format %{ "sdiv rscratch1, $src1, $src2\n"
10607 "msub $dst, rscratch1, $src2, $src1" %}
10608
10609 ins_encode(aarch64_enc_mod(dst, src1, src2));
10610 ins_pipe(ldiv_reg_reg);
10611 %}
10612
10613 // Unsigned Integer Divide
10614
10615 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10616 match(Set dst (UDivI src1 src2));
10617
10618 ins_cost(INSN_COST * 19);
10619 format %{ "udivw $dst, $src1, $src2" %}
10620
10621 ins_encode %{
10622 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10623 %}
10624
10625 ins_pipe(idiv_reg_reg);
10626 %}
10627
10628 // Unsigned Long Divide
10629
10630 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10631 match(Set dst (UDivL src1 src2));
10632
10633 ins_cost(INSN_COST * 35);
10634 format %{ "udiv $dst, $src1, $src2" %}
10635
10636 ins_encode %{
10637 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10638 %}
10639
10640 ins_pipe(ldiv_reg_reg);
10641 %}
10642
10643 // Unsigned Integer Remainder
10644
10645 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10646 match(Set dst (UModI src1 src2));
10647
10648 ins_cost(INSN_COST * 22);
10649 format %{ "udivw rscratch1, $src1, $src2\n\t"
10650 "msubw $dst, rscratch1, $src2, $src1" %}
10651
10652 ins_encode %{
10653 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10654 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10655 %}
10656
10657 ins_pipe(idiv_reg_reg);
10658 %}
10659
10660 // Unsigned Long Remainder
10661
10662 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10663 match(Set dst (UModL src1 src2));
10664
10665 ins_cost(INSN_COST * 38);
10666 format %{ "udiv rscratch1, $src1, $src2\n"
10667 "msub $dst, rscratch1, $src2, $src1" %}
10668
10669 ins_encode %{
10670 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10671 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10672 %}
10673
10674 ins_pipe(ldiv_reg_reg);
10675 %}
10676
10677 // Integer Shifts
10678
10679 // Shift Left Register
10680 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10681 match(Set dst (LShiftI src1 src2));
10682
10683 ins_cost(INSN_COST * 2);
10684 format %{ "lslvw $dst, $src1, $src2" %}
10685
10686 ins_encode %{
10687 __ lslvw(as_Register($dst$$reg),
10688 as_Register($src1$$reg),
10689 as_Register($src2$$reg));
10690 %}
10691
10692 ins_pipe(ialu_reg_reg_vshift);
10693 %}
10694
10695 // Shift Left Immediate
10696 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10697 match(Set dst (LShiftI src1 src2));
10698
10699 ins_cost(INSN_COST);
10700 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10701
10702 ins_encode %{
10703 __ lslw(as_Register($dst$$reg),
10704 as_Register($src1$$reg),
10705 $src2$$constant & 0x1f);
10706 %}
10707
10708 ins_pipe(ialu_reg_shift);
10709 %}
10710
10711 // Shift Right Logical Register
10712 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10713 match(Set dst (URShiftI src1 src2));
10714
10715 ins_cost(INSN_COST * 2);
10716 format %{ "lsrvw $dst, $src1, $src2" %}
10717
10718 ins_encode %{
10719 __ lsrvw(as_Register($dst$$reg),
10720 as_Register($src1$$reg),
10721 as_Register($src2$$reg));
10722 %}
10723
10724 ins_pipe(ialu_reg_reg_vshift);
10725 %}
10726
10727 // Shift Right Logical Immediate
10728 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10729 match(Set dst (URShiftI src1 src2));
10730
10731 ins_cost(INSN_COST);
10732 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10733
10734 ins_encode %{
10735 __ lsrw(as_Register($dst$$reg),
10736 as_Register($src1$$reg),
10737 $src2$$constant & 0x1f);
10738 %}
10739
10740 ins_pipe(ialu_reg_shift);
10741 %}
10742
10743 // Shift Right Arithmetic Register
10744 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10745 match(Set dst (RShiftI src1 src2));
10746
10747 ins_cost(INSN_COST * 2);
10748 format %{ "asrvw $dst, $src1, $src2" %}
10749
10750 ins_encode %{
10751 __ asrvw(as_Register($dst$$reg),
10752 as_Register($src1$$reg),
10753 as_Register($src2$$reg));
10754 %}
10755
10756 ins_pipe(ialu_reg_reg_vshift);
10757 %}
10758
10759 // Shift Right Arithmetic Immediate
10760 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10761 match(Set dst (RShiftI src1 src2));
10762
10763 ins_cost(INSN_COST);
10764 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10765
10766 ins_encode %{
10767 __ asrw(as_Register($dst$$reg),
10768 as_Register($src1$$reg),
10769 $src2$$constant & 0x1f);
10770 %}
10771
10772 ins_pipe(ialu_reg_shift);
10773 %}
10774
10775 // Combined Int Mask and Right Shift (using UBFM)
10776 // TODO
10777
10778 // Long Shifts
10779
10780 // Shift Left Register
10781 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10782 match(Set dst (LShiftL src1 src2));
10783
10784 ins_cost(INSN_COST * 2);
10785 format %{ "lslv $dst, $src1, $src2" %}
10786
10787 ins_encode %{
10788 __ lslv(as_Register($dst$$reg),
10789 as_Register($src1$$reg),
10790 as_Register($src2$$reg));
10791 %}
10792
10793 ins_pipe(ialu_reg_reg_vshift);
10794 %}
10795
10796 // Shift Left Immediate
10797 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10798 match(Set dst (LShiftL src1 src2));
10799
10800 ins_cost(INSN_COST);
10801 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10802
10803 ins_encode %{
10804 __ lsl(as_Register($dst$$reg),
10805 as_Register($src1$$reg),
10806 $src2$$constant & 0x3f);
10807 %}
10808
10809 ins_pipe(ialu_reg_shift);
10810 %}
10811
10812 // Shift Right Logical Register
10813 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10814 match(Set dst (URShiftL src1 src2));
10815
10816 ins_cost(INSN_COST * 2);
10817 format %{ "lsrv $dst, $src1, $src2" %}
10818
10819 ins_encode %{
10820 __ lsrv(as_Register($dst$$reg),
10821 as_Register($src1$$reg),
10822 as_Register($src2$$reg));
10823 %}
10824
10825 ins_pipe(ialu_reg_reg_vshift);
10826 %}
10827
10828 // Shift Right Logical Immediate
10829 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10830 match(Set dst (URShiftL src1 src2));
10831
10832 ins_cost(INSN_COST);
10833 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10834
10835 ins_encode %{
10836 __ lsr(as_Register($dst$$reg),
10837 as_Register($src1$$reg),
10838 $src2$$constant & 0x3f);
10839 %}
10840
10841 ins_pipe(ialu_reg_shift);
10842 %}
10843
10844 // A special-case pattern for card table stores.
10845 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10846 match(Set dst (URShiftL (CastP2X src1) src2));
10847
10848 ins_cost(INSN_COST);
10849 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10850
10851 ins_encode %{
10852 __ lsr(as_Register($dst$$reg),
10853 as_Register($src1$$reg),
10854 $src2$$constant & 0x3f);
10855 %}
10856
10857 ins_pipe(ialu_reg_shift);
10858 %}
10859
10860 // Shift Right Arithmetic Register
10861 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10862 match(Set dst (RShiftL src1 src2));
10863
10864 ins_cost(INSN_COST * 2);
10865 format %{ "asrv $dst, $src1, $src2" %}
10866
10867 ins_encode %{
10868 __ asrv(as_Register($dst$$reg),
10869 as_Register($src1$$reg),
10870 as_Register($src2$$reg));
10871 %}
10872
10873 ins_pipe(ialu_reg_reg_vshift);
10874 %}
10875
10876 // Shift Right Arithmetic Immediate
10877 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10878 match(Set dst (RShiftL src1 src2));
10879
10880 ins_cost(INSN_COST);
10881 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10882
10883 ins_encode %{
10884 __ asr(as_Register($dst$$reg),
10885 as_Register($src1$$reg),
10886 $src2$$constant & 0x3f);
10887 %}
10888
10889 ins_pipe(ialu_reg_shift);
10890 %}
10891
10892 // BEGIN This section of the file is automatically generated. Do not edit --------------
10893 // This section is generated from aarch64_ad.m4
10894
10895 // This pattern is automatically generated from aarch64_ad.m4
10896 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10897 instruct regL_not_reg(iRegLNoSp dst,
10898 iRegL src1, immL_M1 m1,
10899 rFlagsReg cr) %{
10900 match(Set dst (XorL src1 m1));
10901 ins_cost(INSN_COST);
10902 format %{ "eon $dst, $src1, zr" %}
10903
10904 ins_encode %{
10905 __ eon(as_Register($dst$$reg),
10906 as_Register($src1$$reg),
10907 zr,
10908 Assembler::LSL, 0);
10909 %}
10910
10911 ins_pipe(ialu_reg);
10912 %}
10913
10914 // This pattern is automatically generated from aarch64_ad.m4
10915 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10916 instruct regI_not_reg(iRegINoSp dst,
10917 iRegIorL2I src1, immI_M1 m1,
10918 rFlagsReg cr) %{
10919 match(Set dst (XorI src1 m1));
10920 ins_cost(INSN_COST);
10921 format %{ "eonw $dst, $src1, zr" %}
10922
10923 ins_encode %{
10924 __ eonw(as_Register($dst$$reg),
10925 as_Register($src1$$reg),
10926 zr,
10927 Assembler::LSL, 0);
10928 %}
10929
10930 ins_pipe(ialu_reg);
10931 %}
10932
10933 // This pattern is automatically generated from aarch64_ad.m4
10934 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10935 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10936 immI0 zero, iRegIorL2I src1, immI src2) %{
10937 match(Set dst (SubI zero (URShiftI src1 src2)));
10938
10939 ins_cost(1.9 * INSN_COST);
10940 format %{ "negw $dst, $src1, LSR $src2" %}
10941
10942 ins_encode %{
10943 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10944 Assembler::LSR, $src2$$constant & 0x1f);
10945 %}
10946
10947 ins_pipe(ialu_reg_shift);
10948 %}
10949
10950 // This pattern is automatically generated from aarch64_ad.m4
10951 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10952 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10953 immI0 zero, iRegIorL2I src1, immI src2) %{
10954 match(Set dst (SubI zero (RShiftI src1 src2)));
10955
10956 ins_cost(1.9 * INSN_COST);
10957 format %{ "negw $dst, $src1, ASR $src2" %}
10958
10959 ins_encode %{
10960 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10961 Assembler::ASR, $src2$$constant & 0x1f);
10962 %}
10963
10964 ins_pipe(ialu_reg_shift);
10965 %}
10966
10967 // This pattern is automatically generated from aarch64_ad.m4
10968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10969 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10970 immI0 zero, iRegIorL2I src1, immI src2) %{
10971 match(Set dst (SubI zero (LShiftI src1 src2)));
10972
10973 ins_cost(1.9 * INSN_COST);
10974 format %{ "negw $dst, $src1, LSL $src2" %}
10975
10976 ins_encode %{
10977 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10978 Assembler::LSL, $src2$$constant & 0x1f);
10979 %}
10980
10981 ins_pipe(ialu_reg_shift);
10982 %}
10983
10984 // This pattern is automatically generated from aarch64_ad.m4
10985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10986 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10987 immL0 zero, iRegL src1, immI src2) %{
10988 match(Set dst (SubL zero (URShiftL src1 src2)));
10989
10990 ins_cost(1.9 * INSN_COST);
10991 format %{ "neg $dst, $src1, LSR $src2" %}
10992
10993 ins_encode %{
10994 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10995 Assembler::LSR, $src2$$constant & 0x3f);
10996 %}
10997
10998 ins_pipe(ialu_reg_shift);
10999 %}
11000
11001 // This pattern is automatically generated from aarch64_ad.m4
11002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11003 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
11004 immL0 zero, iRegL src1, immI src2) %{
11005 match(Set dst (SubL zero (RShiftL src1 src2)));
11006
11007 ins_cost(1.9 * INSN_COST);
11008 format %{ "neg $dst, $src1, ASR $src2" %}
11009
11010 ins_encode %{
11011 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11012 Assembler::ASR, $src2$$constant & 0x3f);
11013 %}
11014
11015 ins_pipe(ialu_reg_shift);
11016 %}
11017
11018 // This pattern is automatically generated from aarch64_ad.m4
11019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11020 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
11021 immL0 zero, iRegL src1, immI src2) %{
11022 match(Set dst (SubL zero (LShiftL src1 src2)));
11023
11024 ins_cost(1.9 * INSN_COST);
11025 format %{ "neg $dst, $src1, LSL $src2" %}
11026
11027 ins_encode %{
11028 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
11029 Assembler::LSL, $src2$$constant & 0x3f);
11030 %}
11031
11032 ins_pipe(ialu_reg_shift);
11033 %}
11034
11035 // This pattern is automatically generated from aarch64_ad.m4
11036 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11037 instruct AndI_reg_not_reg(iRegINoSp dst,
11038 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11039 match(Set dst (AndI src1 (XorI src2 m1)));
11040 ins_cost(INSN_COST);
11041 format %{ "bicw $dst, $src1, $src2" %}
11042
11043 ins_encode %{
11044 __ bicw(as_Register($dst$$reg),
11045 as_Register($src1$$reg),
11046 as_Register($src2$$reg),
11047 Assembler::LSL, 0);
11048 %}
11049
11050 ins_pipe(ialu_reg_reg);
11051 %}
11052
11053 // This pattern is automatically generated from aarch64_ad.m4
11054 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11055 instruct AndL_reg_not_reg(iRegLNoSp dst,
11056 iRegL src1, iRegL src2, immL_M1 m1) %{
11057 match(Set dst (AndL src1 (XorL src2 m1)));
11058 ins_cost(INSN_COST);
11059 format %{ "bic $dst, $src1, $src2" %}
11060
11061 ins_encode %{
11062 __ bic(as_Register($dst$$reg),
11063 as_Register($src1$$reg),
11064 as_Register($src2$$reg),
11065 Assembler::LSL, 0);
11066 %}
11067
11068 ins_pipe(ialu_reg_reg);
11069 %}
11070
11071 // This pattern is automatically generated from aarch64_ad.m4
11072 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11073 instruct OrI_reg_not_reg(iRegINoSp dst,
11074 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11075 match(Set dst (OrI src1 (XorI src2 m1)));
11076 ins_cost(INSN_COST);
11077 format %{ "ornw $dst, $src1, $src2" %}
11078
11079 ins_encode %{
11080 __ ornw(as_Register($dst$$reg),
11081 as_Register($src1$$reg),
11082 as_Register($src2$$reg),
11083 Assembler::LSL, 0);
11084 %}
11085
11086 ins_pipe(ialu_reg_reg);
11087 %}
11088
11089 // This pattern is automatically generated from aarch64_ad.m4
11090 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11091 instruct OrL_reg_not_reg(iRegLNoSp dst,
11092 iRegL src1, iRegL src2, immL_M1 m1) %{
11093 match(Set dst (OrL src1 (XorL src2 m1)));
11094 ins_cost(INSN_COST);
11095 format %{ "orn $dst, $src1, $src2" %}
11096
11097 ins_encode %{
11098 __ orn(as_Register($dst$$reg),
11099 as_Register($src1$$reg),
11100 as_Register($src2$$reg),
11101 Assembler::LSL, 0);
11102 %}
11103
11104 ins_pipe(ialu_reg_reg);
11105 %}
11106
11107 // This pattern is automatically generated from aarch64_ad.m4
11108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11109 instruct XorI_reg_not_reg(iRegINoSp dst,
11110 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
11111 match(Set dst (XorI m1 (XorI src2 src1)));
11112 ins_cost(INSN_COST);
11113 format %{ "eonw $dst, $src1, $src2" %}
11114
11115 ins_encode %{
11116 __ eonw(as_Register($dst$$reg),
11117 as_Register($src1$$reg),
11118 as_Register($src2$$reg),
11119 Assembler::LSL, 0);
11120 %}
11121
11122 ins_pipe(ialu_reg_reg);
11123 %}
11124
11125 // This pattern is automatically generated from aarch64_ad.m4
11126 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11127 instruct XorL_reg_not_reg(iRegLNoSp dst,
11128 iRegL src1, iRegL src2, immL_M1 m1) %{
11129 match(Set dst (XorL m1 (XorL src2 src1)));
11130 ins_cost(INSN_COST);
11131 format %{ "eon $dst, $src1, $src2" %}
11132
11133 ins_encode %{
11134 __ eon(as_Register($dst$$reg),
11135 as_Register($src1$$reg),
11136 as_Register($src2$$reg),
11137 Assembler::LSL, 0);
11138 %}
11139
11140 ins_pipe(ialu_reg_reg);
11141 %}
11142
11143 // This pattern is automatically generated from aarch64_ad.m4
11144 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11145 // val & (-1 ^ (val >>> shift)) ==> bicw
11146 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
11147 iRegIorL2I src1, iRegIorL2I src2,
11148 immI src3, immI_M1 src4) %{
11149 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
11150 ins_cost(1.9 * INSN_COST);
11151 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
11152
11153 ins_encode %{
11154 __ bicw(as_Register($dst$$reg),
11155 as_Register($src1$$reg),
11156 as_Register($src2$$reg),
11157 Assembler::LSR,
11158 $src3$$constant & 0x1f);
11159 %}
11160
11161 ins_pipe(ialu_reg_reg_shift);
11162 %}
11163
11164 // This pattern is automatically generated from aarch64_ad.m4
11165 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11166 // val & (-1 ^ (val >>> shift)) ==> bic
11167 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
11168 iRegL src1, iRegL src2,
11169 immI src3, immL_M1 src4) %{
11170 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
11171 ins_cost(1.9 * INSN_COST);
11172 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
11173
11174 ins_encode %{
11175 __ bic(as_Register($dst$$reg),
11176 as_Register($src1$$reg),
11177 as_Register($src2$$reg),
11178 Assembler::LSR,
11179 $src3$$constant & 0x3f);
11180 %}
11181
11182 ins_pipe(ialu_reg_reg_shift);
11183 %}
11184
11185 // This pattern is automatically generated from aarch64_ad.m4
11186 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11187 // val & (-1 ^ (val >> shift)) ==> bicw
11188 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
11189 iRegIorL2I src1, iRegIorL2I src2,
11190 immI src3, immI_M1 src4) %{
11191 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
11192 ins_cost(1.9 * INSN_COST);
11193 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
11194
11195 ins_encode %{
11196 __ bicw(as_Register($dst$$reg),
11197 as_Register($src1$$reg),
11198 as_Register($src2$$reg),
11199 Assembler::ASR,
11200 $src3$$constant & 0x1f);
11201 %}
11202
11203 ins_pipe(ialu_reg_reg_shift);
11204 %}
11205
11206 // This pattern is automatically generated from aarch64_ad.m4
11207 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11208 // val & (-1 ^ (val >> shift)) ==> bic
11209 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
11210 iRegL src1, iRegL src2,
11211 immI src3, immL_M1 src4) %{
11212 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
11213 ins_cost(1.9 * INSN_COST);
11214 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
11215
11216 ins_encode %{
11217 __ bic(as_Register($dst$$reg),
11218 as_Register($src1$$reg),
11219 as_Register($src2$$reg),
11220 Assembler::ASR,
11221 $src3$$constant & 0x3f);
11222 %}
11223
11224 ins_pipe(ialu_reg_reg_shift);
11225 %}
11226
11227 // This pattern is automatically generated from aarch64_ad.m4
11228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11229 // val & (-1 ^ (val ror shift)) ==> bicw
11230 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
11231 iRegIorL2I src1, iRegIorL2I src2,
11232 immI src3, immI_M1 src4) %{
11233 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
11234 ins_cost(1.9 * INSN_COST);
11235 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
11236
11237 ins_encode %{
11238 __ bicw(as_Register($dst$$reg),
11239 as_Register($src1$$reg),
11240 as_Register($src2$$reg),
11241 Assembler::ROR,
11242 $src3$$constant & 0x1f);
11243 %}
11244
11245 ins_pipe(ialu_reg_reg_shift);
11246 %}
11247
11248 // This pattern is automatically generated from aarch64_ad.m4
11249 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11250 // val & (-1 ^ (val ror shift)) ==> bic
11251 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
11252 iRegL src1, iRegL src2,
11253 immI src3, immL_M1 src4) %{
11254 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
11255 ins_cost(1.9 * INSN_COST);
11256 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
11257
11258 ins_encode %{
11259 __ bic(as_Register($dst$$reg),
11260 as_Register($src1$$reg),
11261 as_Register($src2$$reg),
11262 Assembler::ROR,
11263 $src3$$constant & 0x3f);
11264 %}
11265
11266 ins_pipe(ialu_reg_reg_shift);
11267 %}
11268
11269 // This pattern is automatically generated from aarch64_ad.m4
11270 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11271 // val & (-1 ^ (val << shift)) ==> bicw
11272 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11273 iRegIorL2I src1, iRegIorL2I src2,
11274 immI src3, immI_M1 src4) %{
11275 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11276 ins_cost(1.9 * INSN_COST);
11277 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11278
11279 ins_encode %{
11280 __ bicw(as_Register($dst$$reg),
11281 as_Register($src1$$reg),
11282 as_Register($src2$$reg),
11283 Assembler::LSL,
11284 $src3$$constant & 0x1f);
11285 %}
11286
11287 ins_pipe(ialu_reg_reg_shift);
11288 %}
11289
11290 // This pattern is automatically generated from aarch64_ad.m4
11291 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11292 // val & (-1 ^ (val << shift)) ==> bic
11293 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11294 iRegL src1, iRegL src2,
11295 immI src3, immL_M1 src4) %{
11296 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11297 ins_cost(1.9 * INSN_COST);
11298 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11299
11300 ins_encode %{
11301 __ bic(as_Register($dst$$reg),
11302 as_Register($src1$$reg),
11303 as_Register($src2$$reg),
11304 Assembler::LSL,
11305 $src3$$constant & 0x3f);
11306 %}
11307
11308 ins_pipe(ialu_reg_reg_shift);
11309 %}
11310
11311 // This pattern is automatically generated from aarch64_ad.m4
11312 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11313 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11314 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11315 iRegIorL2I src1, iRegIorL2I src2,
11316 immI src3, immI_M1 src4) %{
11317 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11318 ins_cost(1.9 * INSN_COST);
11319 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11320
11321 ins_encode %{
11322 __ eonw(as_Register($dst$$reg),
11323 as_Register($src1$$reg),
11324 as_Register($src2$$reg),
11325 Assembler::LSR,
11326 $src3$$constant & 0x1f);
11327 %}
11328
11329 ins_pipe(ialu_reg_reg_shift);
11330 %}
11331
11332 // This pattern is automatically generated from aarch64_ad.m4
11333 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11334 // val ^ (-1 ^ (val >>> shift)) ==> eon
11335 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11336 iRegL src1, iRegL src2,
11337 immI src3, immL_M1 src4) %{
11338 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11339 ins_cost(1.9 * INSN_COST);
11340 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11341
11342 ins_encode %{
11343 __ eon(as_Register($dst$$reg),
11344 as_Register($src1$$reg),
11345 as_Register($src2$$reg),
11346 Assembler::LSR,
11347 $src3$$constant & 0x3f);
11348 %}
11349
11350 ins_pipe(ialu_reg_reg_shift);
11351 %}
11352
11353 // This pattern is automatically generated from aarch64_ad.m4
11354 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11355 // val ^ (-1 ^ (val >> shift)) ==> eonw
11356 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11357 iRegIorL2I src1, iRegIorL2I src2,
11358 immI src3, immI_M1 src4) %{
11359 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11360 ins_cost(1.9 * INSN_COST);
11361 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11362
11363 ins_encode %{
11364 __ eonw(as_Register($dst$$reg),
11365 as_Register($src1$$reg),
11366 as_Register($src2$$reg),
11367 Assembler::ASR,
11368 $src3$$constant & 0x1f);
11369 %}
11370
11371 ins_pipe(ialu_reg_reg_shift);
11372 %}
11373
11374 // This pattern is automatically generated from aarch64_ad.m4
11375 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11376 // val ^ (-1 ^ (val >> shift)) ==> eon
11377 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11378 iRegL src1, iRegL src2,
11379 immI src3, immL_M1 src4) %{
11380 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11381 ins_cost(1.9 * INSN_COST);
11382 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11383
11384 ins_encode %{
11385 __ eon(as_Register($dst$$reg),
11386 as_Register($src1$$reg),
11387 as_Register($src2$$reg),
11388 Assembler::ASR,
11389 $src3$$constant & 0x3f);
11390 %}
11391
11392 ins_pipe(ialu_reg_reg_shift);
11393 %}
11394
11395 // This pattern is automatically generated from aarch64_ad.m4
11396 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11397 // val ^ (-1 ^ (val ror shift)) ==> eonw
11398 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11399 iRegIorL2I src1, iRegIorL2I src2,
11400 immI src3, immI_M1 src4) %{
11401 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11402 ins_cost(1.9 * INSN_COST);
11403 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11404
11405 ins_encode %{
11406 __ eonw(as_Register($dst$$reg),
11407 as_Register($src1$$reg),
11408 as_Register($src2$$reg),
11409 Assembler::ROR,
11410 $src3$$constant & 0x1f);
11411 %}
11412
11413 ins_pipe(ialu_reg_reg_shift);
11414 %}
11415
11416 // This pattern is automatically generated from aarch64_ad.m4
11417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11418 // val ^ (-1 ^ (val ror shift)) ==> eon
11419 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11420 iRegL src1, iRegL src2,
11421 immI src3, immL_M1 src4) %{
11422 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11423 ins_cost(1.9 * INSN_COST);
11424 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11425
11426 ins_encode %{
11427 __ eon(as_Register($dst$$reg),
11428 as_Register($src1$$reg),
11429 as_Register($src2$$reg),
11430 Assembler::ROR,
11431 $src3$$constant & 0x3f);
11432 %}
11433
11434 ins_pipe(ialu_reg_reg_shift);
11435 %}
11436
11437 // This pattern is automatically generated from aarch64_ad.m4
11438 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11439 // val ^ (-1 ^ (val << shift)) ==> eonw
11440 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11441 iRegIorL2I src1, iRegIorL2I src2,
11442 immI src3, immI_M1 src4) %{
11443 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11444 ins_cost(1.9 * INSN_COST);
11445 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11446
11447 ins_encode %{
11448 __ eonw(as_Register($dst$$reg),
11449 as_Register($src1$$reg),
11450 as_Register($src2$$reg),
11451 Assembler::LSL,
11452 $src3$$constant & 0x1f);
11453 %}
11454
11455 ins_pipe(ialu_reg_reg_shift);
11456 %}
11457
11458 // This pattern is automatically generated from aarch64_ad.m4
11459 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11460 // val ^ (-1 ^ (val << shift)) ==> eon
11461 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11462 iRegL src1, iRegL src2,
11463 immI src3, immL_M1 src4) %{
11464 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11465 ins_cost(1.9 * INSN_COST);
11466 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11467
11468 ins_encode %{
11469 __ eon(as_Register($dst$$reg),
11470 as_Register($src1$$reg),
11471 as_Register($src2$$reg),
11472 Assembler::LSL,
11473 $src3$$constant & 0x3f);
11474 %}
11475
11476 ins_pipe(ialu_reg_reg_shift);
11477 %}
11478
11479 // This pattern is automatically generated from aarch64_ad.m4
11480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11481 // val | (-1 ^ (val >>> shift)) ==> ornw
11482 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11483 iRegIorL2I src1, iRegIorL2I src2,
11484 immI src3, immI_M1 src4) %{
11485 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11486 ins_cost(1.9 * INSN_COST);
11487 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11488
11489 ins_encode %{
11490 __ ornw(as_Register($dst$$reg),
11491 as_Register($src1$$reg),
11492 as_Register($src2$$reg),
11493 Assembler::LSR,
11494 $src3$$constant & 0x1f);
11495 %}
11496
11497 ins_pipe(ialu_reg_reg_shift);
11498 %}
11499
11500 // This pattern is automatically generated from aarch64_ad.m4
11501 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11502 // val | (-1 ^ (val >>> shift)) ==> orn
11503 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11504 iRegL src1, iRegL src2,
11505 immI src3, immL_M1 src4) %{
11506 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11507 ins_cost(1.9 * INSN_COST);
11508 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11509
11510 ins_encode %{
11511 __ orn(as_Register($dst$$reg),
11512 as_Register($src1$$reg),
11513 as_Register($src2$$reg),
11514 Assembler::LSR,
11515 $src3$$constant & 0x3f);
11516 %}
11517
11518 ins_pipe(ialu_reg_reg_shift);
11519 %}
11520
11521 // This pattern is automatically generated from aarch64_ad.m4
11522 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11523 // val | (-1 ^ (val >> shift)) ==> ornw
11524 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11525 iRegIorL2I src1, iRegIorL2I src2,
11526 immI src3, immI_M1 src4) %{
11527 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11528 ins_cost(1.9 * INSN_COST);
11529 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11530
11531 ins_encode %{
11532 __ ornw(as_Register($dst$$reg),
11533 as_Register($src1$$reg),
11534 as_Register($src2$$reg),
11535 Assembler::ASR,
11536 $src3$$constant & 0x1f);
11537 %}
11538
11539 ins_pipe(ialu_reg_reg_shift);
11540 %}
11541
11542 // This pattern is automatically generated from aarch64_ad.m4
11543 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11544 // val | (-1 ^ (val >> shift)) ==> orn
11545 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11546 iRegL src1, iRegL src2,
11547 immI src3, immL_M1 src4) %{
11548 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11549 ins_cost(1.9 * INSN_COST);
11550 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11551
11552 ins_encode %{
11553 __ orn(as_Register($dst$$reg),
11554 as_Register($src1$$reg),
11555 as_Register($src2$$reg),
11556 Assembler::ASR,
11557 $src3$$constant & 0x3f);
11558 %}
11559
11560 ins_pipe(ialu_reg_reg_shift);
11561 %}
11562
11563 // This pattern is automatically generated from aarch64_ad.m4
11564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11565 // val | (-1 ^ (val ror shift)) ==> ornw
11566 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11567 iRegIorL2I src1, iRegIorL2I src2,
11568 immI src3, immI_M1 src4) %{
11569 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11570 ins_cost(1.9 * INSN_COST);
11571 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11572
11573 ins_encode %{
11574 __ ornw(as_Register($dst$$reg),
11575 as_Register($src1$$reg),
11576 as_Register($src2$$reg),
11577 Assembler::ROR,
11578 $src3$$constant & 0x1f);
11579 %}
11580
11581 ins_pipe(ialu_reg_reg_shift);
11582 %}
11583
11584 // This pattern is automatically generated from aarch64_ad.m4
11585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11586 // val | (-1 ^ (val ror shift)) ==> orn
11587 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11588 iRegL src1, iRegL src2,
11589 immI src3, immL_M1 src4) %{
11590 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11591 ins_cost(1.9 * INSN_COST);
11592 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11593
11594 ins_encode %{
11595 __ orn(as_Register($dst$$reg),
11596 as_Register($src1$$reg),
11597 as_Register($src2$$reg),
11598 Assembler::ROR,
11599 $src3$$constant & 0x3f);
11600 %}
11601
11602 ins_pipe(ialu_reg_reg_shift);
11603 %}
11604
11605 // This pattern is automatically generated from aarch64_ad.m4
11606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11607 // val | (-1 ^ (val << shift)) ==> ornw
11608 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11609 iRegIorL2I src1, iRegIorL2I src2,
11610 immI src3, immI_M1 src4) %{
11611 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11612 ins_cost(1.9 * INSN_COST);
11613 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11614
11615 ins_encode %{
11616 __ ornw(as_Register($dst$$reg),
11617 as_Register($src1$$reg),
11618 as_Register($src2$$reg),
11619 Assembler::LSL,
11620 $src3$$constant & 0x1f);
11621 %}
11622
11623 ins_pipe(ialu_reg_reg_shift);
11624 %}
11625
11626 // This pattern is automatically generated from aarch64_ad.m4
11627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11628 // val | (-1 ^ (val << shift)) ==> orn
11629 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11630 iRegL src1, iRegL src2,
11631 immI src3, immL_M1 src4) %{
11632 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11633 ins_cost(1.9 * INSN_COST);
11634 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11635
11636 ins_encode %{
11637 __ orn(as_Register($dst$$reg),
11638 as_Register($src1$$reg),
11639 as_Register($src2$$reg),
11640 Assembler::LSL,
11641 $src3$$constant & 0x3f);
11642 %}
11643
11644 ins_pipe(ialu_reg_reg_shift);
11645 %}
11646
11647 // This pattern is automatically generated from aarch64_ad.m4
11648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11649 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11650 iRegIorL2I src1, iRegIorL2I src2,
11651 immI src3) %{
11652 match(Set dst (AndI src1 (URShiftI src2 src3)));
11653
11654 ins_cost(1.9 * INSN_COST);
11655 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11656
11657 ins_encode %{
11658 __ andw(as_Register($dst$$reg),
11659 as_Register($src1$$reg),
11660 as_Register($src2$$reg),
11661 Assembler::LSR,
11662 $src3$$constant & 0x1f);
11663 %}
11664
11665 ins_pipe(ialu_reg_reg_shift);
11666 %}
11667
11668 // This pattern is automatically generated from aarch64_ad.m4
11669 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11670 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11671 iRegL src1, iRegL src2,
11672 immI src3) %{
11673 match(Set dst (AndL src1 (URShiftL src2 src3)));
11674
11675 ins_cost(1.9 * INSN_COST);
11676 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11677
11678 ins_encode %{
11679 __ andr(as_Register($dst$$reg),
11680 as_Register($src1$$reg),
11681 as_Register($src2$$reg),
11682 Assembler::LSR,
11683 $src3$$constant & 0x3f);
11684 %}
11685
11686 ins_pipe(ialu_reg_reg_shift);
11687 %}
11688
11689 // This pattern is automatically generated from aarch64_ad.m4
11690 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11691 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11692 iRegIorL2I src1, iRegIorL2I src2,
11693 immI src3) %{
11694 match(Set dst (AndI src1 (RShiftI src2 src3)));
11695
11696 ins_cost(1.9 * INSN_COST);
11697 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11698
11699 ins_encode %{
11700 __ andw(as_Register($dst$$reg),
11701 as_Register($src1$$reg),
11702 as_Register($src2$$reg),
11703 Assembler::ASR,
11704 $src3$$constant & 0x1f);
11705 %}
11706
11707 ins_pipe(ialu_reg_reg_shift);
11708 %}
11709
11710 // This pattern is automatically generated from aarch64_ad.m4
11711 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11712 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11713 iRegL src1, iRegL src2,
11714 immI src3) %{
11715 match(Set dst (AndL src1 (RShiftL src2 src3)));
11716
11717 ins_cost(1.9 * INSN_COST);
11718 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11719
11720 ins_encode %{
11721 __ andr(as_Register($dst$$reg),
11722 as_Register($src1$$reg),
11723 as_Register($src2$$reg),
11724 Assembler::ASR,
11725 $src3$$constant & 0x3f);
11726 %}
11727
11728 ins_pipe(ialu_reg_reg_shift);
11729 %}
11730
11731 // This pattern is automatically generated from aarch64_ad.m4
11732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11733 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11734 iRegIorL2I src1, iRegIorL2I src2,
11735 immI src3) %{
11736 match(Set dst (AndI src1 (LShiftI src2 src3)));
11737
11738 ins_cost(1.9 * INSN_COST);
11739 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11740
11741 ins_encode %{
11742 __ andw(as_Register($dst$$reg),
11743 as_Register($src1$$reg),
11744 as_Register($src2$$reg),
11745 Assembler::LSL,
11746 $src3$$constant & 0x1f);
11747 %}
11748
11749 ins_pipe(ialu_reg_reg_shift);
11750 %}
11751
11752 // This pattern is automatically generated from aarch64_ad.m4
11753 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11754 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11755 iRegL src1, iRegL src2,
11756 immI src3) %{
11757 match(Set dst (AndL src1 (LShiftL src2 src3)));
11758
11759 ins_cost(1.9 * INSN_COST);
11760 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11761
11762 ins_encode %{
11763 __ andr(as_Register($dst$$reg),
11764 as_Register($src1$$reg),
11765 as_Register($src2$$reg),
11766 Assembler::LSL,
11767 $src3$$constant & 0x3f);
11768 %}
11769
11770 ins_pipe(ialu_reg_reg_shift);
11771 %}
11772
11773 // This pattern is automatically generated from aarch64_ad.m4
11774 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11775 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11776 iRegIorL2I src1, iRegIorL2I src2,
11777 immI src3) %{
11778 match(Set dst (AndI src1 (RotateRight src2 src3)));
11779
11780 ins_cost(1.9 * INSN_COST);
11781 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11782
11783 ins_encode %{
11784 __ andw(as_Register($dst$$reg),
11785 as_Register($src1$$reg),
11786 as_Register($src2$$reg),
11787 Assembler::ROR,
11788 $src3$$constant & 0x1f);
11789 %}
11790
11791 ins_pipe(ialu_reg_reg_shift);
11792 %}
11793
11794 // This pattern is automatically generated from aarch64_ad.m4
11795 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11796 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11797 iRegL src1, iRegL src2,
11798 immI src3) %{
11799 match(Set dst (AndL src1 (RotateRight src2 src3)));
11800
11801 ins_cost(1.9 * INSN_COST);
11802 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11803
11804 ins_encode %{
11805 __ andr(as_Register($dst$$reg),
11806 as_Register($src1$$reg),
11807 as_Register($src2$$reg),
11808 Assembler::ROR,
11809 $src3$$constant & 0x3f);
11810 %}
11811
11812 ins_pipe(ialu_reg_reg_shift);
11813 %}
11814
11815 // This pattern is automatically generated from aarch64_ad.m4
11816 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11817 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11818 iRegIorL2I src1, iRegIorL2I src2,
11819 immI src3) %{
11820 match(Set dst (XorI src1 (URShiftI src2 src3)));
11821
11822 ins_cost(1.9 * INSN_COST);
11823 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11824
11825 ins_encode %{
11826 __ eorw(as_Register($dst$$reg),
11827 as_Register($src1$$reg),
11828 as_Register($src2$$reg),
11829 Assembler::LSR,
11830 $src3$$constant & 0x1f);
11831 %}
11832
11833 ins_pipe(ialu_reg_reg_shift);
11834 %}
11835
11836 // This pattern is automatically generated from aarch64_ad.m4
11837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11838 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11839 iRegL src1, iRegL src2,
11840 immI src3) %{
11841 match(Set dst (XorL src1 (URShiftL src2 src3)));
11842
11843 ins_cost(1.9 * INSN_COST);
11844 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11845
11846 ins_encode %{
11847 __ eor(as_Register($dst$$reg),
11848 as_Register($src1$$reg),
11849 as_Register($src2$$reg),
11850 Assembler::LSR,
11851 $src3$$constant & 0x3f);
11852 %}
11853
11854 ins_pipe(ialu_reg_reg_shift);
11855 %}
11856
11857 // This pattern is automatically generated from aarch64_ad.m4
11858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11859 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11860 iRegIorL2I src1, iRegIorL2I src2,
11861 immI src3) %{
11862 match(Set dst (XorI src1 (RShiftI src2 src3)));
11863
11864 ins_cost(1.9 * INSN_COST);
11865 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11866
11867 ins_encode %{
11868 __ eorw(as_Register($dst$$reg),
11869 as_Register($src1$$reg),
11870 as_Register($src2$$reg),
11871 Assembler::ASR,
11872 $src3$$constant & 0x1f);
11873 %}
11874
11875 ins_pipe(ialu_reg_reg_shift);
11876 %}
11877
11878 // This pattern is automatically generated from aarch64_ad.m4
11879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11880 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11881 iRegL src1, iRegL src2,
11882 immI src3) %{
11883 match(Set dst (XorL src1 (RShiftL src2 src3)));
11884
11885 ins_cost(1.9 * INSN_COST);
11886 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11887
11888 ins_encode %{
11889 __ eor(as_Register($dst$$reg),
11890 as_Register($src1$$reg),
11891 as_Register($src2$$reg),
11892 Assembler::ASR,
11893 $src3$$constant & 0x3f);
11894 %}
11895
11896 ins_pipe(ialu_reg_reg_shift);
11897 %}
11898
11899 // This pattern is automatically generated from aarch64_ad.m4
11900 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11901 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11902 iRegIorL2I src1, iRegIorL2I src2,
11903 immI src3) %{
11904 match(Set dst (XorI src1 (LShiftI src2 src3)));
11905
11906 ins_cost(1.9 * INSN_COST);
11907 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11908
11909 ins_encode %{
11910 __ eorw(as_Register($dst$$reg),
11911 as_Register($src1$$reg),
11912 as_Register($src2$$reg),
11913 Assembler::LSL,
11914 $src3$$constant & 0x1f);
11915 %}
11916
11917 ins_pipe(ialu_reg_reg_shift);
11918 %}
11919
11920 // This pattern is automatically generated from aarch64_ad.m4
11921 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11922 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11923 iRegL src1, iRegL src2,
11924 immI src3) %{
11925 match(Set dst (XorL src1 (LShiftL src2 src3)));
11926
11927 ins_cost(1.9 * INSN_COST);
11928 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11929
11930 ins_encode %{
11931 __ eor(as_Register($dst$$reg),
11932 as_Register($src1$$reg),
11933 as_Register($src2$$reg),
11934 Assembler::LSL,
11935 $src3$$constant & 0x3f);
11936 %}
11937
11938 ins_pipe(ialu_reg_reg_shift);
11939 %}
11940
11941 // This pattern is automatically generated from aarch64_ad.m4
11942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11943 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11944 iRegIorL2I src1, iRegIorL2I src2,
11945 immI src3) %{
11946 match(Set dst (XorI src1 (RotateRight src2 src3)));
11947
11948 ins_cost(1.9 * INSN_COST);
11949 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11950
11951 ins_encode %{
11952 __ eorw(as_Register($dst$$reg),
11953 as_Register($src1$$reg),
11954 as_Register($src2$$reg),
11955 Assembler::ROR,
11956 $src3$$constant & 0x1f);
11957 %}
11958
11959 ins_pipe(ialu_reg_reg_shift);
11960 %}
11961
11962 // This pattern is automatically generated from aarch64_ad.m4
11963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11964 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11965 iRegL src1, iRegL src2,
11966 immI src3) %{
11967 match(Set dst (XorL src1 (RotateRight src2 src3)));
11968
11969 ins_cost(1.9 * INSN_COST);
11970 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11971
11972 ins_encode %{
11973 __ eor(as_Register($dst$$reg),
11974 as_Register($src1$$reg),
11975 as_Register($src2$$reg),
11976 Assembler::ROR,
11977 $src3$$constant & 0x3f);
11978 %}
11979
11980 ins_pipe(ialu_reg_reg_shift);
11981 %}
11982
11983 // This pattern is automatically generated from aarch64_ad.m4
11984 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11985 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11986 iRegIorL2I src1, iRegIorL2I src2,
11987 immI src3) %{
11988 match(Set dst (OrI src1 (URShiftI src2 src3)));
11989
11990 ins_cost(1.9 * INSN_COST);
11991 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11992
11993 ins_encode %{
11994 __ orrw(as_Register($dst$$reg),
11995 as_Register($src1$$reg),
11996 as_Register($src2$$reg),
11997 Assembler::LSR,
11998 $src3$$constant & 0x1f);
11999 %}
12000
12001 ins_pipe(ialu_reg_reg_shift);
12002 %}
12003
12004 // This pattern is automatically generated from aarch64_ad.m4
12005 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12006 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
12007 iRegL src1, iRegL src2,
12008 immI src3) %{
12009 match(Set dst (OrL src1 (URShiftL src2 src3)));
12010
12011 ins_cost(1.9 * INSN_COST);
12012 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
12013
12014 ins_encode %{
12015 __ orr(as_Register($dst$$reg),
12016 as_Register($src1$$reg),
12017 as_Register($src2$$reg),
12018 Assembler::LSR,
12019 $src3$$constant & 0x3f);
12020 %}
12021
12022 ins_pipe(ialu_reg_reg_shift);
12023 %}
12024
12025 // This pattern is automatically generated from aarch64_ad.m4
12026 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12027 instruct OrI_reg_RShift_reg(iRegINoSp dst,
12028 iRegIorL2I src1, iRegIorL2I src2,
12029 immI src3) %{
12030 match(Set dst (OrI src1 (RShiftI src2 src3)));
12031
12032 ins_cost(1.9 * INSN_COST);
12033 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
12034
12035 ins_encode %{
12036 __ orrw(as_Register($dst$$reg),
12037 as_Register($src1$$reg),
12038 as_Register($src2$$reg),
12039 Assembler::ASR,
12040 $src3$$constant & 0x1f);
12041 %}
12042
12043 ins_pipe(ialu_reg_reg_shift);
12044 %}
12045
12046 // This pattern is automatically generated from aarch64_ad.m4
12047 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12048 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
12049 iRegL src1, iRegL src2,
12050 immI src3) %{
12051 match(Set dst (OrL src1 (RShiftL src2 src3)));
12052
12053 ins_cost(1.9 * INSN_COST);
12054 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
12055
12056 ins_encode %{
12057 __ orr(as_Register($dst$$reg),
12058 as_Register($src1$$reg),
12059 as_Register($src2$$reg),
12060 Assembler::ASR,
12061 $src3$$constant & 0x3f);
12062 %}
12063
12064 ins_pipe(ialu_reg_reg_shift);
12065 %}
12066
12067 // This pattern is automatically generated from aarch64_ad.m4
12068 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12069 instruct OrI_reg_LShift_reg(iRegINoSp dst,
12070 iRegIorL2I src1, iRegIorL2I src2,
12071 immI src3) %{
12072 match(Set dst (OrI src1 (LShiftI src2 src3)));
12073
12074 ins_cost(1.9 * INSN_COST);
12075 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
12076
12077 ins_encode %{
12078 __ orrw(as_Register($dst$$reg),
12079 as_Register($src1$$reg),
12080 as_Register($src2$$reg),
12081 Assembler::LSL,
12082 $src3$$constant & 0x1f);
12083 %}
12084
12085 ins_pipe(ialu_reg_reg_shift);
12086 %}
12087
12088 // This pattern is automatically generated from aarch64_ad.m4
12089 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12090 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
12091 iRegL src1, iRegL src2,
12092 immI src3) %{
12093 match(Set dst (OrL src1 (LShiftL src2 src3)));
12094
12095 ins_cost(1.9 * INSN_COST);
12096 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
12097
12098 ins_encode %{
12099 __ orr(as_Register($dst$$reg),
12100 as_Register($src1$$reg),
12101 as_Register($src2$$reg),
12102 Assembler::LSL,
12103 $src3$$constant & 0x3f);
12104 %}
12105
12106 ins_pipe(ialu_reg_reg_shift);
12107 %}
12108
12109 // This pattern is automatically generated from aarch64_ad.m4
12110 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12111 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
12112 iRegIorL2I src1, iRegIorL2I src2,
12113 immI src3) %{
12114 match(Set dst (OrI src1 (RotateRight src2 src3)));
12115
12116 ins_cost(1.9 * INSN_COST);
12117 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
12118
12119 ins_encode %{
12120 __ orrw(as_Register($dst$$reg),
12121 as_Register($src1$$reg),
12122 as_Register($src2$$reg),
12123 Assembler::ROR,
12124 $src3$$constant & 0x1f);
12125 %}
12126
12127 ins_pipe(ialu_reg_reg_shift);
12128 %}
12129
12130 // This pattern is automatically generated from aarch64_ad.m4
12131 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12132 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
12133 iRegL src1, iRegL src2,
12134 immI src3) %{
12135 match(Set dst (OrL src1 (RotateRight src2 src3)));
12136
12137 ins_cost(1.9 * INSN_COST);
12138 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
12139
12140 ins_encode %{
12141 __ orr(as_Register($dst$$reg),
12142 as_Register($src1$$reg),
12143 as_Register($src2$$reg),
12144 Assembler::ROR,
12145 $src3$$constant & 0x3f);
12146 %}
12147
12148 ins_pipe(ialu_reg_reg_shift);
12149 %}
12150
12151 // This pattern is automatically generated from aarch64_ad.m4
12152 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12153 instruct AddI_reg_URShift_reg(iRegINoSp dst,
12154 iRegIorL2I src1, iRegIorL2I src2,
12155 immI src3) %{
12156 match(Set dst (AddI src1 (URShiftI src2 src3)));
12157
12158 ins_cost(1.9 * INSN_COST);
12159 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
12160
12161 ins_encode %{
12162 __ addw(as_Register($dst$$reg),
12163 as_Register($src1$$reg),
12164 as_Register($src2$$reg),
12165 Assembler::LSR,
12166 $src3$$constant & 0x1f);
12167 %}
12168
12169 ins_pipe(ialu_reg_reg_shift);
12170 %}
12171
12172 // This pattern is automatically generated from aarch64_ad.m4
12173 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12174 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
12175 iRegL src1, iRegL src2,
12176 immI src3) %{
12177 match(Set dst (AddL src1 (URShiftL src2 src3)));
12178
12179 ins_cost(1.9 * INSN_COST);
12180 format %{ "add $dst, $src1, $src2, LSR $src3" %}
12181
12182 ins_encode %{
12183 __ add(as_Register($dst$$reg),
12184 as_Register($src1$$reg),
12185 as_Register($src2$$reg),
12186 Assembler::LSR,
12187 $src3$$constant & 0x3f);
12188 %}
12189
12190 ins_pipe(ialu_reg_reg_shift);
12191 %}
12192
12193 // This pattern is automatically generated from aarch64_ad.m4
12194 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12195 instruct AddI_reg_RShift_reg(iRegINoSp dst,
12196 iRegIorL2I src1, iRegIorL2I src2,
12197 immI src3) %{
12198 match(Set dst (AddI src1 (RShiftI src2 src3)));
12199
12200 ins_cost(1.9 * INSN_COST);
12201 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
12202
12203 ins_encode %{
12204 __ addw(as_Register($dst$$reg),
12205 as_Register($src1$$reg),
12206 as_Register($src2$$reg),
12207 Assembler::ASR,
12208 $src3$$constant & 0x1f);
12209 %}
12210
12211 ins_pipe(ialu_reg_reg_shift);
12212 %}
12213
12214 // This pattern is automatically generated from aarch64_ad.m4
12215 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12216 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
12217 iRegL src1, iRegL src2,
12218 immI src3) %{
12219 match(Set dst (AddL src1 (RShiftL src2 src3)));
12220
12221 ins_cost(1.9 * INSN_COST);
12222 format %{ "add $dst, $src1, $src2, ASR $src3" %}
12223
12224 ins_encode %{
12225 __ add(as_Register($dst$$reg),
12226 as_Register($src1$$reg),
12227 as_Register($src2$$reg),
12228 Assembler::ASR,
12229 $src3$$constant & 0x3f);
12230 %}
12231
12232 ins_pipe(ialu_reg_reg_shift);
12233 %}
12234
12235 // This pattern is automatically generated from aarch64_ad.m4
12236 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12237 instruct AddI_reg_LShift_reg(iRegINoSp dst,
12238 iRegIorL2I src1, iRegIorL2I src2,
12239 immI src3) %{
12240 match(Set dst (AddI src1 (LShiftI src2 src3)));
12241
12242 ins_cost(1.9 * INSN_COST);
12243 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
12244
12245 ins_encode %{
12246 __ addw(as_Register($dst$$reg),
12247 as_Register($src1$$reg),
12248 as_Register($src2$$reg),
12249 Assembler::LSL,
12250 $src3$$constant & 0x1f);
12251 %}
12252
12253 ins_pipe(ialu_reg_reg_shift);
12254 %}
12255
12256 // This pattern is automatically generated from aarch64_ad.m4
12257 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12258 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
12259 iRegL src1, iRegL src2,
12260 immI src3) %{
12261 match(Set dst (AddL src1 (LShiftL src2 src3)));
12262
12263 ins_cost(1.9 * INSN_COST);
12264 format %{ "add $dst, $src1, $src2, LSL $src3" %}
12265
12266 ins_encode %{
12267 __ add(as_Register($dst$$reg),
12268 as_Register($src1$$reg),
12269 as_Register($src2$$reg),
12270 Assembler::LSL,
12271 $src3$$constant & 0x3f);
12272 %}
12273
12274 ins_pipe(ialu_reg_reg_shift);
12275 %}
12276
12277 // This pattern is automatically generated from aarch64_ad.m4
12278 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12279 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12280 iRegIorL2I src1, iRegIorL2I src2,
12281 immI src3) %{
12282 match(Set dst (SubI src1 (URShiftI src2 src3)));
12283
12284 ins_cost(1.9 * INSN_COST);
12285 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12286
12287 ins_encode %{
12288 __ subw(as_Register($dst$$reg),
12289 as_Register($src1$$reg),
12290 as_Register($src2$$reg),
12291 Assembler::LSR,
12292 $src3$$constant & 0x1f);
12293 %}
12294
12295 ins_pipe(ialu_reg_reg_shift);
12296 %}
12297
12298 // This pattern is automatically generated from aarch64_ad.m4
12299 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12300 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12301 iRegL src1, iRegL src2,
12302 immI src3) %{
12303 match(Set dst (SubL src1 (URShiftL src2 src3)));
12304
12305 ins_cost(1.9 * INSN_COST);
12306 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12307
12308 ins_encode %{
12309 __ sub(as_Register($dst$$reg),
12310 as_Register($src1$$reg),
12311 as_Register($src2$$reg),
12312 Assembler::LSR,
12313 $src3$$constant & 0x3f);
12314 %}
12315
12316 ins_pipe(ialu_reg_reg_shift);
12317 %}
12318
12319 // This pattern is automatically generated from aarch64_ad.m4
12320 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12321 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12322 iRegIorL2I src1, iRegIorL2I src2,
12323 immI src3) %{
12324 match(Set dst (SubI src1 (RShiftI src2 src3)));
12325
12326 ins_cost(1.9 * INSN_COST);
12327 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12328
12329 ins_encode %{
12330 __ subw(as_Register($dst$$reg),
12331 as_Register($src1$$reg),
12332 as_Register($src2$$reg),
12333 Assembler::ASR,
12334 $src3$$constant & 0x1f);
12335 %}
12336
12337 ins_pipe(ialu_reg_reg_shift);
12338 %}
12339
12340 // This pattern is automatically generated from aarch64_ad.m4
12341 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12342 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12343 iRegL src1, iRegL src2,
12344 immI src3) %{
12345 match(Set dst (SubL src1 (RShiftL src2 src3)));
12346
12347 ins_cost(1.9 * INSN_COST);
12348 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12349
12350 ins_encode %{
12351 __ sub(as_Register($dst$$reg),
12352 as_Register($src1$$reg),
12353 as_Register($src2$$reg),
12354 Assembler::ASR,
12355 $src3$$constant & 0x3f);
12356 %}
12357
12358 ins_pipe(ialu_reg_reg_shift);
12359 %}
12360
12361 // This pattern is automatically generated from aarch64_ad.m4
12362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12363 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12364 iRegIorL2I src1, iRegIorL2I src2,
12365 immI src3) %{
12366 match(Set dst (SubI src1 (LShiftI src2 src3)));
12367
12368 ins_cost(1.9 * INSN_COST);
12369 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12370
12371 ins_encode %{
12372 __ subw(as_Register($dst$$reg),
12373 as_Register($src1$$reg),
12374 as_Register($src2$$reg),
12375 Assembler::LSL,
12376 $src3$$constant & 0x1f);
12377 %}
12378
12379 ins_pipe(ialu_reg_reg_shift);
12380 %}
12381
12382 // This pattern is automatically generated from aarch64_ad.m4
12383 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12384 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12385 iRegL src1, iRegL src2,
12386 immI src3) %{
12387 match(Set dst (SubL src1 (LShiftL src2 src3)));
12388
12389 ins_cost(1.9 * INSN_COST);
12390 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12391
12392 ins_encode %{
12393 __ sub(as_Register($dst$$reg),
12394 as_Register($src1$$reg),
12395 as_Register($src2$$reg),
12396 Assembler::LSL,
12397 $src3$$constant & 0x3f);
12398 %}
12399
12400 ins_pipe(ialu_reg_reg_shift);
12401 %}
12402
12403 // This pattern is automatically generated from aarch64_ad.m4
12404 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12405
12406 // Shift Left followed by Shift Right.
12407 // This idiom is used by the compiler for the i2b bytecode etc.
12408 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12409 %{
12410 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12411 ins_cost(INSN_COST * 2);
12412 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12413 ins_encode %{
12414 int lshift = $lshift_count$$constant & 63;
12415 int rshift = $rshift_count$$constant & 63;
12416 int s = 63 - lshift;
12417 int r = (rshift - lshift) & 63;
12418 __ sbfm(as_Register($dst$$reg),
12419 as_Register($src$$reg),
12420 r, s);
12421 %}
12422
12423 ins_pipe(ialu_reg_shift);
12424 %}
12425
12426 // This pattern is automatically generated from aarch64_ad.m4
12427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12428
12429 // Shift Left followed by Shift Right.
12430 // This idiom is used by the compiler for the i2b bytecode etc.
12431 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12432 %{
12433 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12434 ins_cost(INSN_COST * 2);
12435 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12436 ins_encode %{
12437 int lshift = $lshift_count$$constant & 31;
12438 int rshift = $rshift_count$$constant & 31;
12439 int s = 31 - lshift;
12440 int r = (rshift - lshift) & 31;
12441 __ sbfmw(as_Register($dst$$reg),
12442 as_Register($src$$reg),
12443 r, s);
12444 %}
12445
12446 ins_pipe(ialu_reg_shift);
12447 %}
12448
12449 // This pattern is automatically generated from aarch64_ad.m4
12450 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12451
12452 // Shift Left followed by Shift Right.
12453 // This idiom is used by the compiler for the i2b bytecode etc.
12454 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12455 %{
12456 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12457 ins_cost(INSN_COST * 2);
12458 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12459 ins_encode %{
12460 int lshift = $lshift_count$$constant & 63;
12461 int rshift = $rshift_count$$constant & 63;
12462 int s = 63 - lshift;
12463 int r = (rshift - lshift) & 63;
12464 __ ubfm(as_Register($dst$$reg),
12465 as_Register($src$$reg),
12466 r, s);
12467 %}
12468
12469 ins_pipe(ialu_reg_shift);
12470 %}
12471
12472 // This pattern is automatically generated from aarch64_ad.m4
12473 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12474
12475 // Shift Left followed by Shift Right.
12476 // This idiom is used by the compiler for the i2b bytecode etc.
12477 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12478 %{
12479 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12480 ins_cost(INSN_COST * 2);
12481 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12482 ins_encode %{
12483 int lshift = $lshift_count$$constant & 31;
12484 int rshift = $rshift_count$$constant & 31;
12485 int s = 31 - lshift;
12486 int r = (rshift - lshift) & 31;
12487 __ ubfmw(as_Register($dst$$reg),
12488 as_Register($src$$reg),
12489 r, s);
12490 %}
12491
12492 ins_pipe(ialu_reg_shift);
12493 %}
12494
12495 // Bitfield extract with shift & mask
12496
12497 // This pattern is automatically generated from aarch64_ad.m4
12498 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12499 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12500 %{
12501 match(Set dst (AndI (URShiftI src rshift) mask));
12502 // Make sure we are not going to exceed what ubfxw can do.
12503 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12504
12505 ins_cost(INSN_COST);
12506 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12507 ins_encode %{
12508 int rshift = $rshift$$constant & 31;
12509 intptr_t mask = $mask$$constant;
12510 int width = exact_log2(mask+1);
12511 __ ubfxw(as_Register($dst$$reg),
12512 as_Register($src$$reg), rshift, width);
12513 %}
12514 ins_pipe(ialu_reg_shift);
12515 %}
12516
12517 // This pattern is automatically generated from aarch64_ad.m4
12518 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12519 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12520 %{
12521 match(Set dst (AndL (URShiftL src rshift) mask));
12522 // Make sure we are not going to exceed what ubfx can do.
12523 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12524
12525 ins_cost(INSN_COST);
12526 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12527 ins_encode %{
12528 int rshift = $rshift$$constant & 63;
12529 intptr_t mask = $mask$$constant;
12530 int width = exact_log2_long(mask+1);
12531 __ ubfx(as_Register($dst$$reg),
12532 as_Register($src$$reg), rshift, width);
12533 %}
12534 ins_pipe(ialu_reg_shift);
12535 %}
12536
12537
12538 // This pattern is automatically generated from aarch64_ad.m4
12539 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12540
12541 // We can use ubfx when extending an And with a mask when we know mask
12542 // is positive. We know that because immI_bitmask guarantees it.
12543 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12544 %{
12545 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12546 // Make sure we are not going to exceed what ubfxw can do.
12547 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12548
12549 ins_cost(INSN_COST * 2);
12550 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12551 ins_encode %{
12552 int rshift = $rshift$$constant & 31;
12553 intptr_t mask = $mask$$constant;
12554 int width = exact_log2(mask+1);
12555 __ ubfx(as_Register($dst$$reg),
12556 as_Register($src$$reg), rshift, width);
12557 %}
12558 ins_pipe(ialu_reg_shift);
12559 %}
12560
12561
12562 // This pattern is automatically generated from aarch64_ad.m4
12563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12564
12565 // We can use ubfiz when masking by a positive number and then left shifting the result.
12566 // We know that the mask is positive because immI_bitmask guarantees it.
12567 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12568 %{
12569 match(Set dst (LShiftI (AndI src mask) lshift));
12570 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12571
12572 ins_cost(INSN_COST);
12573 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12574 ins_encode %{
12575 int lshift = $lshift$$constant & 31;
12576 intptr_t mask = $mask$$constant;
12577 int width = exact_log2(mask+1);
12578 __ ubfizw(as_Register($dst$$reg),
12579 as_Register($src$$reg), lshift, width);
12580 %}
12581 ins_pipe(ialu_reg_shift);
12582 %}
12583
12584 // This pattern is automatically generated from aarch64_ad.m4
12585 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12586
12587 // We can use ubfiz when masking by a positive number and then left shifting the result.
12588 // We know that the mask is positive because immL_bitmask guarantees it.
12589 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12590 %{
12591 match(Set dst (LShiftL (AndL src mask) lshift));
12592 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12593
12594 ins_cost(INSN_COST);
12595 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12596 ins_encode %{
12597 int lshift = $lshift$$constant & 63;
12598 intptr_t mask = $mask$$constant;
12599 int width = exact_log2_long(mask+1);
12600 __ ubfiz(as_Register($dst$$reg),
12601 as_Register($src$$reg), lshift, width);
12602 %}
12603 ins_pipe(ialu_reg_shift);
12604 %}
12605
12606 // This pattern is automatically generated from aarch64_ad.m4
12607 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12608
12609 // We can use ubfiz when masking by a positive number and then left shifting the result.
12610 // We know that the mask is positive because immI_bitmask guarantees it.
12611 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12612 %{
12613 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12614 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12615
12616 ins_cost(INSN_COST);
12617 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12618 ins_encode %{
12619 int lshift = $lshift$$constant & 31;
12620 intptr_t mask = $mask$$constant;
12621 int width = exact_log2(mask+1);
12622 __ ubfizw(as_Register($dst$$reg),
12623 as_Register($src$$reg), lshift, width);
12624 %}
12625 ins_pipe(ialu_reg_shift);
12626 %}
12627
12628 // This pattern is automatically generated from aarch64_ad.m4
12629 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12630
12631 // We can use ubfiz when masking by a positive number and then left shifting the result.
12632 // We know that the mask is positive because immL_bitmask guarantees it.
12633 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12634 %{
12635 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12636 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12637
12638 ins_cost(INSN_COST);
12639 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12640 ins_encode %{
12641 int lshift = $lshift$$constant & 63;
12642 intptr_t mask = $mask$$constant;
12643 int width = exact_log2_long(mask+1);
12644 __ ubfiz(as_Register($dst$$reg),
12645 as_Register($src$$reg), lshift, width);
12646 %}
12647 ins_pipe(ialu_reg_shift);
12648 %}
12649
12650
12651 // This pattern is automatically generated from aarch64_ad.m4
12652 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12653
12654 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12655 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12656 %{
12657 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12658 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12659
12660 ins_cost(INSN_COST);
12661 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12662 ins_encode %{
12663 int lshift = $lshift$$constant & 63;
12664 intptr_t mask = $mask$$constant;
12665 int width = exact_log2(mask+1);
12666 __ ubfiz(as_Register($dst$$reg),
12667 as_Register($src$$reg), lshift, width);
12668 %}
12669 ins_pipe(ialu_reg_shift);
12670 %}
12671
12672 // This pattern is automatically generated from aarch64_ad.m4
12673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12674
12675 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12676 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12677 %{
12678 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12679 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12680
12681 ins_cost(INSN_COST);
12682 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12683 ins_encode %{
12684 int lshift = $lshift$$constant & 31;
12685 intptr_t mask = $mask$$constant;
12686 int width = exact_log2(mask+1);
12687 __ ubfiz(as_Register($dst$$reg),
12688 as_Register($src$$reg), lshift, width);
12689 %}
12690 ins_pipe(ialu_reg_shift);
12691 %}
12692
12693 // This pattern is automatically generated from aarch64_ad.m4
12694 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12695
12696 // Can skip int2long conversions after AND with small bitmask
12697 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12698 %{
12699 match(Set dst (ConvI2L (AndI src msk)));
12700 ins_cost(INSN_COST);
12701 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12702 ins_encode %{
12703 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12704 %}
12705 ins_pipe(ialu_reg_shift);
12706 %}
12707
12708
12709 // Rotations
12710
12711 // This pattern is automatically generated from aarch64_ad.m4
12712 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12713 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12714 %{
12715 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12716 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12717
12718 ins_cost(INSN_COST);
12719 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12720
12721 ins_encode %{
12722 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12723 $rshift$$constant & 63);
12724 %}
12725 ins_pipe(ialu_reg_reg_extr);
12726 %}
12727
12728
12729 // This pattern is automatically generated from aarch64_ad.m4
12730 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12731 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12732 %{
12733 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12734 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12735
12736 ins_cost(INSN_COST);
12737 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12738
12739 ins_encode %{
12740 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12741 $rshift$$constant & 31);
12742 %}
12743 ins_pipe(ialu_reg_reg_extr);
12744 %}
12745
12746
12747 // This pattern is automatically generated from aarch64_ad.m4
12748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12749 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12750 %{
12751 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12752 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12753
12754 ins_cost(INSN_COST);
12755 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12756
12757 ins_encode %{
12758 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12759 $rshift$$constant & 63);
12760 %}
12761 ins_pipe(ialu_reg_reg_extr);
12762 %}
12763
12764
12765 // This pattern is automatically generated from aarch64_ad.m4
12766 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12767 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12768 %{
12769 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12770 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12771
12772 ins_cost(INSN_COST);
12773 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12774
12775 ins_encode %{
12776 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12777 $rshift$$constant & 31);
12778 %}
12779 ins_pipe(ialu_reg_reg_extr);
12780 %}
12781
12782 // This pattern is automatically generated from aarch64_ad.m4
12783 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12784 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12785 %{
12786 match(Set dst (RotateRight src shift));
12787
12788 ins_cost(INSN_COST);
12789 format %{ "ror $dst, $src, $shift" %}
12790
12791 ins_encode %{
12792 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12793 $shift$$constant & 0x1f);
12794 %}
12795 ins_pipe(ialu_reg_reg_vshift);
12796 %}
12797
12798 // This pattern is automatically generated from aarch64_ad.m4
12799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12800 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12801 %{
12802 match(Set dst (RotateRight src shift));
12803
12804 ins_cost(INSN_COST);
12805 format %{ "ror $dst, $src, $shift" %}
12806
12807 ins_encode %{
12808 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12809 $shift$$constant & 0x3f);
12810 %}
12811 ins_pipe(ialu_reg_reg_vshift);
12812 %}
12813
12814 // This pattern is automatically generated from aarch64_ad.m4
12815 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12816 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12817 %{
12818 match(Set dst (RotateRight src shift));
12819
12820 ins_cost(INSN_COST);
12821 format %{ "ror $dst, $src, $shift" %}
12822
12823 ins_encode %{
12824 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12825 %}
12826 ins_pipe(ialu_reg_reg_vshift);
12827 %}
12828
12829 // This pattern is automatically generated from aarch64_ad.m4
12830 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12831 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12832 %{
12833 match(Set dst (RotateRight src shift));
12834
12835 ins_cost(INSN_COST);
12836 format %{ "ror $dst, $src, $shift" %}
12837
12838 ins_encode %{
12839 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12840 %}
12841 ins_pipe(ialu_reg_reg_vshift);
12842 %}
12843
12844 // This pattern is automatically generated from aarch64_ad.m4
12845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12846 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12847 %{
12848 match(Set dst (RotateLeft src shift));
12849
12850 ins_cost(INSN_COST);
12851 format %{ "rol $dst, $src, $shift" %}
12852
12853 ins_encode %{
12854 __ subw(rscratch1, zr, as_Register($shift$$reg));
12855 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12856 %}
12857 ins_pipe(ialu_reg_reg_vshift);
12858 %}
12859
12860 // This pattern is automatically generated from aarch64_ad.m4
12861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12862 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12863 %{
12864 match(Set dst (RotateLeft src shift));
12865
12866 ins_cost(INSN_COST);
12867 format %{ "rol $dst, $src, $shift" %}
12868
12869 ins_encode %{
12870 __ subw(rscratch1, zr, as_Register($shift$$reg));
12871 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12872 %}
12873 ins_pipe(ialu_reg_reg_vshift);
12874 %}
12875
12876
12877 // Add/subtract (extended)
12878
12879 // This pattern is automatically generated from aarch64_ad.m4
12880 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12881 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12882 %{
12883 match(Set dst (AddL src1 (ConvI2L src2)));
12884 ins_cost(INSN_COST);
12885 format %{ "add $dst, $src1, $src2, sxtw" %}
12886
12887 ins_encode %{
12888 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12889 as_Register($src2$$reg), ext::sxtw);
12890 %}
12891 ins_pipe(ialu_reg_reg);
12892 %}
12893
12894 // This pattern is automatically generated from aarch64_ad.m4
12895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12896 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12897 %{
12898 match(Set dst (SubL src1 (ConvI2L src2)));
12899 ins_cost(INSN_COST);
12900 format %{ "sub $dst, $src1, $src2, sxtw" %}
12901
12902 ins_encode %{
12903 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12904 as_Register($src2$$reg), ext::sxtw);
12905 %}
12906 ins_pipe(ialu_reg_reg);
12907 %}
12908
12909 // This pattern is automatically generated from aarch64_ad.m4
12910 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12911 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12912 %{
12913 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12914 ins_cost(INSN_COST);
12915 format %{ "add $dst, $src1, $src2, sxth" %}
12916
12917 ins_encode %{
12918 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12919 as_Register($src2$$reg), ext::sxth);
12920 %}
12921 ins_pipe(ialu_reg_reg);
12922 %}
12923
12924 // This pattern is automatically generated from aarch64_ad.m4
12925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12926 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12927 %{
12928 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12929 ins_cost(INSN_COST);
12930 format %{ "add $dst, $src1, $src2, sxtb" %}
12931
12932 ins_encode %{
12933 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12934 as_Register($src2$$reg), ext::sxtb);
12935 %}
12936 ins_pipe(ialu_reg_reg);
12937 %}
12938
12939 // This pattern is automatically generated from aarch64_ad.m4
12940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12941 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12942 %{
12943 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12944 ins_cost(INSN_COST);
12945 format %{ "add $dst, $src1, $src2, uxtb" %}
12946
12947 ins_encode %{
12948 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12949 as_Register($src2$$reg), ext::uxtb);
12950 %}
12951 ins_pipe(ialu_reg_reg);
12952 %}
12953
12954 // This pattern is automatically generated from aarch64_ad.m4
12955 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12956 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12957 %{
12958 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12959 ins_cost(INSN_COST);
12960 format %{ "add $dst, $src1, $src2, sxth" %}
12961
12962 ins_encode %{
12963 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12964 as_Register($src2$$reg), ext::sxth);
12965 %}
12966 ins_pipe(ialu_reg_reg);
12967 %}
12968
12969 // This pattern is automatically generated from aarch64_ad.m4
12970 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12971 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12972 %{
12973 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12974 ins_cost(INSN_COST);
12975 format %{ "add $dst, $src1, $src2, sxtw" %}
12976
12977 ins_encode %{
12978 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12979 as_Register($src2$$reg), ext::sxtw);
12980 %}
12981 ins_pipe(ialu_reg_reg);
12982 %}
12983
12984 // This pattern is automatically generated from aarch64_ad.m4
12985 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12986 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12987 %{
12988 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12989 ins_cost(INSN_COST);
12990 format %{ "add $dst, $src1, $src2, sxtb" %}
12991
12992 ins_encode %{
12993 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12994 as_Register($src2$$reg), ext::sxtb);
12995 %}
12996 ins_pipe(ialu_reg_reg);
12997 %}
12998
12999 // This pattern is automatically generated from aarch64_ad.m4
13000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13001 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
13002 %{
13003 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
13004 ins_cost(INSN_COST);
13005 format %{ "add $dst, $src1, $src2, uxtb" %}
13006
13007 ins_encode %{
13008 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13009 as_Register($src2$$reg), ext::uxtb);
13010 %}
13011 ins_pipe(ialu_reg_reg);
13012 %}
13013
13014 // This pattern is automatically generated from aarch64_ad.m4
13015 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13016 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13017 %{
13018 match(Set dst (AddI src1 (AndI src2 mask)));
13019 ins_cost(INSN_COST);
13020 format %{ "addw $dst, $src1, $src2, uxtb" %}
13021
13022 ins_encode %{
13023 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13024 as_Register($src2$$reg), ext::uxtb);
13025 %}
13026 ins_pipe(ialu_reg_reg);
13027 %}
13028
13029 // This pattern is automatically generated from aarch64_ad.m4
13030 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13031 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13032 %{
13033 match(Set dst (AddI src1 (AndI src2 mask)));
13034 ins_cost(INSN_COST);
13035 format %{ "addw $dst, $src1, $src2, uxth" %}
13036
13037 ins_encode %{
13038 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13039 as_Register($src2$$reg), ext::uxth);
13040 %}
13041 ins_pipe(ialu_reg_reg);
13042 %}
13043
13044 // This pattern is automatically generated from aarch64_ad.m4
13045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13046 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13047 %{
13048 match(Set dst (AddL src1 (AndL src2 mask)));
13049 ins_cost(INSN_COST);
13050 format %{ "add $dst, $src1, $src2, uxtb" %}
13051
13052 ins_encode %{
13053 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13054 as_Register($src2$$reg), ext::uxtb);
13055 %}
13056 ins_pipe(ialu_reg_reg);
13057 %}
13058
13059 // This pattern is automatically generated from aarch64_ad.m4
13060 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13061 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13062 %{
13063 match(Set dst (AddL src1 (AndL src2 mask)));
13064 ins_cost(INSN_COST);
13065 format %{ "add $dst, $src1, $src2, uxth" %}
13066
13067 ins_encode %{
13068 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13069 as_Register($src2$$reg), ext::uxth);
13070 %}
13071 ins_pipe(ialu_reg_reg);
13072 %}
13073
13074 // This pattern is automatically generated from aarch64_ad.m4
13075 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13076 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13077 %{
13078 match(Set dst (AddL src1 (AndL src2 mask)));
13079 ins_cost(INSN_COST);
13080 format %{ "add $dst, $src1, $src2, uxtw" %}
13081
13082 ins_encode %{
13083 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13084 as_Register($src2$$reg), ext::uxtw);
13085 %}
13086 ins_pipe(ialu_reg_reg);
13087 %}
13088
13089 // This pattern is automatically generated from aarch64_ad.m4
13090 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13091 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
13092 %{
13093 match(Set dst (SubI src1 (AndI src2 mask)));
13094 ins_cost(INSN_COST);
13095 format %{ "subw $dst, $src1, $src2, uxtb" %}
13096
13097 ins_encode %{
13098 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13099 as_Register($src2$$reg), ext::uxtb);
13100 %}
13101 ins_pipe(ialu_reg_reg);
13102 %}
13103
13104 // This pattern is automatically generated from aarch64_ad.m4
13105 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13106 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
13107 %{
13108 match(Set dst (SubI src1 (AndI src2 mask)));
13109 ins_cost(INSN_COST);
13110 format %{ "subw $dst, $src1, $src2, uxth" %}
13111
13112 ins_encode %{
13113 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13114 as_Register($src2$$reg), ext::uxth);
13115 %}
13116 ins_pipe(ialu_reg_reg);
13117 %}
13118
13119 // This pattern is automatically generated from aarch64_ad.m4
13120 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13121 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
13122 %{
13123 match(Set dst (SubL src1 (AndL src2 mask)));
13124 ins_cost(INSN_COST);
13125 format %{ "sub $dst, $src1, $src2, uxtb" %}
13126
13127 ins_encode %{
13128 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13129 as_Register($src2$$reg), ext::uxtb);
13130 %}
13131 ins_pipe(ialu_reg_reg);
13132 %}
13133
13134 // This pattern is automatically generated from aarch64_ad.m4
13135 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13136 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
13137 %{
13138 match(Set dst (SubL src1 (AndL src2 mask)));
13139 ins_cost(INSN_COST);
13140 format %{ "sub $dst, $src1, $src2, uxth" %}
13141
13142 ins_encode %{
13143 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13144 as_Register($src2$$reg), ext::uxth);
13145 %}
13146 ins_pipe(ialu_reg_reg);
13147 %}
13148
13149 // This pattern is automatically generated from aarch64_ad.m4
13150 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13151 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
13152 %{
13153 match(Set dst (SubL src1 (AndL src2 mask)));
13154 ins_cost(INSN_COST);
13155 format %{ "sub $dst, $src1, $src2, uxtw" %}
13156
13157 ins_encode %{
13158 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13159 as_Register($src2$$reg), ext::uxtw);
13160 %}
13161 ins_pipe(ialu_reg_reg);
13162 %}
13163
13164
13165 // This pattern is automatically generated from aarch64_ad.m4
13166 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13167 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13168 %{
13169 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13170 ins_cost(1.9 * INSN_COST);
13171 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
13172
13173 ins_encode %{
13174 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13175 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13176 %}
13177 ins_pipe(ialu_reg_reg_shift);
13178 %}
13179
13180 // This pattern is automatically generated from aarch64_ad.m4
13181 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13182 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13183 %{
13184 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13185 ins_cost(1.9 * INSN_COST);
13186 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
13187
13188 ins_encode %{
13189 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13190 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13191 %}
13192 ins_pipe(ialu_reg_reg_shift);
13193 %}
13194
13195 // This pattern is automatically generated from aarch64_ad.m4
13196 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13197 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13198 %{
13199 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13200 ins_cost(1.9 * INSN_COST);
13201 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
13202
13203 ins_encode %{
13204 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13205 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13206 %}
13207 ins_pipe(ialu_reg_reg_shift);
13208 %}
13209
13210 // This pattern is automatically generated from aarch64_ad.m4
13211 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13212 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
13213 %{
13214 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13215 ins_cost(1.9 * INSN_COST);
13216 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
13217
13218 ins_encode %{
13219 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13220 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13221 %}
13222 ins_pipe(ialu_reg_reg_shift);
13223 %}
13224
13225 // This pattern is automatically generated from aarch64_ad.m4
13226 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13227 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
13228 %{
13229 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13230 ins_cost(1.9 * INSN_COST);
13231 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
13232
13233 ins_encode %{
13234 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13235 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13236 %}
13237 ins_pipe(ialu_reg_reg_shift);
13238 %}
13239
13240 // This pattern is automatically generated from aarch64_ad.m4
13241 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13242 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
13243 %{
13244 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
13245 ins_cost(1.9 * INSN_COST);
13246 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
13247
13248 ins_encode %{
13249 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13250 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
13251 %}
13252 ins_pipe(ialu_reg_reg_shift);
13253 %}
13254
13255 // This pattern is automatically generated from aarch64_ad.m4
13256 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13257 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13258 %{
13259 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13260 ins_cost(1.9 * INSN_COST);
13261 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
13262
13263 ins_encode %{
13264 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13265 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13266 %}
13267 ins_pipe(ialu_reg_reg_shift);
13268 %}
13269
13270 // This pattern is automatically generated from aarch64_ad.m4
13271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13272 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13273 %{
13274 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13275 ins_cost(1.9 * INSN_COST);
13276 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13277
13278 ins_encode %{
13279 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13280 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13281 %}
13282 ins_pipe(ialu_reg_reg_shift);
13283 %}
13284
13285 // This pattern is automatically generated from aarch64_ad.m4
13286 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13287 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13288 %{
13289 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13290 ins_cost(1.9 * INSN_COST);
13291 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13292
13293 ins_encode %{
13294 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13295 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13296 %}
13297 ins_pipe(ialu_reg_reg_shift);
13298 %}
13299
13300 // This pattern is automatically generated from aarch64_ad.m4
13301 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13302 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13303 %{
13304 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13305 ins_cost(1.9 * INSN_COST);
13306 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13307
13308 ins_encode %{
13309 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13310 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13311 %}
13312 ins_pipe(ialu_reg_reg_shift);
13313 %}
13314
13315 // This pattern is automatically generated from aarch64_ad.m4
13316 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13317 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13318 %{
13319 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13320 ins_cost(1.9 * INSN_COST);
13321 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13322
13323 ins_encode %{
13324 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13325 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13326 %}
13327 ins_pipe(ialu_reg_reg_shift);
13328 %}
13329
13330 // This pattern is automatically generated from aarch64_ad.m4
13331 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13332 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13333 %{
13334 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13335 ins_cost(1.9 * INSN_COST);
13336 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13337
13338 ins_encode %{
13339 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13340 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13341 %}
13342 ins_pipe(ialu_reg_reg_shift);
13343 %}
13344
13345 // This pattern is automatically generated from aarch64_ad.m4
13346 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13347 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13348 %{
13349 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13350 ins_cost(1.9 * INSN_COST);
13351 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13352
13353 ins_encode %{
13354 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13355 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13356 %}
13357 ins_pipe(ialu_reg_reg_shift);
13358 %}
13359
13360 // This pattern is automatically generated from aarch64_ad.m4
13361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13362 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13363 %{
13364 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13365 ins_cost(1.9 * INSN_COST);
13366 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13367
13368 ins_encode %{
13369 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13370 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13371 %}
13372 ins_pipe(ialu_reg_reg_shift);
13373 %}
13374
13375 // This pattern is automatically generated from aarch64_ad.m4
13376 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13377 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13378 %{
13379 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13380 ins_cost(1.9 * INSN_COST);
13381 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13382
13383 ins_encode %{
13384 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13385 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13386 %}
13387 ins_pipe(ialu_reg_reg_shift);
13388 %}
13389
13390 // This pattern is automatically generated from aarch64_ad.m4
13391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13392 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13393 %{
13394 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13395 ins_cost(1.9 * INSN_COST);
13396 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13397
13398 ins_encode %{
13399 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13400 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13401 %}
13402 ins_pipe(ialu_reg_reg_shift);
13403 %}
13404
13405 // This pattern is automatically generated from aarch64_ad.m4
13406 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13407 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13408 %{
13409 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13410 ins_cost(1.9 * INSN_COST);
13411 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13412
13413 ins_encode %{
13414 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13415 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13416 %}
13417 ins_pipe(ialu_reg_reg_shift);
13418 %}
13419
13420 // This pattern is automatically generated from aarch64_ad.m4
13421 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13422 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13423 %{
13424 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13425 ins_cost(1.9 * INSN_COST);
13426 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13427
13428 ins_encode %{
13429 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13430 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13431 %}
13432 ins_pipe(ialu_reg_reg_shift);
13433 %}
13434
13435 // This pattern is automatically generated from aarch64_ad.m4
13436 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13437 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13438 %{
13439 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13440 ins_cost(1.9 * INSN_COST);
13441 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13442
13443 ins_encode %{
13444 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13445 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13446 %}
13447 ins_pipe(ialu_reg_reg_shift);
13448 %}
13449
13450 // This pattern is automatically generated from aarch64_ad.m4
13451 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13452 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13453 %{
13454 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13455 ins_cost(1.9 * INSN_COST);
13456 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13457
13458 ins_encode %{
13459 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13460 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13461 %}
13462 ins_pipe(ialu_reg_reg_shift);
13463 %}
13464
13465 // This pattern is automatically generated from aarch64_ad.m4
13466 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13467 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13468 %{
13469 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13470 ins_cost(1.9 * INSN_COST);
13471 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13472
13473 ins_encode %{
13474 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13475 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13476 %}
13477 ins_pipe(ialu_reg_reg_shift);
13478 %}
13479
13480 // This pattern is automatically generated from aarch64_ad.m4
13481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13482 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13483 %{
13484 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13485 ins_cost(1.9 * INSN_COST);
13486 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13487
13488 ins_encode %{
13489 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13490 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13491 %}
13492 ins_pipe(ialu_reg_reg_shift);
13493 %}
13494
13495 // This pattern is automatically generated from aarch64_ad.m4
13496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13497 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13498 %{
13499 effect(DEF dst, USE src1, USE src2, USE cr);
13500 ins_cost(INSN_COST * 2);
13501 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13502
13503 ins_encode %{
13504 __ cselw($dst$$Register,
13505 $src1$$Register,
13506 $src2$$Register,
13507 Assembler::LT);
13508 %}
13509 ins_pipe(icond_reg_reg);
13510 %}
13511
13512 // This pattern is automatically generated from aarch64_ad.m4
13513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13514 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13515 %{
13516 effect(DEF dst, USE src1, USE src2, USE cr);
13517 ins_cost(INSN_COST * 2);
13518 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13519
13520 ins_encode %{
13521 __ cselw($dst$$Register,
13522 $src1$$Register,
13523 $src2$$Register,
13524 Assembler::GT);
13525 %}
13526 ins_pipe(icond_reg_reg);
13527 %}
13528
13529 // This pattern is automatically generated from aarch64_ad.m4
13530 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13531 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13532 %{
13533 effect(DEF dst, USE src1, USE cr);
13534 ins_cost(INSN_COST * 2);
13535 format %{ "cselw $dst, $src1, zr lt\t" %}
13536
13537 ins_encode %{
13538 __ cselw($dst$$Register,
13539 $src1$$Register,
13540 zr,
13541 Assembler::LT);
13542 %}
13543 ins_pipe(icond_reg);
13544 %}
13545
13546 // This pattern is automatically generated from aarch64_ad.m4
13547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13548 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13549 %{
13550 effect(DEF dst, USE src1, USE cr);
13551 ins_cost(INSN_COST * 2);
13552 format %{ "cselw $dst, $src1, zr gt\t" %}
13553
13554 ins_encode %{
13555 __ cselw($dst$$Register,
13556 $src1$$Register,
13557 zr,
13558 Assembler::GT);
13559 %}
13560 ins_pipe(icond_reg);
13561 %}
13562
13563 // This pattern is automatically generated from aarch64_ad.m4
13564 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13565 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13566 %{
13567 effect(DEF dst, USE src1, USE cr);
13568 ins_cost(INSN_COST * 2);
13569 format %{ "csincw $dst, $src1, zr le\t" %}
13570
13571 ins_encode %{
13572 __ csincw($dst$$Register,
13573 $src1$$Register,
13574 zr,
13575 Assembler::LE);
13576 %}
13577 ins_pipe(icond_reg);
13578 %}
13579
13580 // This pattern is automatically generated from aarch64_ad.m4
13581 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13582 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13583 %{
13584 effect(DEF dst, USE src1, USE cr);
13585 ins_cost(INSN_COST * 2);
13586 format %{ "csincw $dst, $src1, zr gt\t" %}
13587
13588 ins_encode %{
13589 __ csincw($dst$$Register,
13590 $src1$$Register,
13591 zr,
13592 Assembler::GT);
13593 %}
13594 ins_pipe(icond_reg);
13595 %}
13596
13597 // This pattern is automatically generated from aarch64_ad.m4
13598 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13599 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13600 %{
13601 effect(DEF dst, USE src1, USE cr);
13602 ins_cost(INSN_COST * 2);
13603 format %{ "csinvw $dst, $src1, zr lt\t" %}
13604
13605 ins_encode %{
13606 __ csinvw($dst$$Register,
13607 $src1$$Register,
13608 zr,
13609 Assembler::LT);
13610 %}
13611 ins_pipe(icond_reg);
13612 %}
13613
13614 // This pattern is automatically generated from aarch64_ad.m4
13615 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13616 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13617 %{
13618 effect(DEF dst, USE src1, USE cr);
13619 ins_cost(INSN_COST * 2);
13620 format %{ "csinvw $dst, $src1, zr ge\t" %}
13621
13622 ins_encode %{
13623 __ csinvw($dst$$Register,
13624 $src1$$Register,
13625 zr,
13626 Assembler::GE);
13627 %}
13628 ins_pipe(icond_reg);
13629 %}
13630
13631 // This pattern is automatically generated from aarch64_ad.m4
13632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13633 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13634 %{
13635 match(Set dst (MinI src imm));
13636 ins_cost(INSN_COST * 3);
13637 expand %{
13638 rFlagsReg cr;
13639 compI_reg_imm0(cr, src);
13640 cmovI_reg_imm0_lt(dst, src, cr);
13641 %}
13642 %}
13643
13644 // This pattern is automatically generated from aarch64_ad.m4
13645 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13646 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13647 %{
13648 match(Set dst (MinI imm src));
13649 ins_cost(INSN_COST * 3);
13650 expand %{
13651 rFlagsReg cr;
13652 compI_reg_imm0(cr, src);
13653 cmovI_reg_imm0_lt(dst, src, cr);
13654 %}
13655 %}
13656
13657 // This pattern is automatically generated from aarch64_ad.m4
13658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13659 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13660 %{
13661 match(Set dst (MinI src imm));
13662 ins_cost(INSN_COST * 3);
13663 expand %{
13664 rFlagsReg cr;
13665 compI_reg_imm0(cr, src);
13666 cmovI_reg_imm1_le(dst, src, cr);
13667 %}
13668 %}
13669
13670 // This pattern is automatically generated from aarch64_ad.m4
13671 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13672 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13673 %{
13674 match(Set dst (MinI imm src));
13675 ins_cost(INSN_COST * 3);
13676 expand %{
13677 rFlagsReg cr;
13678 compI_reg_imm0(cr, src);
13679 cmovI_reg_imm1_le(dst, src, cr);
13680 %}
13681 %}
13682
13683 // This pattern is automatically generated from aarch64_ad.m4
13684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13685 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13686 %{
13687 match(Set dst (MinI src imm));
13688 ins_cost(INSN_COST * 3);
13689 expand %{
13690 rFlagsReg cr;
13691 compI_reg_imm0(cr, src);
13692 cmovI_reg_immM1_lt(dst, src, cr);
13693 %}
13694 %}
13695
13696 // This pattern is automatically generated from aarch64_ad.m4
13697 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13698 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13699 %{
13700 match(Set dst (MinI imm src));
13701 ins_cost(INSN_COST * 3);
13702 expand %{
13703 rFlagsReg cr;
13704 compI_reg_imm0(cr, src);
13705 cmovI_reg_immM1_lt(dst, src, cr);
13706 %}
13707 %}
13708
13709 // This pattern is automatically generated from aarch64_ad.m4
13710 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13711 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13712 %{
13713 match(Set dst (MaxI src imm));
13714 ins_cost(INSN_COST * 3);
13715 expand %{
13716 rFlagsReg cr;
13717 compI_reg_imm0(cr, src);
13718 cmovI_reg_imm0_gt(dst, src, cr);
13719 %}
13720 %}
13721
13722 // This pattern is automatically generated from aarch64_ad.m4
13723 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13724 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13725 %{
13726 match(Set dst (MaxI imm src));
13727 ins_cost(INSN_COST * 3);
13728 expand %{
13729 rFlagsReg cr;
13730 compI_reg_imm0(cr, src);
13731 cmovI_reg_imm0_gt(dst, src, cr);
13732 %}
13733 %}
13734
13735 // This pattern is automatically generated from aarch64_ad.m4
13736 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13737 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13738 %{
13739 match(Set dst (MaxI src imm));
13740 ins_cost(INSN_COST * 3);
13741 expand %{
13742 rFlagsReg cr;
13743 compI_reg_imm0(cr, src);
13744 cmovI_reg_imm1_gt(dst, src, cr);
13745 %}
13746 %}
13747
13748 // This pattern is automatically generated from aarch64_ad.m4
13749 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13750 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13751 %{
13752 match(Set dst (MaxI imm src));
13753 ins_cost(INSN_COST * 3);
13754 expand %{
13755 rFlagsReg cr;
13756 compI_reg_imm0(cr, src);
13757 cmovI_reg_imm1_gt(dst, src, cr);
13758 %}
13759 %}
13760
13761 // This pattern is automatically generated from aarch64_ad.m4
13762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13763 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13764 %{
13765 match(Set dst (MaxI src imm));
13766 ins_cost(INSN_COST * 3);
13767 expand %{
13768 rFlagsReg cr;
13769 compI_reg_imm0(cr, src);
13770 cmovI_reg_immM1_ge(dst, src, cr);
13771 %}
13772 %}
13773
13774 // This pattern is automatically generated from aarch64_ad.m4
13775 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13776 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13777 %{
13778 match(Set dst (MaxI imm src));
13779 ins_cost(INSN_COST * 3);
13780 expand %{
13781 rFlagsReg cr;
13782 compI_reg_imm0(cr, src);
13783 cmovI_reg_immM1_ge(dst, src, cr);
13784 %}
13785 %}
13786
13787 // This pattern is automatically generated from aarch64_ad.m4
13788 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13789 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13790 %{
13791 match(Set dst (ReverseI src));
13792 ins_cost(INSN_COST);
13793 format %{ "rbitw $dst, $src" %}
13794 ins_encode %{
13795 __ rbitw($dst$$Register, $src$$Register);
13796 %}
13797 ins_pipe(ialu_reg);
13798 %}
13799
13800 // This pattern is automatically generated from aarch64_ad.m4
13801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13802 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13803 %{
13804 match(Set dst (ReverseL src));
13805 ins_cost(INSN_COST);
13806 format %{ "rbit $dst, $src" %}
13807 ins_encode %{
13808 __ rbit($dst$$Register, $src$$Register);
13809 %}
13810 ins_pipe(ialu_reg);
13811 %}
13812
13813
13814 // END This section of the file is automatically generated. Do not edit --------------
13815
13816
13817 // ============================================================================
13818 // Floating Point Arithmetic Instructions
13819
13820 instruct addHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13821 match(Set dst (AddHF src1 src2));
13822 format %{ "faddh $dst, $src1, $src2" %}
13823 ins_encode %{
13824 __ faddh($dst$$FloatRegister,
13825 $src1$$FloatRegister,
13826 $src2$$FloatRegister);
13827 %}
13828 ins_pipe(fp_dop_reg_reg_s);
13829 %}
13830
13831 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13832 match(Set dst (AddF src1 src2));
13833
13834 ins_cost(INSN_COST * 5);
13835 format %{ "fadds $dst, $src1, $src2" %}
13836
13837 ins_encode %{
13838 __ fadds(as_FloatRegister($dst$$reg),
13839 as_FloatRegister($src1$$reg),
13840 as_FloatRegister($src2$$reg));
13841 %}
13842
13843 ins_pipe(fp_dop_reg_reg_s);
13844 %}
13845
13846 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13847 match(Set dst (AddD src1 src2));
13848
13849 ins_cost(INSN_COST * 5);
13850 format %{ "faddd $dst, $src1, $src2" %}
13851
13852 ins_encode %{
13853 __ faddd(as_FloatRegister($dst$$reg),
13854 as_FloatRegister($src1$$reg),
13855 as_FloatRegister($src2$$reg));
13856 %}
13857
13858 ins_pipe(fp_dop_reg_reg_d);
13859 %}
13860
13861 instruct subHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13862 match(Set dst (SubHF src1 src2));
13863 format %{ "fsubh $dst, $src1, $src2" %}
13864 ins_encode %{
13865 __ fsubh($dst$$FloatRegister,
13866 $src1$$FloatRegister,
13867 $src2$$FloatRegister);
13868 %}
13869 ins_pipe(fp_dop_reg_reg_s);
13870 %}
13871
13872 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13873 match(Set dst (SubF src1 src2));
13874
13875 ins_cost(INSN_COST * 5);
13876 format %{ "fsubs $dst, $src1, $src2" %}
13877
13878 ins_encode %{
13879 __ fsubs(as_FloatRegister($dst$$reg),
13880 as_FloatRegister($src1$$reg),
13881 as_FloatRegister($src2$$reg));
13882 %}
13883
13884 ins_pipe(fp_dop_reg_reg_s);
13885 %}
13886
13887 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13888 match(Set dst (SubD src1 src2));
13889
13890 ins_cost(INSN_COST * 5);
13891 format %{ "fsubd $dst, $src1, $src2" %}
13892
13893 ins_encode %{
13894 __ fsubd(as_FloatRegister($dst$$reg),
13895 as_FloatRegister($src1$$reg),
13896 as_FloatRegister($src2$$reg));
13897 %}
13898
13899 ins_pipe(fp_dop_reg_reg_d);
13900 %}
13901
13902 instruct mulHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13903 match(Set dst (MulHF src1 src2));
13904 format %{ "fmulh $dst, $src1, $src2" %}
13905 ins_encode %{
13906 __ fmulh($dst$$FloatRegister,
13907 $src1$$FloatRegister,
13908 $src2$$FloatRegister);
13909 %}
13910 ins_pipe(fp_dop_reg_reg_s);
13911 %}
13912
13913 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13914 match(Set dst (MulF src1 src2));
13915
13916 ins_cost(INSN_COST * 6);
13917 format %{ "fmuls $dst, $src1, $src2" %}
13918
13919 ins_encode %{
13920 __ fmuls(as_FloatRegister($dst$$reg),
13921 as_FloatRegister($src1$$reg),
13922 as_FloatRegister($src2$$reg));
13923 %}
13924
13925 ins_pipe(fp_dop_reg_reg_s);
13926 %}
13927
13928 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13929 match(Set dst (MulD src1 src2));
13930
13931 ins_cost(INSN_COST * 6);
13932 format %{ "fmuld $dst, $src1, $src2" %}
13933
13934 ins_encode %{
13935 __ fmuld(as_FloatRegister($dst$$reg),
13936 as_FloatRegister($src1$$reg),
13937 as_FloatRegister($src2$$reg));
13938 %}
13939
13940 ins_pipe(fp_dop_reg_reg_d);
13941 %}
13942
13943 // src1 * src2 + src3 (half-precision float)
13944 instruct maddHF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13945 match(Set dst (FmaHF src3 (Binary src1 src2)));
13946 format %{ "fmaddh $dst, $src1, $src2, $src3" %}
13947 ins_encode %{
13948 assert(UseFMA, "Needs FMA instructions support.");
13949 __ fmaddh($dst$$FloatRegister,
13950 $src1$$FloatRegister,
13951 $src2$$FloatRegister,
13952 $src3$$FloatRegister);
13953 %}
13954 ins_pipe(pipe_class_default);
13955 %}
13956
13957 // src1 * src2 + src3
13958 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13959 match(Set dst (FmaF src3 (Binary src1 src2)));
13960
13961 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13962
13963 ins_encode %{
13964 assert(UseFMA, "Needs FMA instructions support.");
13965 __ fmadds(as_FloatRegister($dst$$reg),
13966 as_FloatRegister($src1$$reg),
13967 as_FloatRegister($src2$$reg),
13968 as_FloatRegister($src3$$reg));
13969 %}
13970
13971 ins_pipe(pipe_class_default);
13972 %}
13973
13974 // src1 * src2 + src3
13975 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13976 match(Set dst (FmaD src3 (Binary src1 src2)));
13977
13978 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13979
13980 ins_encode %{
13981 assert(UseFMA, "Needs FMA instructions support.");
13982 __ fmaddd(as_FloatRegister($dst$$reg),
13983 as_FloatRegister($src1$$reg),
13984 as_FloatRegister($src2$$reg),
13985 as_FloatRegister($src3$$reg));
13986 %}
13987
13988 ins_pipe(pipe_class_default);
13989 %}
13990
13991 // src1 * (-src2) + src3
13992 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13993 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13994 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13995
13996 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13997
13998 ins_encode %{
13999 assert(UseFMA, "Needs FMA instructions support.");
14000 __ fmsubs(as_FloatRegister($dst$$reg),
14001 as_FloatRegister($src1$$reg),
14002 as_FloatRegister($src2$$reg),
14003 as_FloatRegister($src3$$reg));
14004 %}
14005
14006 ins_pipe(pipe_class_default);
14007 %}
14008
14009 // src1 * (-src2) + src3
14010 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
14011 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14012 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
14013
14014 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
14015
14016 ins_encode %{
14017 assert(UseFMA, "Needs FMA instructions support.");
14018 __ fmsubd(as_FloatRegister($dst$$reg),
14019 as_FloatRegister($src1$$reg),
14020 as_FloatRegister($src2$$reg),
14021 as_FloatRegister($src3$$reg));
14022 %}
14023
14024 ins_pipe(pipe_class_default);
14025 %}
14026
14027 // src1 * (-src2) - src3
14028 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14029 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
14030 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
14031
14032 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
14033
14034 ins_encode %{
14035 assert(UseFMA, "Needs FMA instructions support.");
14036 __ fnmadds(as_FloatRegister($dst$$reg),
14037 as_FloatRegister($src1$$reg),
14038 as_FloatRegister($src2$$reg),
14039 as_FloatRegister($src3$$reg));
14040 %}
14041
14042 ins_pipe(pipe_class_default);
14043 %}
14044
14045 // src1 * (-src2) - src3
14046 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
14047 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
14048 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
14049
14050 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
14051
14052 ins_encode %{
14053 assert(UseFMA, "Needs FMA instructions support.");
14054 __ fnmaddd(as_FloatRegister($dst$$reg),
14055 as_FloatRegister($src1$$reg),
14056 as_FloatRegister($src2$$reg),
14057 as_FloatRegister($src3$$reg));
14058 %}
14059
14060 ins_pipe(pipe_class_default);
14061 %}
14062
14063 // src1 * src2 - src3
14064 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
14065 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
14066
14067 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
14068
14069 ins_encode %{
14070 assert(UseFMA, "Needs FMA instructions support.");
14071 __ fnmsubs(as_FloatRegister($dst$$reg),
14072 as_FloatRegister($src1$$reg),
14073 as_FloatRegister($src2$$reg),
14074 as_FloatRegister($src3$$reg));
14075 %}
14076
14077 ins_pipe(pipe_class_default);
14078 %}
14079
14080 // src1 * src2 - src3
14081 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
14082 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
14083
14084 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
14085
14086 ins_encode %{
14087 assert(UseFMA, "Needs FMA instructions support.");
14088 // n.b. insn name should be fnmsubd
14089 __ fnmsub(as_FloatRegister($dst$$reg),
14090 as_FloatRegister($src1$$reg),
14091 as_FloatRegister($src2$$reg),
14092 as_FloatRegister($src3$$reg));
14093 %}
14094
14095 ins_pipe(pipe_class_default);
14096 %}
14097
14098 // Math.max(HH)H (half-precision float)
14099 instruct maxHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14100 match(Set dst (MaxHF src1 src2));
14101 format %{ "fmaxh $dst, $src1, $src2" %}
14102 ins_encode %{
14103 __ fmaxh($dst$$FloatRegister,
14104 $src1$$FloatRegister,
14105 $src2$$FloatRegister);
14106 %}
14107 ins_pipe(fp_dop_reg_reg_s);
14108 %}
14109
14110 // Math.min(HH)H (half-precision float)
14111 instruct minHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14112 match(Set dst (MinHF src1 src2));
14113 format %{ "fminh $dst, $src1, $src2" %}
14114 ins_encode %{
14115 __ fminh($dst$$FloatRegister,
14116 $src1$$FloatRegister,
14117 $src2$$FloatRegister);
14118 %}
14119 ins_pipe(fp_dop_reg_reg_s);
14120 %}
14121
14122 // Math.max(FF)F
14123 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14124 match(Set dst (MaxF src1 src2));
14125
14126 format %{ "fmaxs $dst, $src1, $src2" %}
14127 ins_encode %{
14128 __ fmaxs(as_FloatRegister($dst$$reg),
14129 as_FloatRegister($src1$$reg),
14130 as_FloatRegister($src2$$reg));
14131 %}
14132
14133 ins_pipe(fp_dop_reg_reg_s);
14134 %}
14135
14136 // Math.min(FF)F
14137 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14138 match(Set dst (MinF src1 src2));
14139
14140 format %{ "fmins $dst, $src1, $src2" %}
14141 ins_encode %{
14142 __ fmins(as_FloatRegister($dst$$reg),
14143 as_FloatRegister($src1$$reg),
14144 as_FloatRegister($src2$$reg));
14145 %}
14146
14147 ins_pipe(fp_dop_reg_reg_s);
14148 %}
14149
14150 // Math.max(DD)D
14151 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14152 match(Set dst (MaxD src1 src2));
14153
14154 format %{ "fmaxd $dst, $src1, $src2" %}
14155 ins_encode %{
14156 __ fmaxd(as_FloatRegister($dst$$reg),
14157 as_FloatRegister($src1$$reg),
14158 as_FloatRegister($src2$$reg));
14159 %}
14160
14161 ins_pipe(fp_dop_reg_reg_d);
14162 %}
14163
14164 // Math.min(DD)D
14165 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14166 match(Set dst (MinD src1 src2));
14167
14168 format %{ "fmind $dst, $src1, $src2" %}
14169 ins_encode %{
14170 __ fmind(as_FloatRegister($dst$$reg),
14171 as_FloatRegister($src1$$reg),
14172 as_FloatRegister($src2$$reg));
14173 %}
14174
14175 ins_pipe(fp_dop_reg_reg_d);
14176 %}
14177
14178 instruct divHF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14179 match(Set dst (DivHF src1 src2));
14180 format %{ "fdivh $dst, $src1, $src2" %}
14181 ins_encode %{
14182 __ fdivh($dst$$FloatRegister,
14183 $src1$$FloatRegister,
14184 $src2$$FloatRegister);
14185 %}
14186 ins_pipe(fp_div_s);
14187 %}
14188
14189 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
14190 match(Set dst (DivF src1 src2));
14191
14192 ins_cost(INSN_COST * 18);
14193 format %{ "fdivs $dst, $src1, $src2" %}
14194
14195 ins_encode %{
14196 __ fdivs(as_FloatRegister($dst$$reg),
14197 as_FloatRegister($src1$$reg),
14198 as_FloatRegister($src2$$reg));
14199 %}
14200
14201 ins_pipe(fp_div_s);
14202 %}
14203
14204 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
14205 match(Set dst (DivD src1 src2));
14206
14207 ins_cost(INSN_COST * 32);
14208 format %{ "fdivd $dst, $src1, $src2" %}
14209
14210 ins_encode %{
14211 __ fdivd(as_FloatRegister($dst$$reg),
14212 as_FloatRegister($src1$$reg),
14213 as_FloatRegister($src2$$reg));
14214 %}
14215
14216 ins_pipe(fp_div_d);
14217 %}
14218
14219 instruct negF_reg_reg(vRegF dst, vRegF src) %{
14220 match(Set dst (NegF src));
14221
14222 ins_cost(INSN_COST * 3);
14223 format %{ "fneg $dst, $src" %}
14224
14225 ins_encode %{
14226 __ fnegs(as_FloatRegister($dst$$reg),
14227 as_FloatRegister($src$$reg));
14228 %}
14229
14230 ins_pipe(fp_uop_s);
14231 %}
14232
14233 instruct negD_reg_reg(vRegD dst, vRegD src) %{
14234 match(Set dst (NegD src));
14235
14236 ins_cost(INSN_COST * 3);
14237 format %{ "fnegd $dst, $src" %}
14238
14239 ins_encode %{
14240 __ fnegd(as_FloatRegister($dst$$reg),
14241 as_FloatRegister($src$$reg));
14242 %}
14243
14244 ins_pipe(fp_uop_d);
14245 %}
14246
14247 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
14248 %{
14249 match(Set dst (AbsI src));
14250
14251 effect(KILL cr);
14252 ins_cost(INSN_COST * 2);
14253 format %{ "cmpw $src, zr\n\t"
14254 "cnegw $dst, $src, Assembler::LT\t# int abs"
14255 %}
14256
14257 ins_encode %{
14258 __ cmpw(as_Register($src$$reg), zr);
14259 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14260 %}
14261 ins_pipe(pipe_class_default);
14262 %}
14263
14264 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
14265 %{
14266 match(Set dst (AbsL src));
14267
14268 effect(KILL cr);
14269 ins_cost(INSN_COST * 2);
14270 format %{ "cmp $src, zr\n\t"
14271 "cneg $dst, $src, Assembler::LT\t# long abs"
14272 %}
14273
14274 ins_encode %{
14275 __ cmp(as_Register($src$$reg), zr);
14276 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
14277 %}
14278 ins_pipe(pipe_class_default);
14279 %}
14280
14281 instruct absF_reg(vRegF dst, vRegF src) %{
14282 match(Set dst (AbsF src));
14283
14284 ins_cost(INSN_COST * 3);
14285 format %{ "fabss $dst, $src" %}
14286 ins_encode %{
14287 __ fabss(as_FloatRegister($dst$$reg),
14288 as_FloatRegister($src$$reg));
14289 %}
14290
14291 ins_pipe(fp_uop_s);
14292 %}
14293
14294 instruct absD_reg(vRegD dst, vRegD src) %{
14295 match(Set dst (AbsD src));
14296
14297 ins_cost(INSN_COST * 3);
14298 format %{ "fabsd $dst, $src" %}
14299 ins_encode %{
14300 __ fabsd(as_FloatRegister($dst$$reg),
14301 as_FloatRegister($src$$reg));
14302 %}
14303
14304 ins_pipe(fp_uop_d);
14305 %}
14306
14307 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14308 match(Set dst (AbsF (SubF src1 src2)));
14309
14310 ins_cost(INSN_COST * 3);
14311 format %{ "fabds $dst, $src1, $src2" %}
14312 ins_encode %{
14313 __ fabds(as_FloatRegister($dst$$reg),
14314 as_FloatRegister($src1$$reg),
14315 as_FloatRegister($src2$$reg));
14316 %}
14317
14318 ins_pipe(fp_uop_s);
14319 %}
14320
14321 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
14322 match(Set dst (AbsD (SubD src1 src2)));
14323
14324 ins_cost(INSN_COST * 3);
14325 format %{ "fabdd $dst, $src1, $src2" %}
14326 ins_encode %{
14327 __ fabdd(as_FloatRegister($dst$$reg),
14328 as_FloatRegister($src1$$reg),
14329 as_FloatRegister($src2$$reg));
14330 %}
14331
14332 ins_pipe(fp_uop_d);
14333 %}
14334
14335 instruct sqrtD_reg(vRegD dst, vRegD src) %{
14336 match(Set dst (SqrtD src));
14337
14338 ins_cost(INSN_COST * 50);
14339 format %{ "fsqrtd $dst, $src" %}
14340 ins_encode %{
14341 __ fsqrtd(as_FloatRegister($dst$$reg),
14342 as_FloatRegister($src$$reg));
14343 %}
14344
14345 ins_pipe(fp_div_s);
14346 %}
14347
14348 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14349 match(Set dst (SqrtF src));
14350
14351 ins_cost(INSN_COST * 50);
14352 format %{ "fsqrts $dst, $src" %}
14353 ins_encode %{
14354 __ fsqrts(as_FloatRegister($dst$$reg),
14355 as_FloatRegister($src$$reg));
14356 %}
14357
14358 ins_pipe(fp_div_d);
14359 %}
14360
14361 instruct sqrtHF_reg(vRegF dst, vRegF src) %{
14362 match(Set dst (SqrtHF src));
14363 format %{ "fsqrth $dst, $src" %}
14364 ins_encode %{
14365 __ fsqrth($dst$$FloatRegister,
14366 $src$$FloatRegister);
14367 %}
14368 ins_pipe(fp_div_s);
14369 %}
14370
14371 // Math.rint, floor, ceil
14372 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14373 match(Set dst (RoundDoubleMode src rmode));
14374 format %{ "frint $dst, $src, $rmode" %}
14375 ins_encode %{
14376 switch ($rmode$$constant) {
14377 case RoundDoubleModeNode::rmode_rint:
14378 __ frintnd(as_FloatRegister($dst$$reg),
14379 as_FloatRegister($src$$reg));
14380 break;
14381 case RoundDoubleModeNode::rmode_floor:
14382 __ frintmd(as_FloatRegister($dst$$reg),
14383 as_FloatRegister($src$$reg));
14384 break;
14385 case RoundDoubleModeNode::rmode_ceil:
14386 __ frintpd(as_FloatRegister($dst$$reg),
14387 as_FloatRegister($src$$reg));
14388 break;
14389 }
14390 %}
14391 ins_pipe(fp_uop_d);
14392 %}
14393
14394 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14395 match(Set dst (CopySignD src1 (Binary src2 zero)));
14396 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14397 format %{ "CopySignD $dst $src1 $src2" %}
14398 ins_encode %{
14399 FloatRegister dst = as_FloatRegister($dst$$reg),
14400 src1 = as_FloatRegister($src1$$reg),
14401 src2 = as_FloatRegister($src2$$reg),
14402 zero = as_FloatRegister($zero$$reg);
14403 __ fnegd(dst, zero);
14404 __ bsl(dst, __ T8B, src2, src1);
14405 %}
14406 ins_pipe(fp_uop_d);
14407 %}
14408
14409 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14410 match(Set dst (CopySignF src1 src2));
14411 effect(TEMP_DEF dst, USE src1, USE src2);
14412 format %{ "CopySignF $dst $src1 $src2" %}
14413 ins_encode %{
14414 FloatRegister dst = as_FloatRegister($dst$$reg),
14415 src1 = as_FloatRegister($src1$$reg),
14416 src2 = as_FloatRegister($src2$$reg);
14417 __ movi(dst, __ T2S, 0x80, 24);
14418 __ bsl(dst, __ T8B, src2, src1);
14419 %}
14420 ins_pipe(fp_uop_d);
14421 %}
14422
14423 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14424 match(Set dst (SignumD src (Binary zero one)));
14425 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14426 format %{ "signumD $dst, $src" %}
14427 ins_encode %{
14428 FloatRegister src = as_FloatRegister($src$$reg),
14429 dst = as_FloatRegister($dst$$reg),
14430 zero = as_FloatRegister($zero$$reg),
14431 one = as_FloatRegister($one$$reg);
14432 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14433 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14434 // Bit selection instruction gets bit from "one" for each enabled bit in
14435 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14436 // NaN the whole "src" will be copied because "dst" is zero. For all other
14437 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14438 // from "src", and all other bits are copied from 1.0.
14439 __ bsl(dst, __ T8B, one, src);
14440 %}
14441 ins_pipe(fp_uop_d);
14442 %}
14443
14444 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14445 match(Set dst (SignumF src (Binary zero one)));
14446 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14447 format %{ "signumF $dst, $src" %}
14448 ins_encode %{
14449 FloatRegister src = as_FloatRegister($src$$reg),
14450 dst = as_FloatRegister($dst$$reg),
14451 zero = as_FloatRegister($zero$$reg),
14452 one = as_FloatRegister($one$$reg);
14453 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14454 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14455 // Bit selection instruction gets bit from "one" for each enabled bit in
14456 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14457 // NaN the whole "src" will be copied because "dst" is zero. For all other
14458 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14459 // from "src", and all other bits are copied from 1.0.
14460 __ bsl(dst, __ T8B, one, src);
14461 %}
14462 ins_pipe(fp_uop_d);
14463 %}
14464
14465 instruct onspinwait() %{
14466 match(OnSpinWait);
14467 ins_cost(INSN_COST);
14468
14469 format %{ "onspinwait" %}
14470
14471 ins_encode %{
14472 __ spin_wait();
14473 %}
14474 ins_pipe(pipe_class_empty);
14475 %}
14476
14477 // ============================================================================
14478 // Logical Instructions
14479
14480 // Integer Logical Instructions
14481
14482 // And Instructions
14483
14484
14485 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14486 match(Set dst (AndI src1 src2));
14487
14488 format %{ "andw $dst, $src1, $src2\t# int" %}
14489
14490 ins_cost(INSN_COST);
14491 ins_encode %{
14492 __ andw(as_Register($dst$$reg),
14493 as_Register($src1$$reg),
14494 as_Register($src2$$reg));
14495 %}
14496
14497 ins_pipe(ialu_reg_reg);
14498 %}
14499
14500 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14501 match(Set dst (AndI src1 src2));
14502
14503 format %{ "andsw $dst, $src1, $src2\t# int" %}
14504
14505 ins_cost(INSN_COST);
14506 ins_encode %{
14507 __ andw(as_Register($dst$$reg),
14508 as_Register($src1$$reg),
14509 (uint64_t)($src2$$constant));
14510 %}
14511
14512 ins_pipe(ialu_reg_imm);
14513 %}
14514
14515 // Or Instructions
14516
14517 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14518 match(Set dst (OrI src1 src2));
14519
14520 format %{ "orrw $dst, $src1, $src2\t# int" %}
14521
14522 ins_cost(INSN_COST);
14523 ins_encode %{
14524 __ orrw(as_Register($dst$$reg),
14525 as_Register($src1$$reg),
14526 as_Register($src2$$reg));
14527 %}
14528
14529 ins_pipe(ialu_reg_reg);
14530 %}
14531
14532 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14533 match(Set dst (OrI src1 src2));
14534
14535 format %{ "orrw $dst, $src1, $src2\t# int" %}
14536
14537 ins_cost(INSN_COST);
14538 ins_encode %{
14539 __ orrw(as_Register($dst$$reg),
14540 as_Register($src1$$reg),
14541 (uint64_t)($src2$$constant));
14542 %}
14543
14544 ins_pipe(ialu_reg_imm);
14545 %}
14546
14547 // Xor Instructions
14548
14549 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14550 match(Set dst (XorI src1 src2));
14551
14552 format %{ "eorw $dst, $src1, $src2\t# int" %}
14553
14554 ins_cost(INSN_COST);
14555 ins_encode %{
14556 __ eorw(as_Register($dst$$reg),
14557 as_Register($src1$$reg),
14558 as_Register($src2$$reg));
14559 %}
14560
14561 ins_pipe(ialu_reg_reg);
14562 %}
14563
14564 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14565 match(Set dst (XorI src1 src2));
14566
14567 format %{ "eorw $dst, $src1, $src2\t# int" %}
14568
14569 ins_cost(INSN_COST);
14570 ins_encode %{
14571 __ eorw(as_Register($dst$$reg),
14572 as_Register($src1$$reg),
14573 (uint64_t)($src2$$constant));
14574 %}
14575
14576 ins_pipe(ialu_reg_imm);
14577 %}
14578
14579 // Long Logical Instructions
14580 // TODO
14581
14582 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14583 match(Set dst (AndL src1 src2));
14584
14585 format %{ "and $dst, $src1, $src2\t# int" %}
14586
14587 ins_cost(INSN_COST);
14588 ins_encode %{
14589 __ andr(as_Register($dst$$reg),
14590 as_Register($src1$$reg),
14591 as_Register($src2$$reg));
14592 %}
14593
14594 ins_pipe(ialu_reg_reg);
14595 %}
14596
14597 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14598 match(Set dst (AndL src1 src2));
14599
14600 format %{ "and $dst, $src1, $src2\t# int" %}
14601
14602 ins_cost(INSN_COST);
14603 ins_encode %{
14604 __ andr(as_Register($dst$$reg),
14605 as_Register($src1$$reg),
14606 (uint64_t)($src2$$constant));
14607 %}
14608
14609 ins_pipe(ialu_reg_imm);
14610 %}
14611
14612 // Or Instructions
14613
14614 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14615 match(Set dst (OrL src1 src2));
14616
14617 format %{ "orr $dst, $src1, $src2\t# int" %}
14618
14619 ins_cost(INSN_COST);
14620 ins_encode %{
14621 __ orr(as_Register($dst$$reg),
14622 as_Register($src1$$reg),
14623 as_Register($src2$$reg));
14624 %}
14625
14626 ins_pipe(ialu_reg_reg);
14627 %}
14628
14629 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14630 match(Set dst (OrL src1 src2));
14631
14632 format %{ "orr $dst, $src1, $src2\t# int" %}
14633
14634 ins_cost(INSN_COST);
14635 ins_encode %{
14636 __ orr(as_Register($dst$$reg),
14637 as_Register($src1$$reg),
14638 (uint64_t)($src2$$constant));
14639 %}
14640
14641 ins_pipe(ialu_reg_imm);
14642 %}
14643
14644 // Xor Instructions
14645
14646 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14647 match(Set dst (XorL src1 src2));
14648
14649 format %{ "eor $dst, $src1, $src2\t# int" %}
14650
14651 ins_cost(INSN_COST);
14652 ins_encode %{
14653 __ eor(as_Register($dst$$reg),
14654 as_Register($src1$$reg),
14655 as_Register($src2$$reg));
14656 %}
14657
14658 ins_pipe(ialu_reg_reg);
14659 %}
14660
14661 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14662 match(Set dst (XorL src1 src2));
14663
14664 ins_cost(INSN_COST);
14665 format %{ "eor $dst, $src1, $src2\t# int" %}
14666
14667 ins_encode %{
14668 __ eor(as_Register($dst$$reg),
14669 as_Register($src1$$reg),
14670 (uint64_t)($src2$$constant));
14671 %}
14672
14673 ins_pipe(ialu_reg_imm);
14674 %}
14675
14676 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14677 %{
14678 match(Set dst (ConvI2L src));
14679
14680 ins_cost(INSN_COST);
14681 format %{ "sxtw $dst, $src\t# i2l" %}
14682 ins_encode %{
14683 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14684 %}
14685 ins_pipe(ialu_reg_shift);
14686 %}
14687
14688 // this pattern occurs in bigmath arithmetic
14689 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14690 %{
14691 match(Set dst (AndL (ConvI2L src) mask));
14692
14693 ins_cost(INSN_COST);
14694 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14695 ins_encode %{
14696 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14697 %}
14698
14699 ins_pipe(ialu_reg_shift);
14700 %}
14701
14702 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14703 match(Set dst (ConvL2I src));
14704
14705 ins_cost(INSN_COST);
14706 format %{ "movw $dst, $src \t// l2i" %}
14707
14708 ins_encode %{
14709 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14710 %}
14711
14712 ins_pipe(ialu_reg);
14713 %}
14714
14715 instruct convD2F_reg(vRegF dst, vRegD src) %{
14716 match(Set dst (ConvD2F src));
14717
14718 ins_cost(INSN_COST * 5);
14719 format %{ "fcvtd $dst, $src \t// d2f" %}
14720
14721 ins_encode %{
14722 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14723 %}
14724
14725 ins_pipe(fp_d2f);
14726 %}
14727
14728 instruct convF2D_reg(vRegD dst, vRegF src) %{
14729 match(Set dst (ConvF2D src));
14730
14731 ins_cost(INSN_COST * 5);
14732 format %{ "fcvts $dst, $src \t// f2d" %}
14733
14734 ins_encode %{
14735 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14736 %}
14737
14738 ins_pipe(fp_f2d);
14739 %}
14740
14741 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14742 match(Set dst (ConvF2I src));
14743
14744 ins_cost(INSN_COST * 5);
14745 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14746
14747 ins_encode %{
14748 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14749 %}
14750
14751 ins_pipe(fp_f2i);
14752 %}
14753
14754 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14755 match(Set dst (ConvF2L src));
14756
14757 ins_cost(INSN_COST * 5);
14758 format %{ "fcvtzs $dst, $src \t// f2l" %}
14759
14760 ins_encode %{
14761 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14762 %}
14763
14764 ins_pipe(fp_f2l);
14765 %}
14766
14767 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14768 match(Set dst (ConvF2HF src));
14769 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14770 "smov $dst, $tmp\t# move result from $tmp to $dst"
14771 %}
14772 effect(TEMP tmp);
14773 ins_encode %{
14774 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14775 %}
14776 ins_pipe(pipe_slow);
14777 %}
14778
14779 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14780 match(Set dst (ConvHF2F src));
14781 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14782 "fcvt $dst, $tmp\t# convert half to single precision"
14783 %}
14784 effect(TEMP tmp);
14785 ins_encode %{
14786 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14787 %}
14788 ins_pipe(pipe_slow);
14789 %}
14790
14791 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14792 match(Set dst (ConvI2F src));
14793
14794 ins_cost(INSN_COST * 5);
14795 format %{ "scvtfws $dst, $src \t// i2f" %}
14796
14797 ins_encode %{
14798 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14799 %}
14800
14801 ins_pipe(fp_i2f);
14802 %}
14803
14804 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14805 match(Set dst (ConvL2F src));
14806
14807 ins_cost(INSN_COST * 5);
14808 format %{ "scvtfs $dst, $src \t// l2f" %}
14809
14810 ins_encode %{
14811 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14812 %}
14813
14814 ins_pipe(fp_l2f);
14815 %}
14816
14817 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14818 match(Set dst (ConvD2I src));
14819
14820 ins_cost(INSN_COST * 5);
14821 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14822
14823 ins_encode %{
14824 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14825 %}
14826
14827 ins_pipe(fp_d2i);
14828 %}
14829
14830 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14831 match(Set dst (ConvD2L src));
14832
14833 ins_cost(INSN_COST * 5);
14834 format %{ "fcvtzd $dst, $src \t// d2l" %}
14835
14836 ins_encode %{
14837 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14838 %}
14839
14840 ins_pipe(fp_d2l);
14841 %}
14842
14843 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14844 match(Set dst (ConvI2D src));
14845
14846 ins_cost(INSN_COST * 5);
14847 format %{ "scvtfwd $dst, $src \t// i2d" %}
14848
14849 ins_encode %{
14850 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14851 %}
14852
14853 ins_pipe(fp_i2d);
14854 %}
14855
14856 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14857 match(Set dst (ConvL2D src));
14858
14859 ins_cost(INSN_COST * 5);
14860 format %{ "scvtfd $dst, $src \t// l2d" %}
14861
14862 ins_encode %{
14863 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14864 %}
14865
14866 ins_pipe(fp_l2d);
14867 %}
14868
14869 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14870 %{
14871 match(Set dst (RoundD src));
14872 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14873 format %{ "java_round_double $dst,$src"%}
14874 ins_encode %{
14875 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14876 as_FloatRegister($ftmp$$reg));
14877 %}
14878 ins_pipe(pipe_slow);
14879 %}
14880
14881 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14882 %{
14883 match(Set dst (RoundF src));
14884 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14885 format %{ "java_round_float $dst,$src"%}
14886 ins_encode %{
14887 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14888 as_FloatRegister($ftmp$$reg));
14889 %}
14890 ins_pipe(pipe_slow);
14891 %}
14892
14893 // stack <-> reg and reg <-> reg shuffles with no conversion
14894
14895 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14896
14897 match(Set dst (MoveF2I src));
14898
14899 effect(DEF dst, USE src);
14900
14901 ins_cost(4 * INSN_COST);
14902
14903 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14904
14905 ins_encode %{
14906 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14907 %}
14908
14909 ins_pipe(iload_reg_reg);
14910
14911 %}
14912
14913 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14914
14915 match(Set dst (MoveI2F src));
14916
14917 effect(DEF dst, USE src);
14918
14919 ins_cost(4 * INSN_COST);
14920
14921 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14922
14923 ins_encode %{
14924 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14925 %}
14926
14927 ins_pipe(pipe_class_memory);
14928
14929 %}
14930
14931 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14932
14933 match(Set dst (MoveD2L src));
14934
14935 effect(DEF dst, USE src);
14936
14937 ins_cost(4 * INSN_COST);
14938
14939 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14940
14941 ins_encode %{
14942 __ ldr($dst$$Register, Address(sp, $src$$disp));
14943 %}
14944
14945 ins_pipe(iload_reg_reg);
14946
14947 %}
14948
14949 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14950
14951 match(Set dst (MoveL2D src));
14952
14953 effect(DEF dst, USE src);
14954
14955 ins_cost(4 * INSN_COST);
14956
14957 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14958
14959 ins_encode %{
14960 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14961 %}
14962
14963 ins_pipe(pipe_class_memory);
14964
14965 %}
14966
14967 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14968
14969 match(Set dst (MoveF2I src));
14970
14971 effect(DEF dst, USE src);
14972
14973 ins_cost(INSN_COST);
14974
14975 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14976
14977 ins_encode %{
14978 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14979 %}
14980
14981 ins_pipe(pipe_class_memory);
14982
14983 %}
14984
14985 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14986
14987 match(Set dst (MoveI2F src));
14988
14989 effect(DEF dst, USE src);
14990
14991 ins_cost(INSN_COST);
14992
14993 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14994
14995 ins_encode %{
14996 __ strw($src$$Register, Address(sp, $dst$$disp));
14997 %}
14998
14999 ins_pipe(istore_reg_reg);
15000
15001 %}
15002
15003 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
15004
15005 match(Set dst (MoveD2L src));
15006
15007 effect(DEF dst, USE src);
15008
15009 ins_cost(INSN_COST);
15010
15011 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
15012
15013 ins_encode %{
15014 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
15015 %}
15016
15017 ins_pipe(pipe_class_memory);
15018
15019 %}
15020
15021 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
15022
15023 match(Set dst (MoveL2D src));
15024
15025 effect(DEF dst, USE src);
15026
15027 ins_cost(INSN_COST);
15028
15029 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
15030
15031 ins_encode %{
15032 __ str($src$$Register, Address(sp, $dst$$disp));
15033 %}
15034
15035 ins_pipe(istore_reg_reg);
15036
15037 %}
15038
15039 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
15040
15041 match(Set dst (MoveF2I src));
15042
15043 effect(DEF dst, USE src);
15044
15045 ins_cost(INSN_COST);
15046
15047 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
15048
15049 ins_encode %{
15050 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
15051 %}
15052
15053 ins_pipe(fp_f2i);
15054
15055 %}
15056
15057 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
15058
15059 match(Set dst (MoveI2F src));
15060
15061 effect(DEF dst, USE src);
15062
15063 ins_cost(INSN_COST);
15064
15065 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
15066
15067 ins_encode %{
15068 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
15069 %}
15070
15071 ins_pipe(fp_i2f);
15072
15073 %}
15074
15075 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
15076
15077 match(Set dst (MoveD2L src));
15078
15079 effect(DEF dst, USE src);
15080
15081 ins_cost(INSN_COST);
15082
15083 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
15084
15085 ins_encode %{
15086 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
15087 %}
15088
15089 ins_pipe(fp_d2l);
15090
15091 %}
15092
15093 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
15094
15095 match(Set dst (MoveL2D src));
15096
15097 effect(DEF dst, USE src);
15098
15099 ins_cost(INSN_COST);
15100
15101 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15102
15103 ins_encode %{
15104 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15105 %}
15106
15107 ins_pipe(fp_l2d);
15108
15109 %}
15110
15111 // ============================================================================
15112 // clearing of an array
15113
15114 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15115 %{
15116 match(Set dummy (ClearArray cnt base));
15117 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15118
15119 ins_cost(4 * INSN_COST);
15120 format %{ "ClearArray $cnt, $base" %}
15121
15122 ins_encode %{
15123 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15124 if (tpc == nullptr) {
15125 ciEnv::current()->record_failure("CodeCache is full");
15126 return;
15127 }
15128 %}
15129
15130 ins_pipe(pipe_class_memory);
15131 %}
15132
15133 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15134 %{
15135 predicate((uint64_t)n->in(2)->get_long()
15136 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15137 match(Set dummy (ClearArray cnt base));
15138 effect(TEMP temp, USE_KILL base, KILL cr);
15139
15140 ins_cost(4 * INSN_COST);
15141 format %{ "ClearArray $cnt, $base" %}
15142
15143 ins_encode %{
15144 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15145 if (tpc == nullptr) {
15146 ciEnv::current()->record_failure("CodeCache is full");
15147 return;
15148 }
15149 %}
15150
15151 ins_pipe(pipe_class_memory);
15152 %}
15153
15154 // ============================================================================
15155 // Overflow Math Instructions
15156
15157 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15158 %{
15159 match(Set cr (OverflowAddI op1 op2));
15160
15161 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15162 ins_cost(INSN_COST);
15163 ins_encode %{
15164 __ cmnw($op1$$Register, $op2$$Register);
15165 %}
15166
15167 ins_pipe(icmp_reg_reg);
15168 %}
15169
15170 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15171 %{
15172 match(Set cr (OverflowAddI op1 op2));
15173
15174 format %{ "cmnw $op1, $op2\t# overflow check int" %}
15175 ins_cost(INSN_COST);
15176 ins_encode %{
15177 __ cmnw($op1$$Register, $op2$$constant);
15178 %}
15179
15180 ins_pipe(icmp_reg_imm);
15181 %}
15182
15183 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15184 %{
15185 match(Set cr (OverflowAddL op1 op2));
15186
15187 format %{ "cmn $op1, $op2\t# overflow check long" %}
15188 ins_cost(INSN_COST);
15189 ins_encode %{
15190 __ cmn($op1$$Register, $op2$$Register);
15191 %}
15192
15193 ins_pipe(icmp_reg_reg);
15194 %}
15195
15196 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15197 %{
15198 match(Set cr (OverflowAddL op1 op2));
15199
15200 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
15201 ins_cost(INSN_COST);
15202 ins_encode %{
15203 __ adds(zr, $op1$$Register, $op2$$constant);
15204 %}
15205
15206 ins_pipe(icmp_reg_imm);
15207 %}
15208
15209 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15210 %{
15211 match(Set cr (OverflowSubI op1 op2));
15212
15213 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15214 ins_cost(INSN_COST);
15215 ins_encode %{
15216 __ cmpw($op1$$Register, $op2$$Register);
15217 %}
15218
15219 ins_pipe(icmp_reg_reg);
15220 %}
15221
15222 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
15223 %{
15224 match(Set cr (OverflowSubI op1 op2));
15225
15226 format %{ "cmpw $op1, $op2\t# overflow check int" %}
15227 ins_cost(INSN_COST);
15228 ins_encode %{
15229 __ cmpw($op1$$Register, $op2$$constant);
15230 %}
15231
15232 ins_pipe(icmp_reg_imm);
15233 %}
15234
15235 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15236 %{
15237 match(Set cr (OverflowSubL op1 op2));
15238
15239 format %{ "cmp $op1, $op2\t# overflow check long" %}
15240 ins_cost(INSN_COST);
15241 ins_encode %{
15242 __ cmp($op1$$Register, $op2$$Register);
15243 %}
15244
15245 ins_pipe(icmp_reg_reg);
15246 %}
15247
15248 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
15249 %{
15250 match(Set cr (OverflowSubL op1 op2));
15251
15252 format %{ "cmp $op1, $op2\t# overflow check long" %}
15253 ins_cost(INSN_COST);
15254 ins_encode %{
15255 __ subs(zr, $op1$$Register, $op2$$constant);
15256 %}
15257
15258 ins_pipe(icmp_reg_imm);
15259 %}
15260
15261 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
15262 %{
15263 match(Set cr (OverflowSubI zero op1));
15264
15265 format %{ "cmpw zr, $op1\t# overflow check int" %}
15266 ins_cost(INSN_COST);
15267 ins_encode %{
15268 __ cmpw(zr, $op1$$Register);
15269 %}
15270
15271 ins_pipe(icmp_reg_imm);
15272 %}
15273
15274 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
15275 %{
15276 match(Set cr (OverflowSubL zero op1));
15277
15278 format %{ "cmp zr, $op1\t# overflow check long" %}
15279 ins_cost(INSN_COST);
15280 ins_encode %{
15281 __ cmp(zr, $op1$$Register);
15282 %}
15283
15284 ins_pipe(icmp_reg_imm);
15285 %}
15286
15287 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15288 %{
15289 match(Set cr (OverflowMulI op1 op2));
15290
15291 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15292 "cmp rscratch1, rscratch1, sxtw\n\t"
15293 "movw rscratch1, #0x80000000\n\t"
15294 "cselw rscratch1, rscratch1, zr, NE\n\t"
15295 "cmpw rscratch1, #1" %}
15296 ins_cost(5 * INSN_COST);
15297 ins_encode %{
15298 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15299 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15300 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15301 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15302 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15303 %}
15304
15305 ins_pipe(pipe_slow);
15306 %}
15307
15308 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
15309 %{
15310 match(If cmp (OverflowMulI op1 op2));
15311 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15312 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15313 effect(USE labl, KILL cr);
15314
15315 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
15316 "cmp rscratch1, rscratch1, sxtw\n\t"
15317 "b$cmp $labl" %}
15318 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
15319 ins_encode %{
15320 Label* L = $labl$$label;
15321 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15322 __ smull(rscratch1, $op1$$Register, $op2$$Register);
15323 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
15324 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15325 %}
15326
15327 ins_pipe(pipe_serial);
15328 %}
15329
15330 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15331 %{
15332 match(Set cr (OverflowMulL op1 op2));
15333
15334 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15335 "smulh rscratch2, $op1, $op2\n\t"
15336 "cmp rscratch2, rscratch1, ASR #63\n\t"
15337 "movw rscratch1, #0x80000000\n\t"
15338 "cselw rscratch1, rscratch1, zr, NE\n\t"
15339 "cmpw rscratch1, #1" %}
15340 ins_cost(6 * INSN_COST);
15341 ins_encode %{
15342 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15343 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15344 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15345 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
15346 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
15347 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
15348 %}
15349
15350 ins_pipe(pipe_slow);
15351 %}
15352
15353 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
15354 %{
15355 match(If cmp (OverflowMulL op1 op2));
15356 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15357 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15358 effect(USE labl, KILL cr);
15359
15360 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15361 "smulh rscratch2, $op1, $op2\n\t"
15362 "cmp rscratch2, rscratch1, ASR #63\n\t"
15363 "b$cmp $labl" %}
15364 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15365 ins_encode %{
15366 Label* L = $labl$$label;
15367 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15368 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15369 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15370 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15371 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15372 %}
15373
15374 ins_pipe(pipe_serial);
15375 %}
15376
15377 // ============================================================================
15378 // Compare Instructions
15379
15380 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15381 %{
15382 match(Set cr (CmpI op1 op2));
15383
15384 effect(DEF cr, USE op1, USE op2);
15385
15386 ins_cost(INSN_COST);
15387 format %{ "cmpw $op1, $op2" %}
15388
15389 ins_encode(aarch64_enc_cmpw(op1, op2));
15390
15391 ins_pipe(icmp_reg_reg);
15392 %}
15393
15394 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15395 %{
15396 match(Set cr (CmpI op1 zero));
15397
15398 effect(DEF cr, USE op1);
15399
15400 ins_cost(INSN_COST);
15401 format %{ "cmpw $op1, 0" %}
15402
15403 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15404
15405 ins_pipe(icmp_reg_imm);
15406 %}
15407
15408 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15409 %{
15410 match(Set cr (CmpI op1 op2));
15411
15412 effect(DEF cr, USE op1);
15413
15414 ins_cost(INSN_COST);
15415 format %{ "cmpw $op1, $op2" %}
15416
15417 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15418
15419 ins_pipe(icmp_reg_imm);
15420 %}
15421
15422 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15423 %{
15424 match(Set cr (CmpI op1 op2));
15425
15426 effect(DEF cr, USE op1);
15427
15428 ins_cost(INSN_COST * 2);
15429 format %{ "cmpw $op1, $op2" %}
15430
15431 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15432
15433 ins_pipe(icmp_reg_imm);
15434 %}
15435
15436 // Unsigned compare Instructions; really, same as signed compare
15437 // except it should only be used to feed an If or a CMovI which takes a
15438 // cmpOpU.
15439
15440 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15441 %{
15442 match(Set cr (CmpU op1 op2));
15443
15444 effect(DEF cr, USE op1, USE op2);
15445
15446 ins_cost(INSN_COST);
15447 format %{ "cmpw $op1, $op2\t# unsigned" %}
15448
15449 ins_encode(aarch64_enc_cmpw(op1, op2));
15450
15451 ins_pipe(icmp_reg_reg);
15452 %}
15453
15454 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15455 %{
15456 match(Set cr (CmpU op1 zero));
15457
15458 effect(DEF cr, USE op1);
15459
15460 ins_cost(INSN_COST);
15461 format %{ "cmpw $op1, #0\t# unsigned" %}
15462
15463 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15464
15465 ins_pipe(icmp_reg_imm);
15466 %}
15467
15468 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15469 %{
15470 match(Set cr (CmpU op1 op2));
15471
15472 effect(DEF cr, USE op1);
15473
15474 ins_cost(INSN_COST);
15475 format %{ "cmpw $op1, $op2\t# unsigned" %}
15476
15477 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15478
15479 ins_pipe(icmp_reg_imm);
15480 %}
15481
15482 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15483 %{
15484 match(Set cr (CmpU op1 op2));
15485
15486 effect(DEF cr, USE op1);
15487
15488 ins_cost(INSN_COST * 2);
15489 format %{ "cmpw $op1, $op2\t# unsigned" %}
15490
15491 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15492
15493 ins_pipe(icmp_reg_imm);
15494 %}
15495
15496 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15497 %{
15498 match(Set cr (CmpL op1 op2));
15499
15500 effect(DEF cr, USE op1, USE op2);
15501
15502 ins_cost(INSN_COST);
15503 format %{ "cmp $op1, $op2" %}
15504
15505 ins_encode(aarch64_enc_cmp(op1, op2));
15506
15507 ins_pipe(icmp_reg_reg);
15508 %}
15509
15510 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15511 %{
15512 match(Set cr (CmpL op1 zero));
15513
15514 effect(DEF cr, USE op1);
15515
15516 ins_cost(INSN_COST);
15517 format %{ "tst $op1" %}
15518
15519 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15520
15521 ins_pipe(icmp_reg_imm);
15522 %}
15523
15524 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15525 %{
15526 match(Set cr (CmpL op1 op2));
15527
15528 effect(DEF cr, USE op1);
15529
15530 ins_cost(INSN_COST);
15531 format %{ "cmp $op1, $op2" %}
15532
15533 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15534
15535 ins_pipe(icmp_reg_imm);
15536 %}
15537
15538 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15539 %{
15540 match(Set cr (CmpL op1 op2));
15541
15542 effect(DEF cr, USE op1);
15543
15544 ins_cost(INSN_COST * 2);
15545 format %{ "cmp $op1, $op2" %}
15546
15547 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15548
15549 ins_pipe(icmp_reg_imm);
15550 %}
15551
15552 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15553 %{
15554 match(Set cr (CmpUL op1 op2));
15555
15556 effect(DEF cr, USE op1, USE op2);
15557
15558 ins_cost(INSN_COST);
15559 format %{ "cmp $op1, $op2" %}
15560
15561 ins_encode(aarch64_enc_cmp(op1, op2));
15562
15563 ins_pipe(icmp_reg_reg);
15564 %}
15565
15566 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15567 %{
15568 match(Set cr (CmpUL op1 zero));
15569
15570 effect(DEF cr, USE op1);
15571
15572 ins_cost(INSN_COST);
15573 format %{ "tst $op1" %}
15574
15575 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15576
15577 ins_pipe(icmp_reg_imm);
15578 %}
15579
15580 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15581 %{
15582 match(Set cr (CmpUL op1 op2));
15583
15584 effect(DEF cr, USE op1);
15585
15586 ins_cost(INSN_COST);
15587 format %{ "cmp $op1, $op2" %}
15588
15589 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15590
15591 ins_pipe(icmp_reg_imm);
15592 %}
15593
15594 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15595 %{
15596 match(Set cr (CmpUL op1 op2));
15597
15598 effect(DEF cr, USE op1);
15599
15600 ins_cost(INSN_COST * 2);
15601 format %{ "cmp $op1, $op2" %}
15602
15603 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15604
15605 ins_pipe(icmp_reg_imm);
15606 %}
15607
15608 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15609 %{
15610 match(Set cr (CmpP op1 op2));
15611
15612 effect(DEF cr, USE op1, USE op2);
15613
15614 ins_cost(INSN_COST);
15615 format %{ "cmp $op1, $op2\t // ptr" %}
15616
15617 ins_encode(aarch64_enc_cmpp(op1, op2));
15618
15619 ins_pipe(icmp_reg_reg);
15620 %}
15621
15622 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15623 %{
15624 match(Set cr (CmpN op1 op2));
15625
15626 effect(DEF cr, USE op1, USE op2);
15627
15628 ins_cost(INSN_COST);
15629 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15630
15631 ins_encode(aarch64_enc_cmpn(op1, op2));
15632
15633 ins_pipe(icmp_reg_reg);
15634 %}
15635
15636 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15637 %{
15638 match(Set cr (CmpP op1 zero));
15639
15640 effect(DEF cr, USE op1, USE zero);
15641
15642 ins_cost(INSN_COST);
15643 format %{ "cmp $op1, 0\t // ptr" %}
15644
15645 ins_encode(aarch64_enc_testp(op1));
15646
15647 ins_pipe(icmp_reg_imm);
15648 %}
15649
15650 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15651 %{
15652 match(Set cr (CmpN op1 zero));
15653
15654 effect(DEF cr, USE op1, USE zero);
15655
15656 ins_cost(INSN_COST);
15657 format %{ "cmp $op1, 0\t // compressed ptr" %}
15658
15659 ins_encode(aarch64_enc_testn(op1));
15660
15661 ins_pipe(icmp_reg_imm);
15662 %}
15663
15664 // FP comparisons
15665 //
15666 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15667 // using normal cmpOp. See declaration of rFlagsReg for details.
15668
15669 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15670 %{
15671 match(Set cr (CmpF src1 src2));
15672
15673 ins_cost(3 * INSN_COST);
15674 format %{ "fcmps $src1, $src2" %}
15675
15676 ins_encode %{
15677 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15678 %}
15679
15680 ins_pipe(pipe_class_compare);
15681 %}
15682
15683 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15684 %{
15685 match(Set cr (CmpF src1 src2));
15686
15687 ins_cost(3 * INSN_COST);
15688 format %{ "fcmps $src1, 0.0" %}
15689
15690 ins_encode %{
15691 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15692 %}
15693
15694 ins_pipe(pipe_class_compare);
15695 %}
15696 // FROM HERE
15697
15698 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15699 %{
15700 match(Set cr (CmpD src1 src2));
15701
15702 ins_cost(3 * INSN_COST);
15703 format %{ "fcmpd $src1, $src2" %}
15704
15705 ins_encode %{
15706 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15707 %}
15708
15709 ins_pipe(pipe_class_compare);
15710 %}
15711
15712 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15713 %{
15714 match(Set cr (CmpD src1 src2));
15715
15716 ins_cost(3 * INSN_COST);
15717 format %{ "fcmpd $src1, 0.0" %}
15718
15719 ins_encode %{
15720 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15721 %}
15722
15723 ins_pipe(pipe_class_compare);
15724 %}
15725
15726 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15727 %{
15728 match(Set dst (CmpF3 src1 src2));
15729 effect(KILL cr);
15730
15731 ins_cost(5 * INSN_COST);
15732 format %{ "fcmps $src1, $src2\n\t"
15733 "csinvw($dst, zr, zr, eq\n\t"
15734 "csnegw($dst, $dst, $dst, lt)"
15735 %}
15736
15737 ins_encode %{
15738 Label done;
15739 FloatRegister s1 = as_FloatRegister($src1$$reg);
15740 FloatRegister s2 = as_FloatRegister($src2$$reg);
15741 Register d = as_Register($dst$$reg);
15742 __ fcmps(s1, s2);
15743 // installs 0 if EQ else -1
15744 __ csinvw(d, zr, zr, Assembler::EQ);
15745 // keeps -1 if less or unordered else installs 1
15746 __ csnegw(d, d, d, Assembler::LT);
15747 __ bind(done);
15748 %}
15749
15750 ins_pipe(pipe_class_default);
15751
15752 %}
15753
15754 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15755 %{
15756 match(Set dst (CmpD3 src1 src2));
15757 effect(KILL cr);
15758
15759 ins_cost(5 * INSN_COST);
15760 format %{ "fcmpd $src1, $src2\n\t"
15761 "csinvw($dst, zr, zr, eq\n\t"
15762 "csnegw($dst, $dst, $dst, lt)"
15763 %}
15764
15765 ins_encode %{
15766 Label done;
15767 FloatRegister s1 = as_FloatRegister($src1$$reg);
15768 FloatRegister s2 = as_FloatRegister($src2$$reg);
15769 Register d = as_Register($dst$$reg);
15770 __ fcmpd(s1, s2);
15771 // installs 0 if EQ else -1
15772 __ csinvw(d, zr, zr, Assembler::EQ);
15773 // keeps -1 if less or unordered else installs 1
15774 __ csnegw(d, d, d, Assembler::LT);
15775 __ bind(done);
15776 %}
15777 ins_pipe(pipe_class_default);
15778
15779 %}
15780
15781 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15782 %{
15783 match(Set dst (CmpF3 src1 zero));
15784 effect(KILL cr);
15785
15786 ins_cost(5 * INSN_COST);
15787 format %{ "fcmps $src1, 0.0\n\t"
15788 "csinvw($dst, zr, zr, eq\n\t"
15789 "csnegw($dst, $dst, $dst, lt)"
15790 %}
15791
15792 ins_encode %{
15793 Label done;
15794 FloatRegister s1 = as_FloatRegister($src1$$reg);
15795 Register d = as_Register($dst$$reg);
15796 __ fcmps(s1, 0.0);
15797 // installs 0 if EQ else -1
15798 __ csinvw(d, zr, zr, Assembler::EQ);
15799 // keeps -1 if less or unordered else installs 1
15800 __ csnegw(d, d, d, Assembler::LT);
15801 __ bind(done);
15802 %}
15803
15804 ins_pipe(pipe_class_default);
15805
15806 %}
15807
15808 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15809 %{
15810 match(Set dst (CmpD3 src1 zero));
15811 effect(KILL cr);
15812
15813 ins_cost(5 * INSN_COST);
15814 format %{ "fcmpd $src1, 0.0\n\t"
15815 "csinvw($dst, zr, zr, eq\n\t"
15816 "csnegw($dst, $dst, $dst, lt)"
15817 %}
15818
15819 ins_encode %{
15820 Label done;
15821 FloatRegister s1 = as_FloatRegister($src1$$reg);
15822 Register d = as_Register($dst$$reg);
15823 __ fcmpd(s1, 0.0);
15824 // installs 0 if EQ else -1
15825 __ csinvw(d, zr, zr, Assembler::EQ);
15826 // keeps -1 if less or unordered else installs 1
15827 __ csnegw(d, d, d, Assembler::LT);
15828 __ bind(done);
15829 %}
15830 ins_pipe(pipe_class_default);
15831
15832 %}
15833
15834 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15835 %{
15836 match(Set dst (CmpLTMask p q));
15837 effect(KILL cr);
15838
15839 ins_cost(3 * INSN_COST);
15840
15841 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15842 "csetw $dst, lt\n\t"
15843 "subw $dst, zr, $dst"
15844 %}
15845
15846 ins_encode %{
15847 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15848 __ csetw(as_Register($dst$$reg), Assembler::LT);
15849 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15850 %}
15851
15852 ins_pipe(ialu_reg_reg);
15853 %}
15854
15855 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15856 %{
15857 match(Set dst (CmpLTMask src zero));
15858 effect(KILL cr);
15859
15860 ins_cost(INSN_COST);
15861
15862 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15863
15864 ins_encode %{
15865 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15866 %}
15867
15868 ins_pipe(ialu_reg_shift);
15869 %}
15870
15871 // ============================================================================
15872 // Max and Min
15873
15874 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15875
15876 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15877 %{
15878 effect(DEF cr, USE src);
15879 ins_cost(INSN_COST);
15880 format %{ "cmpw $src, 0" %}
15881
15882 ins_encode %{
15883 __ cmpw($src$$Register, 0);
15884 %}
15885 ins_pipe(icmp_reg_imm);
15886 %}
15887
15888 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15889 %{
15890 match(Set dst (MinI src1 src2));
15891 ins_cost(INSN_COST * 3);
15892
15893 expand %{
15894 rFlagsReg cr;
15895 compI_reg_reg(cr, src1, src2);
15896 cmovI_reg_reg_lt(dst, src1, src2, cr);
15897 %}
15898 %}
15899
15900 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15901 %{
15902 match(Set dst (MaxI src1 src2));
15903 ins_cost(INSN_COST * 3);
15904
15905 expand %{
15906 rFlagsReg cr;
15907 compI_reg_reg(cr, src1, src2);
15908 cmovI_reg_reg_gt(dst, src1, src2, cr);
15909 %}
15910 %}
15911
15912
15913 // ============================================================================
15914 // Branch Instructions
15915
15916 // Direct Branch.
15917 instruct branch(label lbl)
15918 %{
15919 match(Goto);
15920
15921 effect(USE lbl);
15922
15923 ins_cost(BRANCH_COST);
15924 format %{ "b $lbl" %}
15925
15926 ins_encode(aarch64_enc_b(lbl));
15927
15928 ins_pipe(pipe_branch);
15929 %}
15930
15931 // Conditional Near Branch
15932 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15933 %{
15934 // Same match rule as `branchConFar'.
15935 match(If cmp cr);
15936
15937 effect(USE lbl);
15938
15939 ins_cost(BRANCH_COST);
15940 // If set to 1 this indicates that the current instruction is a
15941 // short variant of a long branch. This avoids using this
15942 // instruction in first-pass matching. It will then only be used in
15943 // the `Shorten_branches' pass.
15944 // ins_short_branch(1);
15945 format %{ "b$cmp $lbl" %}
15946
15947 ins_encode(aarch64_enc_br_con(cmp, lbl));
15948
15949 ins_pipe(pipe_branch_cond);
15950 %}
15951
15952 // Conditional Near Branch Unsigned
15953 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15954 %{
15955 // Same match rule as `branchConFar'.
15956 match(If cmp cr);
15957
15958 effect(USE lbl);
15959
15960 ins_cost(BRANCH_COST);
15961 // If set to 1 this indicates that the current instruction is a
15962 // short variant of a long branch. This avoids using this
15963 // instruction in first-pass matching. It will then only be used in
15964 // the `Shorten_branches' pass.
15965 // ins_short_branch(1);
15966 format %{ "b$cmp $lbl\t# unsigned" %}
15967
15968 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15969
15970 ins_pipe(pipe_branch_cond);
15971 %}
15972
15973 // Make use of CBZ and CBNZ. These instructions, as well as being
15974 // shorter than (cmp; branch), have the additional benefit of not
15975 // killing the flags.
15976
15977 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15978 match(If cmp (CmpI op1 op2));
15979 effect(USE labl);
15980
15981 ins_cost(BRANCH_COST);
15982 format %{ "cbw$cmp $op1, $labl" %}
15983 ins_encode %{
15984 Label* L = $labl$$label;
15985 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15986 if (cond == Assembler::EQ)
15987 __ cbzw($op1$$Register, *L);
15988 else
15989 __ cbnzw($op1$$Register, *L);
15990 %}
15991 ins_pipe(pipe_cmp_branch);
15992 %}
15993
15994 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15995 match(If cmp (CmpL op1 op2));
15996 effect(USE labl);
15997
15998 ins_cost(BRANCH_COST);
15999 format %{ "cb$cmp $op1, $labl" %}
16000 ins_encode %{
16001 Label* L = $labl$$label;
16002 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16003 if (cond == Assembler::EQ)
16004 __ cbz($op1$$Register, *L);
16005 else
16006 __ cbnz($op1$$Register, *L);
16007 %}
16008 ins_pipe(pipe_cmp_branch);
16009 %}
16010
16011 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
16012 match(If cmp (CmpP op1 op2));
16013 effect(USE labl);
16014
16015 ins_cost(BRANCH_COST);
16016 format %{ "cb$cmp $op1, $labl" %}
16017 ins_encode %{
16018 Label* L = $labl$$label;
16019 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16020 if (cond == Assembler::EQ)
16021 __ cbz($op1$$Register, *L);
16022 else
16023 __ cbnz($op1$$Register, *L);
16024 %}
16025 ins_pipe(pipe_cmp_branch);
16026 %}
16027
16028 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
16029 match(If cmp (CmpN op1 op2));
16030 effect(USE labl);
16031
16032 ins_cost(BRANCH_COST);
16033 format %{ "cbw$cmp $op1, $labl" %}
16034 ins_encode %{
16035 Label* L = $labl$$label;
16036 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16037 if (cond == Assembler::EQ)
16038 __ cbzw($op1$$Register, *L);
16039 else
16040 __ cbnzw($op1$$Register, *L);
16041 %}
16042 ins_pipe(pipe_cmp_branch);
16043 %}
16044
16045 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
16046 match(If cmp (CmpP (DecodeN oop) zero));
16047 effect(USE labl);
16048
16049 ins_cost(BRANCH_COST);
16050 format %{ "cb$cmp $oop, $labl" %}
16051 ins_encode %{
16052 Label* L = $labl$$label;
16053 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16054 if (cond == Assembler::EQ)
16055 __ cbzw($oop$$Register, *L);
16056 else
16057 __ cbnzw($oop$$Register, *L);
16058 %}
16059 ins_pipe(pipe_cmp_branch);
16060 %}
16061
16062 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16063 match(If cmp (CmpU op1 op2));
16064 effect(USE labl);
16065
16066 ins_cost(BRANCH_COST);
16067 format %{ "cbw$cmp $op1, $labl" %}
16068 ins_encode %{
16069 Label* L = $labl$$label;
16070 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16071 if (cond == Assembler::EQ || cond == Assembler::LS) {
16072 __ cbzw($op1$$Register, *L);
16073 } else {
16074 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16075 __ cbnzw($op1$$Register, *L);
16076 }
16077 %}
16078 ins_pipe(pipe_cmp_branch);
16079 %}
16080
16081 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
16082 match(If cmp (CmpUL op1 op2));
16083 effect(USE labl);
16084
16085 ins_cost(BRANCH_COST);
16086 format %{ "cb$cmp $op1, $labl" %}
16087 ins_encode %{
16088 Label* L = $labl$$label;
16089 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16090 if (cond == Assembler::EQ || cond == Assembler::LS) {
16091 __ cbz($op1$$Register, *L);
16092 } else {
16093 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
16094 __ cbnz($op1$$Register, *L);
16095 }
16096 %}
16097 ins_pipe(pipe_cmp_branch);
16098 %}
16099
16100 // Test bit and Branch
16101
16102 // Patterns for short (< 32KiB) variants
16103 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16104 match(If cmp (CmpL op1 op2));
16105 effect(USE labl);
16106
16107 ins_cost(BRANCH_COST);
16108 format %{ "cb$cmp $op1, $labl # long" %}
16109 ins_encode %{
16110 Label* L = $labl$$label;
16111 Assembler::Condition cond =
16112 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16113 __ tbr(cond, $op1$$Register, 63, *L);
16114 %}
16115 ins_pipe(pipe_cmp_branch);
16116 ins_short_branch(1);
16117 %}
16118
16119 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16120 match(If cmp (CmpI op1 op2));
16121 effect(USE labl);
16122
16123 ins_cost(BRANCH_COST);
16124 format %{ "cb$cmp $op1, $labl # int" %}
16125 ins_encode %{
16126 Label* L = $labl$$label;
16127 Assembler::Condition cond =
16128 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16129 __ tbr(cond, $op1$$Register, 31, *L);
16130 %}
16131 ins_pipe(pipe_cmp_branch);
16132 ins_short_branch(1);
16133 %}
16134
16135 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16136 match(If cmp (CmpL (AndL op1 op2) op3));
16137 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16138 effect(USE labl);
16139
16140 ins_cost(BRANCH_COST);
16141 format %{ "tb$cmp $op1, $op2, $labl" %}
16142 ins_encode %{
16143 Label* L = $labl$$label;
16144 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16145 int bit = exact_log2_long($op2$$constant);
16146 __ tbr(cond, $op1$$Register, bit, *L);
16147 %}
16148 ins_pipe(pipe_cmp_branch);
16149 ins_short_branch(1);
16150 %}
16151
16152 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16153 match(If cmp (CmpI (AndI op1 op2) op3));
16154 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16155 effect(USE labl);
16156
16157 ins_cost(BRANCH_COST);
16158 format %{ "tb$cmp $op1, $op2, $labl" %}
16159 ins_encode %{
16160 Label* L = $labl$$label;
16161 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16162 int bit = exact_log2((juint)$op2$$constant);
16163 __ tbr(cond, $op1$$Register, bit, *L);
16164 %}
16165 ins_pipe(pipe_cmp_branch);
16166 ins_short_branch(1);
16167 %}
16168
16169 // And far variants
16170 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
16171 match(If cmp (CmpL op1 op2));
16172 effect(USE labl);
16173
16174 ins_cost(BRANCH_COST);
16175 format %{ "cb$cmp $op1, $labl # long" %}
16176 ins_encode %{
16177 Label* L = $labl$$label;
16178 Assembler::Condition cond =
16179 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16180 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
16181 %}
16182 ins_pipe(pipe_cmp_branch);
16183 %}
16184
16185 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
16186 match(If cmp (CmpI op1 op2));
16187 effect(USE labl);
16188
16189 ins_cost(BRANCH_COST);
16190 format %{ "cb$cmp $op1, $labl # int" %}
16191 ins_encode %{
16192 Label* L = $labl$$label;
16193 Assembler::Condition cond =
16194 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
16195 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
16196 %}
16197 ins_pipe(pipe_cmp_branch);
16198 %}
16199
16200 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
16201 match(If cmp (CmpL (AndL op1 op2) op3));
16202 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
16203 effect(USE labl);
16204
16205 ins_cost(BRANCH_COST);
16206 format %{ "tb$cmp $op1, $op2, $labl" %}
16207 ins_encode %{
16208 Label* L = $labl$$label;
16209 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16210 int bit = exact_log2_long($op2$$constant);
16211 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16212 %}
16213 ins_pipe(pipe_cmp_branch);
16214 %}
16215
16216 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
16217 match(If cmp (CmpI (AndI op1 op2) op3));
16218 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
16219 effect(USE labl);
16220
16221 ins_cost(BRANCH_COST);
16222 format %{ "tb$cmp $op1, $op2, $labl" %}
16223 ins_encode %{
16224 Label* L = $labl$$label;
16225 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
16226 int bit = exact_log2((juint)$op2$$constant);
16227 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
16228 %}
16229 ins_pipe(pipe_cmp_branch);
16230 %}
16231
16232 // Test bits
16233
16234 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
16235 match(Set cr (CmpL (AndL op1 op2) op3));
16236 predicate(Assembler::operand_valid_for_logical_immediate
16237 (/*is_32*/false, n->in(1)->in(2)->get_long()));
16238
16239 ins_cost(INSN_COST);
16240 format %{ "tst $op1, $op2 # long" %}
16241 ins_encode %{
16242 __ tst($op1$$Register, $op2$$constant);
16243 %}
16244 ins_pipe(ialu_reg_reg);
16245 %}
16246
16247 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
16248 match(Set cr (CmpI (AndI op1 op2) op3));
16249 predicate(Assembler::operand_valid_for_logical_immediate
16250 (/*is_32*/true, n->in(1)->in(2)->get_int()));
16251
16252 ins_cost(INSN_COST);
16253 format %{ "tst $op1, $op2 # int" %}
16254 ins_encode %{
16255 __ tstw($op1$$Register, $op2$$constant);
16256 %}
16257 ins_pipe(ialu_reg_reg);
16258 %}
16259
16260 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
16261 match(Set cr (CmpL (AndL op1 op2) op3));
16262
16263 ins_cost(INSN_COST);
16264 format %{ "tst $op1, $op2 # long" %}
16265 ins_encode %{
16266 __ tst($op1$$Register, $op2$$Register);
16267 %}
16268 ins_pipe(ialu_reg_reg);
16269 %}
16270
16271 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
16272 match(Set cr (CmpI (AndI op1 op2) op3));
16273
16274 ins_cost(INSN_COST);
16275 format %{ "tstw $op1, $op2 # int" %}
16276 ins_encode %{
16277 __ tstw($op1$$Register, $op2$$Register);
16278 %}
16279 ins_pipe(ialu_reg_reg);
16280 %}
16281
16282
16283 // Conditional Far Branch
16284 // Conditional Far Branch Unsigned
16285 // TODO: fixme
16286
16287 // counted loop end branch near
16288 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
16289 %{
16290 match(CountedLoopEnd cmp cr);
16291
16292 effect(USE lbl);
16293
16294 ins_cost(BRANCH_COST);
16295 // short variant.
16296 // ins_short_branch(1);
16297 format %{ "b$cmp $lbl \t// counted loop end" %}
16298
16299 ins_encode(aarch64_enc_br_con(cmp, lbl));
16300
16301 ins_pipe(pipe_branch);
16302 %}
16303
16304 // counted loop end branch far
16305 // TODO: fixme
16306
16307 // ============================================================================
16308 // inlined locking and unlocking
16309
16310 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16311 %{
16312 match(Set cr (FastLock object box));
16313 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16314
16315 ins_cost(5 * INSN_COST);
16316 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
16317
16318 ins_encode %{
16319 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16320 %}
16321
16322 ins_pipe(pipe_serial);
16323 %}
16324
16325 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16326 %{
16327 match(Set cr (FastUnlock object box));
16328 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16329
16330 ins_cost(5 * INSN_COST);
16331 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16332
16333 ins_encode %{
16334 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16335 %}
16336
16337 ins_pipe(pipe_serial);
16338 %}
16339
16340 // ============================================================================
16341 // Safepoint Instructions
16342
16343 // TODO
16344 // provide a near and far version of this code
16345
16346 instruct safePoint(rFlagsReg cr, iRegP poll)
16347 %{
16348 match(SafePoint poll);
16349 effect(KILL cr);
16350
16351 format %{
16352 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16353 %}
16354 ins_encode %{
16355 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16356 %}
16357 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16358 %}
16359
16360
16361 // ============================================================================
16362 // Procedure Call/Return Instructions
16363
16364 // Call Java Static Instruction
16365
16366 instruct CallStaticJavaDirect(method meth)
16367 %{
16368 match(CallStaticJava);
16369
16370 effect(USE meth);
16371
16372 ins_cost(CALL_COST);
16373
16374 format %{ "call,static $meth \t// ==> " %}
16375
16376 ins_encode(aarch64_enc_java_static_call(meth),
16377 aarch64_enc_call_epilog);
16378
16379 ins_pipe(pipe_class_call);
16380 %}
16381
16382 // TO HERE
16383
16384 // Call Java Dynamic Instruction
16385 instruct CallDynamicJavaDirect(method meth)
16386 %{
16387 match(CallDynamicJava);
16388
16389 effect(USE meth);
16390
16391 ins_cost(CALL_COST);
16392
16393 format %{ "CALL,dynamic $meth \t// ==> " %}
16394
16395 ins_encode(aarch64_enc_java_dynamic_call(meth),
16396 aarch64_enc_call_epilog);
16397
16398 ins_pipe(pipe_class_call);
16399 %}
16400
16401 // Call Runtime Instruction
16402
16403 instruct CallRuntimeDirect(method meth)
16404 %{
16405 match(CallRuntime);
16406
16407 effect(USE meth);
16408
16409 ins_cost(CALL_COST);
16410
16411 format %{ "CALL, runtime $meth" %}
16412
16413 ins_encode( aarch64_enc_java_to_runtime(meth) );
16414
16415 ins_pipe(pipe_class_call);
16416 %}
16417
16418 // Call Runtime Instruction
16419
16420 instruct CallLeafDirect(method meth)
16421 %{
16422 match(CallLeaf);
16423
16424 effect(USE meth);
16425
16426 ins_cost(CALL_COST);
16427
16428 format %{ "CALL, runtime leaf $meth" %}
16429
16430 ins_encode( aarch64_enc_java_to_runtime(meth) );
16431
16432 ins_pipe(pipe_class_call);
16433 %}
16434
16435 // Call Runtime Instruction without safepoint and with vector arguments
16436 instruct CallLeafDirectVector(method meth)
16437 %{
16438 match(CallLeafVector);
16439
16440 effect(USE meth);
16441
16442 ins_cost(CALL_COST);
16443
16444 format %{ "CALL, runtime leaf vector $meth" %}
16445
16446 ins_encode(aarch64_enc_java_to_runtime(meth));
16447
16448 ins_pipe(pipe_class_call);
16449 %}
16450
16451 // Call Runtime Instruction
16452
16453 instruct CallLeafNoFPDirect(method meth)
16454 %{
16455 match(CallLeafNoFP);
16456
16457 effect(USE meth);
16458
16459 ins_cost(CALL_COST);
16460
16461 format %{ "CALL, runtime leaf nofp $meth" %}
16462
16463 ins_encode( aarch64_enc_java_to_runtime(meth) );
16464
16465 ins_pipe(pipe_class_call);
16466 %}
16467
16468 // Tail Call; Jump from runtime stub to Java code.
16469 // Also known as an 'interprocedural jump'.
16470 // Target of jump will eventually return to caller.
16471 // TailJump below removes the return address.
16472 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16473 // emitted just above the TailCall which has reset rfp to the caller state.
16474 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16475 %{
16476 match(TailCall jump_target method_ptr);
16477
16478 ins_cost(CALL_COST);
16479
16480 format %{ "br $jump_target\t# $method_ptr holds method" %}
16481
16482 ins_encode(aarch64_enc_tail_call(jump_target));
16483
16484 ins_pipe(pipe_class_call);
16485 %}
16486
16487 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16488 %{
16489 match(TailJump jump_target ex_oop);
16490
16491 ins_cost(CALL_COST);
16492
16493 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16494
16495 ins_encode(aarch64_enc_tail_jmp(jump_target));
16496
16497 ins_pipe(pipe_class_call);
16498 %}
16499
16500 // Forward exception.
16501 instruct ForwardExceptionjmp()
16502 %{
16503 match(ForwardException);
16504 ins_cost(CALL_COST);
16505
16506 format %{ "b forward_exception_stub" %}
16507 ins_encode %{
16508 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16509 %}
16510 ins_pipe(pipe_class_call);
16511 %}
16512
16513 // Create exception oop: created by stack-crawling runtime code.
16514 // Created exception is now available to this handler, and is setup
16515 // just prior to jumping to this handler. No code emitted.
16516 // TODO check
16517 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16518 instruct CreateException(iRegP_R0 ex_oop)
16519 %{
16520 match(Set ex_oop (CreateEx));
16521
16522 format %{ " -- \t// exception oop; no code emitted" %}
16523
16524 size(0);
16525
16526 ins_encode( /*empty*/ );
16527
16528 ins_pipe(pipe_class_empty);
16529 %}
16530
16531 // Rethrow exception: The exception oop will come in the first
16532 // argument position. Then JUMP (not call) to the rethrow stub code.
16533 instruct RethrowException() %{
16534 match(Rethrow);
16535 ins_cost(CALL_COST);
16536
16537 format %{ "b rethrow_stub" %}
16538
16539 ins_encode( aarch64_enc_rethrow() );
16540
16541 ins_pipe(pipe_class_call);
16542 %}
16543
16544
16545 // Return Instruction
16546 // epilog node loads ret address into lr as part of frame pop
16547 instruct Ret()
16548 %{
16549 match(Return);
16550
16551 format %{ "ret\t// return register" %}
16552
16553 ins_encode( aarch64_enc_ret() );
16554
16555 ins_pipe(pipe_branch);
16556 %}
16557
16558 // Die now.
16559 instruct ShouldNotReachHere() %{
16560 match(Halt);
16561
16562 ins_cost(CALL_COST);
16563 format %{ "ShouldNotReachHere" %}
16564
16565 ins_encode %{
16566 if (is_reachable()) {
16567 const char* str = __ code_string(_halt_reason);
16568 __ stop(str);
16569 }
16570 %}
16571
16572 ins_pipe(pipe_class_default);
16573 %}
16574
16575 // ============================================================================
16576 // Partial Subtype Check
16577 //
16578 // superklass array for an instance of the superklass. Set a hidden
16579 // internal cache on a hit (cache is checked with exposed code in
16580 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16581 // encoding ALSO sets flags.
16582
16583 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16584 %{
16585 match(Set result (PartialSubtypeCheck sub super));
16586 predicate(!UseSecondarySupersTable);
16587 effect(KILL cr, KILL temp);
16588
16589 ins_cost(20 * INSN_COST); // slightly larger than the next version
16590 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16591
16592 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16593
16594 opcode(0x1); // Force zero of result reg on hit
16595
16596 ins_pipe(pipe_class_memory);
16597 %}
16598
16599 // Two versions of partialSubtypeCheck, both used when we need to
16600 // search for a super class in the secondary supers array. The first
16601 // is used when we don't know _a priori_ the class being searched
16602 // for. The second, far more common, is used when we do know: this is
16603 // used for instanceof, checkcast, and any case where C2 can determine
16604 // it by constant propagation.
16605
16606 instruct partialSubtypeCheckVarSuper(iRegP_R4 sub, iRegP_R0 super, vRegD_V0 vtemp, iRegP_R5 result,
16607 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16608 rFlagsReg cr)
16609 %{
16610 match(Set result (PartialSubtypeCheck sub super));
16611 predicate(UseSecondarySupersTable);
16612 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16613
16614 ins_cost(10 * INSN_COST); // slightly larger than the next version
16615 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16616
16617 ins_encode %{
16618 __ lookup_secondary_supers_table_var($sub$$Register, $super$$Register,
16619 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16620 $vtemp$$FloatRegister,
16621 $result$$Register, /*L_success*/nullptr);
16622 %}
16623
16624 ins_pipe(pipe_class_memory);
16625 %}
16626
16627 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16628 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16629 rFlagsReg cr)
16630 %{
16631 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16632 predicate(UseSecondarySupersTable);
16633 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16634
16635 ins_cost(5 * INSN_COST); // smaller than the next version
16636 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16637
16638 ins_encode %{
16639 bool success = false;
16640 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16641 if (InlineSecondarySupersTest) {
16642 success =
16643 __ lookup_secondary_supers_table_const($sub$$Register, $super_reg$$Register,
16644 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16645 $vtemp$$FloatRegister,
16646 $result$$Register,
16647 super_klass_slot);
16648 } else {
16649 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16650 success = (call != nullptr);
16651 }
16652 if (!success) {
16653 ciEnv::current()->record_failure("CodeCache is full");
16654 return;
16655 }
16656 %}
16657
16658 ins_pipe(pipe_class_memory);
16659 %}
16660
16661 // Intrisics for String.compareTo()
16662
16663 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16664 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16665 %{
16666 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16667 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16668 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16669
16670 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16671 ins_encode %{
16672 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16673 __ string_compare($str1$$Register, $str2$$Register,
16674 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16675 $tmp1$$Register, $tmp2$$Register,
16676 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16677 %}
16678 ins_pipe(pipe_class_memory);
16679 %}
16680
16681 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16682 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16683 %{
16684 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16685 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16686 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16687
16688 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16689 ins_encode %{
16690 __ string_compare($str1$$Register, $str2$$Register,
16691 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16692 $tmp1$$Register, $tmp2$$Register,
16693 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16694 %}
16695 ins_pipe(pipe_class_memory);
16696 %}
16697
16698 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16699 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16700 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16701 %{
16702 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16703 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16704 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16705 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16706
16707 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16708 ins_encode %{
16709 __ string_compare($str1$$Register, $str2$$Register,
16710 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16711 $tmp1$$Register, $tmp2$$Register,
16712 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16713 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16714 %}
16715 ins_pipe(pipe_class_memory);
16716 %}
16717
16718 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16719 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16720 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16721 %{
16722 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16723 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16724 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16725 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16726
16727 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16728 ins_encode %{
16729 __ string_compare($str1$$Register, $str2$$Register,
16730 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16731 $tmp1$$Register, $tmp2$$Register,
16732 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16733 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16734 %}
16735 ins_pipe(pipe_class_memory);
16736 %}
16737
16738 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16739 // these string_compare variants as NEON register type for convenience so that the prototype of
16740 // string_compare can be shared with all variants.
16741
16742 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16743 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16744 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16745 pRegGov_P1 pgtmp2, rFlagsReg cr)
16746 %{
16747 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16748 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16749 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16750 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16751
16752 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16753 ins_encode %{
16754 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16755 __ string_compare($str1$$Register, $str2$$Register,
16756 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16757 $tmp1$$Register, $tmp2$$Register,
16758 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16759 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16760 StrIntrinsicNode::LL);
16761 %}
16762 ins_pipe(pipe_class_memory);
16763 %}
16764
16765 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16766 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16767 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16768 pRegGov_P1 pgtmp2, rFlagsReg cr)
16769 %{
16770 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16771 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16772 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16773 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16774
16775 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16776 ins_encode %{
16777 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16778 __ string_compare($str1$$Register, $str2$$Register,
16779 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16780 $tmp1$$Register, $tmp2$$Register,
16781 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16782 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16783 StrIntrinsicNode::LU);
16784 %}
16785 ins_pipe(pipe_class_memory);
16786 %}
16787
16788 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16789 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16790 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16791 pRegGov_P1 pgtmp2, rFlagsReg cr)
16792 %{
16793 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16794 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16795 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16796 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16797
16798 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16799 ins_encode %{
16800 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16801 __ string_compare($str1$$Register, $str2$$Register,
16802 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16803 $tmp1$$Register, $tmp2$$Register,
16804 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16805 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16806 StrIntrinsicNode::UL);
16807 %}
16808 ins_pipe(pipe_class_memory);
16809 %}
16810
16811 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16812 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16813 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16814 pRegGov_P1 pgtmp2, rFlagsReg cr)
16815 %{
16816 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16817 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16818 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16819 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16820
16821 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16822 ins_encode %{
16823 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16824 __ string_compare($str1$$Register, $str2$$Register,
16825 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16826 $tmp1$$Register, $tmp2$$Register,
16827 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16828 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16829 StrIntrinsicNode::UU);
16830 %}
16831 ins_pipe(pipe_class_memory);
16832 %}
16833
16834 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16835 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16836 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16837 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16838 %{
16839 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16840 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16841 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16842 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16843 TEMP vtmp0, TEMP vtmp1, KILL cr);
16844 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16845 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16846
16847 ins_encode %{
16848 __ string_indexof($str1$$Register, $str2$$Register,
16849 $cnt1$$Register, $cnt2$$Register,
16850 $tmp1$$Register, $tmp2$$Register,
16851 $tmp3$$Register, $tmp4$$Register,
16852 $tmp5$$Register, $tmp6$$Register,
16853 -1, $result$$Register, StrIntrinsicNode::UU);
16854 %}
16855 ins_pipe(pipe_class_memory);
16856 %}
16857
16858 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16859 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16860 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16861 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16862 %{
16863 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16864 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16865 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16866 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16867 TEMP vtmp0, TEMP vtmp1, KILL cr);
16868 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16869 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16870
16871 ins_encode %{
16872 __ string_indexof($str1$$Register, $str2$$Register,
16873 $cnt1$$Register, $cnt2$$Register,
16874 $tmp1$$Register, $tmp2$$Register,
16875 $tmp3$$Register, $tmp4$$Register,
16876 $tmp5$$Register, $tmp6$$Register,
16877 -1, $result$$Register, StrIntrinsicNode::LL);
16878 %}
16879 ins_pipe(pipe_class_memory);
16880 %}
16881
16882 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16883 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16884 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16885 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16886 %{
16887 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16888 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16889 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16890 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16891 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16892 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16893 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16894
16895 ins_encode %{
16896 __ string_indexof($str1$$Register, $str2$$Register,
16897 $cnt1$$Register, $cnt2$$Register,
16898 $tmp1$$Register, $tmp2$$Register,
16899 $tmp3$$Register, $tmp4$$Register,
16900 $tmp5$$Register, $tmp6$$Register,
16901 -1, $result$$Register, StrIntrinsicNode::UL);
16902 %}
16903 ins_pipe(pipe_class_memory);
16904 %}
16905
16906 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16907 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16908 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16909 %{
16910 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16911 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16912 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16913 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16914 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16915 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16916
16917 ins_encode %{
16918 int icnt2 = (int)$int_cnt2$$constant;
16919 __ string_indexof($str1$$Register, $str2$$Register,
16920 $cnt1$$Register, zr,
16921 $tmp1$$Register, $tmp2$$Register,
16922 $tmp3$$Register, $tmp4$$Register, zr, zr,
16923 icnt2, $result$$Register, StrIntrinsicNode::UU);
16924 %}
16925 ins_pipe(pipe_class_memory);
16926 %}
16927
16928 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16929 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16930 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16931 %{
16932 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16933 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16934 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16935 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16936 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16937 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16938
16939 ins_encode %{
16940 int icnt2 = (int)$int_cnt2$$constant;
16941 __ string_indexof($str1$$Register, $str2$$Register,
16942 $cnt1$$Register, zr,
16943 $tmp1$$Register, $tmp2$$Register,
16944 $tmp3$$Register, $tmp4$$Register, zr, zr,
16945 icnt2, $result$$Register, StrIntrinsicNode::LL);
16946 %}
16947 ins_pipe(pipe_class_memory);
16948 %}
16949
16950 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16951 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16952 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16953 %{
16954 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16955 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16956 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16957 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16958 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16959 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16960
16961 ins_encode %{
16962 int icnt2 = (int)$int_cnt2$$constant;
16963 __ string_indexof($str1$$Register, $str2$$Register,
16964 $cnt1$$Register, zr,
16965 $tmp1$$Register, $tmp2$$Register,
16966 $tmp3$$Register, $tmp4$$Register, zr, zr,
16967 icnt2, $result$$Register, StrIntrinsicNode::UL);
16968 %}
16969 ins_pipe(pipe_class_memory);
16970 %}
16971
16972 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16973 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16974 iRegINoSp tmp3, rFlagsReg cr)
16975 %{
16976 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16977 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16978 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16979 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16980
16981 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16982
16983 ins_encode %{
16984 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16985 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16986 $tmp3$$Register);
16987 %}
16988 ins_pipe(pipe_class_memory);
16989 %}
16990
16991 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16992 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16993 iRegINoSp tmp3, rFlagsReg cr)
16994 %{
16995 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16996 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16997 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16998 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16999
17000 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
17001
17002 ins_encode %{
17003 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
17004 $result$$Register, $tmp1$$Register, $tmp2$$Register,
17005 $tmp3$$Register);
17006 %}
17007 ins_pipe(pipe_class_memory);
17008 %}
17009
17010 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17011 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17012 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17013 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
17014 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17015 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17016 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17017 ins_encode %{
17018 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17019 $result$$Register, $ztmp1$$FloatRegister,
17020 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17021 $ptmp$$PRegister, true /* isL */);
17022 %}
17023 ins_pipe(pipe_class_memory);
17024 %}
17025
17026 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
17027 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
17028 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
17029 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
17030 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
17031 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
17032 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
17033 ins_encode %{
17034 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
17035 $result$$Register, $ztmp1$$FloatRegister,
17036 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
17037 $ptmp$$PRegister, false /* isL */);
17038 %}
17039 ins_pipe(pipe_class_memory);
17040 %}
17041
17042 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
17043 iRegI_R0 result, rFlagsReg cr)
17044 %{
17045 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
17046 match(Set result (StrEquals (Binary str1 str2) cnt));
17047 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
17048
17049 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
17050 ins_encode %{
17051 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
17052 __ string_equals($str1$$Register, $str2$$Register,
17053 $result$$Register, $cnt$$Register);
17054 %}
17055 ins_pipe(pipe_class_memory);
17056 %}
17057
17058 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17059 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17060 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17061 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17062 iRegP_R10 tmp, rFlagsReg cr)
17063 %{
17064 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
17065 match(Set result (AryEq ary1 ary2));
17066 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17067 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17068 TEMP vtmp6, TEMP vtmp7, KILL cr);
17069
17070 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17071 ins_encode %{
17072 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17073 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17074 $result$$Register, $tmp$$Register, 1);
17075 if (tpc == nullptr) {
17076 ciEnv::current()->record_failure("CodeCache is full");
17077 return;
17078 }
17079 %}
17080 ins_pipe(pipe_class_memory);
17081 %}
17082
17083 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
17084 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
17085 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17086 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17087 iRegP_R10 tmp, rFlagsReg cr)
17088 %{
17089 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
17090 match(Set result (AryEq ary1 ary2));
17091 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
17092 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17093 TEMP vtmp6, TEMP vtmp7, KILL cr);
17094
17095 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
17096 ins_encode %{
17097 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
17098 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
17099 $result$$Register, $tmp$$Register, 2);
17100 if (tpc == nullptr) {
17101 ciEnv::current()->record_failure("CodeCache is full");
17102 return;
17103 }
17104 %}
17105 ins_pipe(pipe_class_memory);
17106 %}
17107
17108 instruct arrays_hashcode(iRegP_R1 ary, iRegI_R2 cnt, iRegI_R0 result, immI basic_type,
17109 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17110 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
17111 vRegD_V12 vtmp8, vRegD_V13 vtmp9, rFlagsReg cr)
17112 %{
17113 match(Set result (VectorizedHashCode (Binary ary cnt) (Binary result basic_type)));
17114 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5, TEMP vtmp6,
17115 TEMP vtmp7, TEMP vtmp8, TEMP vtmp9, USE_KILL ary, USE_KILL cnt, USE basic_type, KILL cr);
17116
17117 format %{ "Array HashCode array[] $ary,$cnt,$result,$basic_type -> $result // KILL all" %}
17118 ins_encode %{
17119 address tpc = __ arrays_hashcode($ary$$Register, $cnt$$Register, $result$$Register,
17120 $vtmp3$$FloatRegister, $vtmp2$$FloatRegister,
17121 $vtmp1$$FloatRegister, $vtmp0$$FloatRegister,
17122 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister,
17123 $vtmp6$$FloatRegister, $vtmp7$$FloatRegister,
17124 $vtmp8$$FloatRegister, $vtmp9$$FloatRegister,
17125 (BasicType)$basic_type$$constant);
17126 if (tpc == nullptr) {
17127 ciEnv::current()->record_failure("CodeCache is full");
17128 return;
17129 }
17130 %}
17131 ins_pipe(pipe_class_memory);
17132 %}
17133
17134 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
17135 %{
17136 match(Set result (CountPositives ary1 len));
17137 effect(USE_KILL ary1, USE_KILL len, KILL cr);
17138 format %{ "count positives byte[] $ary1,$len -> $result" %}
17139 ins_encode %{
17140 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
17141 if (tpc == nullptr) {
17142 ciEnv::current()->record_failure("CodeCache is full");
17143 return;
17144 }
17145 %}
17146 ins_pipe( pipe_slow );
17147 %}
17148
17149 // fast char[] to byte[] compression
17150 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17151 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17152 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17153 iRegI_R0 result, rFlagsReg cr)
17154 %{
17155 match(Set result (StrCompressedCopy src (Binary dst len)));
17156 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
17157 USE_KILL src, USE_KILL dst, USE len, KILL cr);
17158
17159 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17160 ins_encode %{
17161 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
17162 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17163 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17164 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17165 %}
17166 ins_pipe(pipe_slow);
17167 %}
17168
17169 // fast byte[] to char[] inflation
17170 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
17171 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
17172 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
17173 %{
17174 match(Set dummy (StrInflatedCopy src (Binary dst len)));
17175 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
17176 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
17177 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
17178
17179 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
17180 ins_encode %{
17181 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
17182 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17183 $vtmp2$$FloatRegister, $tmp$$Register);
17184 if (tpc == nullptr) {
17185 ciEnv::current()->record_failure("CodeCache is full");
17186 return;
17187 }
17188 %}
17189 ins_pipe(pipe_class_memory);
17190 %}
17191
17192 // encode char[] to byte[] in ISO_8859_1
17193 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17194 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17195 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17196 iRegI_R0 result, rFlagsReg cr)
17197 %{
17198 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
17199 match(Set result (EncodeISOArray src (Binary dst len)));
17200 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17201 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17202
17203 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17204 ins_encode %{
17205 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17206 $result$$Register, false,
17207 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17208 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17209 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17210 %}
17211 ins_pipe(pipe_class_memory);
17212 %}
17213
17214 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
17215 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
17216 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
17217 iRegI_R0 result, rFlagsReg cr)
17218 %{
17219 predicate(((EncodeISOArrayNode*)n)->is_ascii());
17220 match(Set result (EncodeISOArray src (Binary dst len)));
17221 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
17222 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
17223
17224 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
17225 ins_encode %{
17226 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
17227 $result$$Register, true,
17228 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
17229 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
17230 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
17231 %}
17232 ins_pipe(pipe_class_memory);
17233 %}
17234
17235 //----------------------------- CompressBits/ExpandBits ------------------------
17236
17237 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17238 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17239 match(Set dst (CompressBits src mask));
17240 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17241 format %{ "mov $tsrc, $src\n\t"
17242 "mov $tmask, $mask\n\t"
17243 "bext $tdst, $tsrc, $tmask\n\t"
17244 "mov $dst, $tdst"
17245 %}
17246 ins_encode %{
17247 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17248 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17249 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17250 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17251 %}
17252 ins_pipe(pipe_slow);
17253 %}
17254
17255 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17256 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17257 match(Set dst (CompressBits (LoadI mem) mask));
17258 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17259 format %{ "ldrs $tsrc, $mem\n\t"
17260 "ldrs $tmask, $mask\n\t"
17261 "bext $tdst, $tsrc, $tmask\n\t"
17262 "mov $dst, $tdst"
17263 %}
17264 ins_encode %{
17265 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17266 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17267 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17268 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17269 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17270 %}
17271 ins_pipe(pipe_slow);
17272 %}
17273
17274 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17275 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17276 match(Set dst (CompressBits src mask));
17277 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17278 format %{ "mov $tsrc, $src\n\t"
17279 "mov $tmask, $mask\n\t"
17280 "bext $tdst, $tsrc, $tmask\n\t"
17281 "mov $dst, $tdst"
17282 %}
17283 ins_encode %{
17284 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17285 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17286 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17287 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17288 %}
17289 ins_pipe(pipe_slow);
17290 %}
17291
17292 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
17293 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17294 match(Set dst (CompressBits (LoadL mem) mask));
17295 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17296 format %{ "ldrd $tsrc, $mem\n\t"
17297 "ldrd $tmask, $mask\n\t"
17298 "bext $tdst, $tsrc, $tmask\n\t"
17299 "mov $dst, $tdst"
17300 %}
17301 ins_encode %{
17302 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17303 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17304 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17305 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17306 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17307 %}
17308 ins_pipe(pipe_slow);
17309 %}
17310
17311 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
17312 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17313 match(Set dst (ExpandBits src mask));
17314 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17315 format %{ "mov $tsrc, $src\n\t"
17316 "mov $tmask, $mask\n\t"
17317 "bdep $tdst, $tsrc, $tmask\n\t"
17318 "mov $dst, $tdst"
17319 %}
17320 ins_encode %{
17321 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
17322 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
17323 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17324 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17325 %}
17326 ins_pipe(pipe_slow);
17327 %}
17328
17329 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
17330 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17331 match(Set dst (ExpandBits (LoadI mem) mask));
17332 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17333 format %{ "ldrs $tsrc, $mem\n\t"
17334 "ldrs $tmask, $mask\n\t"
17335 "bdep $tdst, $tsrc, $tmask\n\t"
17336 "mov $dst, $tdst"
17337 %}
17338 ins_encode %{
17339 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
17340 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
17341 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
17342 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17343 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
17344 %}
17345 ins_pipe(pipe_slow);
17346 %}
17347
17348 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
17349 vRegD tdst, vRegD tsrc, vRegD tmask) %{
17350 match(Set dst (ExpandBits src mask));
17351 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17352 format %{ "mov $tsrc, $src\n\t"
17353 "mov $tmask, $mask\n\t"
17354 "bdep $tdst, $tsrc, $tmask\n\t"
17355 "mov $dst, $tdst"
17356 %}
17357 ins_encode %{
17358 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
17359 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
17360 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17361 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17362 %}
17363 ins_pipe(pipe_slow);
17364 %}
17365
17366
17367 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
17368 vRegF tdst, vRegF tsrc, vRegF tmask) %{
17369 match(Set dst (ExpandBits (LoadL mem) mask));
17370 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
17371 format %{ "ldrd $tsrc, $mem\n\t"
17372 "ldrd $tmask, $mask\n\t"
17373 "bdep $tdst, $tsrc, $tmask\n\t"
17374 "mov $dst, $tdst"
17375 %}
17376 ins_encode %{
17377 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
17378 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
17379 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
17380 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17381 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17382 %}
17383 ins_pipe(pipe_slow);
17384 %}
17385
17386 //----------------------------- Reinterpret ----------------------------------
17387 // Reinterpret a half-precision float value in a floating point register to a general purpose register
17388 instruct reinterpretHF2S(iRegINoSp dst, vRegF src) %{
17389 match(Set dst (ReinterpretHF2S src));
17390 format %{ "reinterpretHF2S $dst, $src" %}
17391 ins_encode %{
17392 __ smov($dst$$Register, $src$$FloatRegister, __ H, 0);
17393 %}
17394 ins_pipe(pipe_slow);
17395 %}
17396
17397 // Reinterpret a half-precision float value in a general purpose register to a floating point register
17398 instruct reinterpretS2HF(vRegF dst, iRegINoSp src) %{
17399 match(Set dst (ReinterpretS2HF src));
17400 format %{ "reinterpretS2HF $dst, $src" %}
17401 ins_encode %{
17402 __ mov($dst$$FloatRegister, __ H, 0, $src$$Register);
17403 %}
17404 ins_pipe(pipe_slow);
17405 %}
17406
17407 // Without this optimization, ReinterpretS2HF (ConvF2HF src) would result in the following
17408 // instructions (the first two are for ConvF2HF and the last instruction is for ReinterpretS2HF) -
17409 // fcvt $tmp1_fpr, $src_fpr // Convert float to half-precision float
17410 // mov $tmp2_gpr, $tmp1_fpr // Move half-precision float in FPR to a GPR
17411 // mov $dst_fpr, $tmp2_gpr // Move the result from a GPR to an FPR
17412 // The move from FPR to GPR in ConvF2HF and the move from GPR to FPR in ReinterpretS2HF
17413 // can be omitted in this pattern, resulting in -
17414 // fcvt $dst, $src // Convert float to half-precision float
17415 instruct convF2HFAndS2HF(vRegF dst, vRegF src)
17416 %{
17417 match(Set dst (ReinterpretS2HF (ConvF2HF src)));
17418 format %{ "convF2HFAndS2HF $dst, $src" %}
17419 ins_encode %{
17420 __ fcvtsh($dst$$FloatRegister, $src$$FloatRegister);
17421 %}
17422 ins_pipe(pipe_slow);
17423 %}
17424
17425 // Without this optimization, ConvHF2F (ReinterpretHF2S src) would result in the following
17426 // instructions (the first one is for ReinterpretHF2S and the last two are for ConvHF2F) -
17427 // mov $tmp1_gpr, $src_fpr // Move the half-precision float from an FPR to a GPR
17428 // mov $tmp2_fpr, $tmp1_gpr // Move the same value from GPR to an FPR
17429 // fcvt $dst_fpr, $tmp2_fpr // Convert the half-precision float to 32-bit float
17430 // The move from FPR to GPR in ReinterpretHF2S and the move from GPR to FPR in ConvHF2F
17431 // can be omitted as the input (src) is already in an FPR required for the fcvths instruction
17432 // resulting in -
17433 // fcvt $dst, $src // Convert half-precision float to a 32-bit float
17434 instruct convHF2SAndHF2F(vRegF dst, vRegF src)
17435 %{
17436 match(Set dst (ConvHF2F (ReinterpretHF2S src)));
17437 format %{ "convHF2SAndHF2F $dst, $src" %}
17438 ins_encode %{
17439 __ fcvths($dst$$FloatRegister, $src$$FloatRegister);
17440 %}
17441 ins_pipe(pipe_slow);
17442 %}
17443
17444 // ============================================================================
17445 // This name is KNOWN by the ADLC and cannot be changed.
17446 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17447 // for this guy.
17448 instruct tlsLoadP(thread_RegP dst)
17449 %{
17450 match(Set dst (ThreadLocal));
17451
17452 ins_cost(0);
17453
17454 format %{ " -- \t// $dst=Thread::current(), empty" %}
17455
17456 size(0);
17457
17458 ins_encode( /*empty*/ );
17459
17460 ins_pipe(pipe_class_empty);
17461 %}
17462
17463 //----------PEEPHOLE RULES-----------------------------------------------------
17464 // These must follow all instruction definitions as they use the names
17465 // defined in the instructions definitions.
17466 //
17467 // peepmatch ( root_instr_name [preceding_instruction]* );
17468 //
17469 // peepconstraint %{
17470 // (instruction_number.operand_name relational_op instruction_number.operand_name
17471 // [, ...] );
17472 // // instruction numbers are zero-based using left to right order in peepmatch
17473 //
17474 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17475 // // provide an instruction_number.operand_name for each operand that appears
17476 // // in the replacement instruction's match rule
17477 //
17478 // ---------VM FLAGS---------------------------------------------------------
17479 //
17480 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17481 //
17482 // Each peephole rule is given an identifying number starting with zero and
17483 // increasing by one in the order seen by the parser. An individual peephole
17484 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17485 // on the command-line.
17486 //
17487 // ---------CURRENT LIMITATIONS----------------------------------------------
17488 //
17489 // Only match adjacent instructions in same basic block
17490 // Only equality constraints
17491 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17492 // Only one replacement instruction
17493 //
17494 // ---------EXAMPLE----------------------------------------------------------
17495 //
17496 // // pertinent parts of existing instructions in architecture description
17497 // instruct movI(iRegINoSp dst, iRegI src)
17498 // %{
17499 // match(Set dst (CopyI src));
17500 // %}
17501 //
17502 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17503 // %{
17504 // match(Set dst (AddI dst src));
17505 // effect(KILL cr);
17506 // %}
17507 //
17508 // // Change (inc mov) to lea
17509 // peephole %{
17510 // // increment preceded by register-register move
17511 // peepmatch ( incI_iReg movI );
17512 // // require that the destination register of the increment
17513 // // match the destination register of the move
17514 // peepconstraint ( 0.dst == 1.dst );
17515 // // construct a replacement instruction that sets
17516 // // the destination to ( move's source register + one )
17517 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17518 // %}
17519 //
17520
17521 // Implementation no longer uses movX instructions since
17522 // machine-independent system no longer uses CopyX nodes.
17523 //
17524 // peephole
17525 // %{
17526 // peepmatch (incI_iReg movI);
17527 // peepconstraint (0.dst == 1.dst);
17528 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17529 // %}
17530
17531 // peephole
17532 // %{
17533 // peepmatch (decI_iReg movI);
17534 // peepconstraint (0.dst == 1.dst);
17535 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17536 // %}
17537
17538 // peephole
17539 // %{
17540 // peepmatch (addI_iReg_imm movI);
17541 // peepconstraint (0.dst == 1.dst);
17542 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17543 // %}
17544
17545 // peephole
17546 // %{
17547 // peepmatch (incL_iReg movL);
17548 // peepconstraint (0.dst == 1.dst);
17549 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17550 // %}
17551
17552 // peephole
17553 // %{
17554 // peepmatch (decL_iReg movL);
17555 // peepconstraint (0.dst == 1.dst);
17556 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17557 // %}
17558
17559 // peephole
17560 // %{
17561 // peepmatch (addL_iReg_imm movL);
17562 // peepconstraint (0.dst == 1.dst);
17563 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17564 // %}
17565
17566 // peephole
17567 // %{
17568 // peepmatch (addP_iReg_imm movP);
17569 // peepconstraint (0.dst == 1.dst);
17570 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17571 // %}
17572
17573 // // Change load of spilled value to only a spill
17574 // instruct storeI(memory mem, iRegI src)
17575 // %{
17576 // match(Set mem (StoreI mem src));
17577 // %}
17578 //
17579 // instruct loadI(iRegINoSp dst, memory mem)
17580 // %{
17581 // match(Set dst (LoadI mem));
17582 // %}
17583 //
17584
17585 //----------SMARTSPILL RULES---------------------------------------------------
17586 // These must follow all instruction definitions as they use the names
17587 // defined in the instructions definitions.
17588
17589 // Local Variables:
17590 // mode: c++
17591 // End: