1 //
2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::ptrs_base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // lea(rscratch1, RuntimeAddress(addr)
1652 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else {
1658 return 6 * NativeInstruction::instruction_size;
1659 }
1660 }
1661
1662 //=============================================================================
1663
1664 #ifndef PRODUCT
1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1666 st->print("BREAKPOINT");
1667 }
1668 #endif
1669
1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1671 __ brk(0);
1672 }
1673
1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1675 return MachNode::size(ra_);
1676 }
1677
1678 //=============================================================================
1679
1680 #ifndef PRODUCT
1681 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1682 st->print("nop \t# %d bytes pad for loops and calls", _count);
1683 }
1684 #endif
1685
1686 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1687 for (int i = 0; i < _count; i++) {
1688 __ nop();
1689 }
1690 }
1691
1692 uint MachNopNode::size(PhaseRegAlloc*) const {
1693 return _count * NativeInstruction::instruction_size;
1694 }
1695
1696 //=============================================================================
1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1698
1699 int ConstantTable::calculate_table_base_offset() const {
1700 return 0; // absolute addressing, no offset
1701 }
1702
1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1705 ShouldNotReachHere();
1706 }
1707
1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1709 // Empty encoding
1710 }
1711
1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1713 return 0;
1714 }
1715
1716 #ifndef PRODUCT
1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1718 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1719 }
1720 #endif
1721
1722 #ifndef PRODUCT
1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1724 Compile* C = ra_->C;
1725
1726 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1727
1728 if (C->output()->need_stack_bang(framesize))
1729 st->print("# stack bang size=%d\n\t", framesize);
1730
1731 if (VM_Version::use_rop_protection()) {
1732 st->print("ldr zr, [lr]\n\t");
1733 st->print("paciaz\n\t");
1734 }
1735 if (framesize < ((1 << 9) + 2 * wordSize)) {
1736 st->print("sub sp, sp, #%d\n\t", framesize);
1737 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1738 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1739 } else {
1740 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1741 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1742 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1743 st->print("sub sp, sp, rscratch1");
1744 }
1745 if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1746 st->print("\n\t");
1747 st->print("ldr rscratch1, [guard]\n\t");
1748 st->print("dmb ishld\n\t");
1749 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1750 st->print("cmp rscratch1, rscratch2\n\t");
1751 st->print("b.eq skip");
1752 st->print("\n\t");
1753 st->print("blr #nmethod_entry_barrier_stub\n\t");
1754 st->print("b skip\n\t");
1755 st->print("guard: int\n\t");
1756 st->print("\n\t");
1757 st->print("skip:\n\t");
1758 }
1759 }
1760 #endif
1761
1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1763 Compile* C = ra_->C;
1764
1765 // n.b. frame size includes space for return pc and rfp
1766 const int framesize = C->output()->frame_size_in_bytes();
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 if (C->clinit_barrier_on_entry()) {
1773 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1774
1775 Label L_skip_barrier;
1776
1777 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1778 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1779 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1780 __ bind(L_skip_barrier);
1781 }
1782
1783 if (C->max_vector_size() > 0) {
1784 __ reinitialize_ptrue();
1785 }
1786
1787 int bangsize = C->output()->bang_size_in_bytes();
1788 if (C->output()->need_stack_bang(bangsize))
1789 __ generate_stack_overflow_check(bangsize);
1790
1791 __ build_frame(framesize);
1792
1793 if (C->stub_function() == nullptr) {
1794 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1795 if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1796 // Dummy labels for just measuring the code size
1797 Label dummy_slow_path;
1798 Label dummy_continuation;
1799 Label dummy_guard;
1800 Label* slow_path = &dummy_slow_path;
1801 Label* continuation = &dummy_continuation;
1802 Label* guard = &dummy_guard;
1803 if (!Compile::current()->output()->in_scratch_emit_size()) {
1804 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1805 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1806 Compile::current()->output()->add_stub(stub);
1807 slow_path = &stub->entry();
1808 continuation = &stub->continuation();
1809 guard = &stub->guard();
1810 }
1811 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1812 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1813 }
1814 }
1815
1816 if (VerifyStackAtCalls) {
1817 Unimplemented();
1818 }
1819
1820 C->output()->set_frame_complete(__ offset());
1821
1822 if (C->has_mach_constant_base_node()) {
1823 // NOTE: We set the table base offset here because users might be
1824 // emitted before MachConstantBaseNode.
1825 ConstantTable& constant_table = C->output()->constant_table();
1826 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1827 }
1828 }
1829
1830 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1831 {
1832 return MachNode::size(ra_); // too many variables; just compute it
1833 // the hard way
1834 }
1835
1836 int MachPrologNode::reloc() const
1837 {
1838 return 0;
1839 }
1840
1841 //=============================================================================
1842
1843 #ifndef PRODUCT
1844 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1845 Compile* C = ra_->C;
1846 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1847
1848 st->print("# pop frame %d\n\t",framesize);
1849
1850 if (framesize == 0) {
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1853 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1854 st->print("add sp, sp, #%d\n\t", framesize);
1855 } else {
1856 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1857 st->print("add sp, sp, rscratch1\n\t");
1858 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1859 }
1860 if (VM_Version::use_rop_protection()) {
1861 st->print("autiaz\n\t");
1862 st->print("ldr zr, [lr]\n\t");
1863 }
1864
1865 if (do_polling() && C->is_method_compilation()) {
1866 st->print("# test polling word\n\t");
1867 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1868 st->print("cmp sp, rscratch1\n\t");
1869 st->print("bhi #slow_path");
1870 }
1871 }
1872 #endif
1873
1874 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 __ remove_frame(framesize);
1879
1880 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1881 __ reserved_stack_check();
1882 }
1883
1884 if (do_polling() && C->is_method_compilation()) {
1885 Label dummy_label;
1886 Label* code_stub = &dummy_label;
1887 if (!C->output()->in_scratch_emit_size()) {
1888 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1889 C->output()->add_stub(stub);
1890 code_stub = &stub->entry();
1891 }
1892 __ relocate(relocInfo::poll_return_type);
1893 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1894 }
1895 }
1896
1897 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1898 // Variable size. Determine dynamically.
1899 return MachNode::size(ra_);
1900 }
1901
1902 int MachEpilogNode::reloc() const {
1903 // Return number of relocatable values contained in this instruction.
1904 return 1; // 1 for polling page.
1905 }
1906
1907 const Pipeline * MachEpilogNode::pipeline() const {
1908 return MachNode::pipeline_class();
1909 }
1910
1911 //=============================================================================
1912
1913 static enum RC rc_class(OptoReg::Name reg) {
1914
1915 if (reg == OptoReg::Bad) {
1916 return rc_bad;
1917 }
1918
1919 // we have 32 int registers * 2 halves
1920 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1921
1922 if (reg < slots_of_int_registers) {
1923 return rc_int;
1924 }
1925
1926 // we have 32 float register * 8 halves
1927 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1928 if (reg < slots_of_int_registers + slots_of_float_registers) {
1929 return rc_float;
1930 }
1931
1932 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1933 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1934 return rc_predicate;
1935 }
1936
1937 // Between predicate regs & stack is the flags.
1938 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1939
1940 return rc_stack;
1941 }
1942
1943 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1944 Compile* C = ra_->C;
1945
1946 // Get registers to move.
1947 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1948 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1949 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1950 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1951
1952 enum RC src_hi_rc = rc_class(src_hi);
1953 enum RC src_lo_rc = rc_class(src_lo);
1954 enum RC dst_hi_rc = rc_class(dst_hi);
1955 enum RC dst_lo_rc = rc_class(dst_lo);
1956
1957 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1958
1959 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1960 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1961 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1962 "expected aligned-adjacent pairs");
1963 }
1964
1965 if (src_lo == dst_lo && src_hi == dst_hi) {
1966 return 0; // Self copy, no move.
1967 }
1968
1969 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1970 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1971 int src_offset = ra_->reg2offset(src_lo);
1972 int dst_offset = ra_->reg2offset(dst_lo);
1973
1974 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1975 uint ireg = ideal_reg();
1976 if (ireg == Op_VecA && masm) {
1977 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1978 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1979 // stack->stack
1980 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1981 sve_vector_reg_size_in_bytes);
1982 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1983 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1984 sve_vector_reg_size_in_bytes);
1985 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1986 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1987 sve_vector_reg_size_in_bytes);
1988 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1989 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1990 as_FloatRegister(Matcher::_regEncode[src_lo]),
1991 as_FloatRegister(Matcher::_regEncode[src_lo]));
1992 } else {
1993 ShouldNotReachHere();
1994 }
1995 } else if (masm) {
1996 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1997 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1998 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1999 // stack->stack
2000 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2001 if (ireg == Op_VecD) {
2002 __ unspill(rscratch1, true, src_offset);
2003 __ spill(rscratch1, true, dst_offset);
2004 } else {
2005 __ spill_copy128(src_offset, dst_offset);
2006 }
2007 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2008 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2009 ireg == Op_VecD ? __ T8B : __ T16B,
2010 as_FloatRegister(Matcher::_regEncode[src_lo]));
2011 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2012 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2013 ireg == Op_VecD ? __ D : __ Q,
2014 ra_->reg2offset(dst_lo));
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 ireg == Op_VecD ? __ D : __ Q,
2018 ra_->reg2offset(src_lo));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 }
2023 } else if (masm) {
2024 switch (src_lo_rc) {
2025 case rc_int:
2026 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2027 if (is64) {
2028 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2029 as_Register(Matcher::_regEncode[src_lo]));
2030 } else {
2031 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2032 as_Register(Matcher::_regEncode[src_lo]));
2033 }
2034 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2035 if (is64) {
2036 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2037 as_Register(Matcher::_regEncode[src_lo]));
2038 } else {
2039 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2040 as_Register(Matcher::_regEncode[src_lo]));
2041 }
2042 } else { // gpr --> stack spill
2043 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2044 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2045 }
2046 break;
2047 case rc_float:
2048 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2049 if (is64) {
2050 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2051 as_FloatRegister(Matcher::_regEncode[src_lo]));
2052 } else {
2053 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2054 as_FloatRegister(Matcher::_regEncode[src_lo]));
2055 }
2056 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2057 if (is64) {
2058 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2059 as_FloatRegister(Matcher::_regEncode[src_lo]));
2060 } else {
2061 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2062 as_FloatRegister(Matcher::_regEncode[src_lo]));
2063 }
2064 } else { // fpr --> stack spill
2065 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2066 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2067 is64 ? __ D : __ S, dst_offset);
2068 }
2069 break;
2070 case rc_stack:
2071 if (dst_lo_rc == rc_int) { // stack --> gpr load
2072 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2073 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2074 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2075 is64 ? __ D : __ S, src_offset);
2076 } else if (dst_lo_rc == rc_predicate) {
2077 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2078 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2079 } else { // stack --> stack copy
2080 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2081 if (ideal_reg() == Op_RegVectMask) {
2082 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2083 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2084 } else {
2085 __ unspill(rscratch1, is64, src_offset);
2086 __ spill(rscratch1, is64, dst_offset);
2087 }
2088 }
2089 break;
2090 case rc_predicate:
2091 if (dst_lo_rc == rc_predicate) {
2092 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2093 } else if (dst_lo_rc == rc_stack) {
2094 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2095 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2096 } else {
2097 assert(false, "bad src and dst rc_class combination.");
2098 ShouldNotReachHere();
2099 }
2100 break;
2101 default:
2102 assert(false, "bad rc_class for spill");
2103 ShouldNotReachHere();
2104 }
2105 }
2106
2107 if (st) {
2108 st->print("spill ");
2109 if (src_lo_rc == rc_stack) {
2110 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2111 } else {
2112 st->print("%s -> ", Matcher::regName[src_lo]);
2113 }
2114 if (dst_lo_rc == rc_stack) {
2115 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2116 } else {
2117 st->print("%s", Matcher::regName[dst_lo]);
2118 }
2119 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2120 int vsize = 0;
2121 switch (ideal_reg()) {
2122 case Op_VecD:
2123 vsize = 64;
2124 break;
2125 case Op_VecX:
2126 vsize = 128;
2127 break;
2128 case Op_VecA:
2129 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2130 break;
2131 default:
2132 assert(false, "bad register type for spill");
2133 ShouldNotReachHere();
2134 }
2135 st->print("\t# vector spill size = %d", vsize);
2136 } else if (ideal_reg() == Op_RegVectMask) {
2137 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2138 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2139 st->print("\t# predicate spill size = %d", vsize);
2140 } else {
2141 st->print("\t# spill size = %d", is64 ? 64 : 32);
2142 }
2143 }
2144
2145 return 0;
2146
2147 }
2148
2149 #ifndef PRODUCT
2150 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2151 if (!ra_)
2152 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2153 else
2154 implementation(nullptr, ra_, false, st);
2155 }
2156 #endif
2157
2158 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2159 implementation(masm, ra_, false, nullptr);
2160 }
2161
2162 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2163 return MachNode::size(ra_);
2164 }
2165
2166 //=============================================================================
2167
2168 #ifndef PRODUCT
2169 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2170 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2171 int reg = ra_->get_reg_first(this);
2172 st->print("add %s, rsp, #%d]\t# box lock",
2173 Matcher::regName[reg], offset);
2174 }
2175 #endif
2176
2177 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2178 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2179 int reg = ra_->get_encode(this);
2180
2181 // This add will handle any 24-bit signed offset. 24 bits allows an
2182 // 8 megabyte stack frame.
2183 __ add(as_Register(reg), sp, offset);
2184 }
2185
2186 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2187 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2188 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2189
2190 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2191 return NativeInstruction::instruction_size;
2192 } else {
2193 return 2 * NativeInstruction::instruction_size;
2194 }
2195 }
2196
2197 //=============================================================================
2198
2199 #ifndef PRODUCT
2200 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2201 {
2202 st->print_cr("# MachUEPNode");
2203 if (UseCompressedClassPointers) {
2204 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2205 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2206 st->print_cr("\tcmpw rscratch1, r10");
2207 } else {
2208 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2209 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2210 st->print_cr("\tcmp rscratch1, r10");
2211 }
2212 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2213 }
2214 #endif
2215
2216 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2217 {
2218 __ ic_check(InteriorEntryAlignment);
2219 }
2220
2221 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2222 {
2223 return MachNode::size(ra_);
2224 }
2225
2226 // REQUIRED EMIT CODE
2227
2228 //=============================================================================
2229
2230 // Emit exception handler code.
2231 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2232 {
2233 // mov rscratch1 #exception_blob_entry_point
2234 // br rscratch1
2235 // Note that the code buffer's insts_mark is always relative to insts.
2236 // That's why we must use the macroassembler to generate a handler.
2237 address base = __ start_a_stub(size_exception_handler());
2238 if (base == nullptr) {
2239 ciEnv::current()->record_failure("CodeCache is full");
2240 return 0; // CodeBuffer::expand failed
2241 }
2242 int offset = __ offset();
2243 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2244 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2245 __ end_a_stub();
2246 return offset;
2247 }
2248
2249 // Emit deopt handler code.
2250 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2251 {
2252 // Note that the code buffer's insts_mark is always relative to insts.
2253 // That's why we must use the macroassembler to generate a handler.
2254 address base = __ start_a_stub(size_deopt_handler());
2255 if (base == nullptr) {
2256 ciEnv::current()->record_failure("CodeCache is full");
2257 return 0; // CodeBuffer::expand failed
2258 }
2259 int offset = __ offset();
2260
2261 __ adr(lr, __ pc());
2262 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2263
2264 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2265 __ end_a_stub();
2266 return offset;
2267 }
2268
2269 // REQUIRED MATCHER CODE
2270
2271 //=============================================================================
2272
2273 bool Matcher::match_rule_supported(int opcode) {
2274 if (!has_match_rule(opcode))
2275 return false;
2276
2277 switch (opcode) {
2278 case Op_OnSpinWait:
2279 return VM_Version::supports_on_spin_wait();
2280 case Op_CacheWB:
2281 case Op_CacheWBPreSync:
2282 case Op_CacheWBPostSync:
2283 if (!VM_Version::supports_data_cache_line_flush()) {
2284 return false;
2285 }
2286 break;
2287 case Op_ExpandBits:
2288 case Op_CompressBits:
2289 if (!VM_Version::supports_svebitperm()) {
2290 return false;
2291 }
2292 break;
2293 case Op_FmaF:
2294 case Op_FmaD:
2295 case Op_FmaVF:
2296 case Op_FmaVD:
2297 if (!UseFMA) {
2298 return false;
2299 }
2300 break;
2301 }
2302
2303 return true; // Per default match rules are supported.
2304 }
2305
2306 const RegMask* Matcher::predicate_reg_mask(void) {
2307 return &_PR_REG_mask;
2308 }
2309
2310 const TypeVectMask* Matcher::predicate_reg_type(const Type* elemTy, int length) {
2311 return new TypeVectMask(elemTy, length);
2312 }
2313
2314 // Vector calling convention not yet implemented.
2315 bool Matcher::supports_vector_calling_convention(void) {
2316 return false;
2317 }
2318
2319 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2320 Unimplemented();
2321 return OptoRegPair(0, 0);
2322 }
2323
2324 // Is this branch offset short enough that a short branch can be used?
2325 //
2326 // NOTE: If the platform does not provide any short branch variants, then
2327 // this method should return false for offset 0.
2328 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2329 // The passed offset is relative to address of the branch.
2330
2331 return (-32768 <= offset && offset < 32768);
2332 }
2333
2334 // Vector width in bytes.
2335 int Matcher::vector_width_in_bytes(BasicType bt) {
2336 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2337 int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize);
2338 // Minimum 2 values in vector
2339 if (size < 2*type2aelembytes(bt)) size = 0;
2340 // But never < 4
2341 if (size < 4) size = 0;
2342 return size;
2343 }
2344
2345 // Limits on vector size (number of elements) loaded into vector.
2346 int Matcher::max_vector_size(const BasicType bt) {
2347 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2348 }
2349
2350 int Matcher::min_vector_size(const BasicType bt) {
2351 int max_size = max_vector_size(bt);
2352 // Limit the min vector size to 8 bytes.
2353 int size = 8 / type2aelembytes(bt);
2354 if (bt == T_BYTE) {
2355 // To support vector api shuffle/rearrange.
2356 size = 4;
2357 } else if (bt == T_BOOLEAN) {
2358 // To support vector api load/store mask.
2359 size = 2;
2360 }
2361 if (size < 2) size = 2;
2362 return MIN2(size, max_size);
2363 }
2364
2365 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2366 return Matcher::max_vector_size(bt);
2367 }
2368
2369 // Actual max scalable vector register length.
2370 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2371 return Matcher::max_vector_size(bt);
2372 }
2373
2374 // Vector ideal reg.
2375 uint Matcher::vector_ideal_reg(int len) {
2376 if (UseSVE > 0 && 16 < len && len <= 256) {
2377 return Op_VecA;
2378 }
2379 switch(len) {
2380 // For 16-bit/32-bit mask vector, reuse VecD.
2381 case 2:
2382 case 4:
2383 case 8: return Op_VecD;
2384 case 16: return Op_VecX;
2385 }
2386 ShouldNotReachHere();
2387 return 0;
2388 }
2389
2390 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2391 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2392 switch (ideal_reg) {
2393 case Op_VecA: return new vecAOper();
2394 case Op_VecD: return new vecDOper();
2395 case Op_VecX: return new vecXOper();
2396 }
2397 ShouldNotReachHere();
2398 return nullptr;
2399 }
2400
2401 bool Matcher::is_reg2reg_move(MachNode* m) {
2402 return false;
2403 }
2404
2405 bool Matcher::is_generic_vector(MachOper* opnd) {
2406 return opnd->opcode() == VREG;
2407 }
2408
2409 // Return whether or not this register is ever used as an argument.
2410 // This function is used on startup to build the trampoline stubs in
2411 // generateOptoStub. Registers not mentioned will be killed by the VM
2412 // call in the trampoline, and arguments in those registers not be
2413 // available to the callee.
2414 bool Matcher::can_be_java_arg(int reg)
2415 {
2416 return
2417 reg == R0_num || reg == R0_H_num ||
2418 reg == R1_num || reg == R1_H_num ||
2419 reg == R2_num || reg == R2_H_num ||
2420 reg == R3_num || reg == R3_H_num ||
2421 reg == R4_num || reg == R4_H_num ||
2422 reg == R5_num || reg == R5_H_num ||
2423 reg == R6_num || reg == R6_H_num ||
2424 reg == R7_num || reg == R7_H_num ||
2425 reg == V0_num || reg == V0_H_num ||
2426 reg == V1_num || reg == V1_H_num ||
2427 reg == V2_num || reg == V2_H_num ||
2428 reg == V3_num || reg == V3_H_num ||
2429 reg == V4_num || reg == V4_H_num ||
2430 reg == V5_num || reg == V5_H_num ||
2431 reg == V6_num || reg == V6_H_num ||
2432 reg == V7_num || reg == V7_H_num;
2433 }
2434
2435 bool Matcher::is_spillable_arg(int reg)
2436 {
2437 return can_be_java_arg(reg);
2438 }
2439
2440 uint Matcher::int_pressure_limit()
2441 {
2442 // JDK-8183543: When taking the number of available registers as int
2443 // register pressure threshold, the jtreg test:
2444 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2445 // failed due to C2 compilation failure with
2446 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2447 //
2448 // A derived pointer is live at CallNode and then is flagged by RA
2449 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2450 // derived pointers and lastly fail to spill after reaching maximum
2451 // number of iterations. Lowering the default pressure threshold to
2452 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2453 // a high register pressure area of the code so that split_DEF can
2454 // generate DefinitionSpillCopy for the derived pointer.
2455 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2456 if (!PreserveFramePointer) {
2457 // When PreserveFramePointer is off, frame pointer is allocatable,
2458 // but different from other SOC registers, it is excluded from
2459 // fatproj's mask because its save type is No-Save. Decrease 1 to
2460 // ensure high pressure at fatproj when PreserveFramePointer is off.
2461 // See check_pressure_at_fatproj().
2462 default_int_pressure_threshold--;
2463 }
2464 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2465 }
2466
2467 uint Matcher::float_pressure_limit()
2468 {
2469 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2470 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2471 }
2472
2473 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2474 return false;
2475 }
2476
2477 RegMask Matcher::divI_proj_mask() {
2478 ShouldNotReachHere();
2479 return RegMask();
2480 }
2481
2482 // Register for MODI projection of divmodI.
2483 RegMask Matcher::modI_proj_mask() {
2484 ShouldNotReachHere();
2485 return RegMask();
2486 }
2487
2488 // Register for DIVL projection of divmodL.
2489 RegMask Matcher::divL_proj_mask() {
2490 ShouldNotReachHere();
2491 return RegMask();
2492 }
2493
2494 // Register for MODL projection of divmodL.
2495 RegMask Matcher::modL_proj_mask() {
2496 ShouldNotReachHere();
2497 return RegMask();
2498 }
2499
2500 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2501 return FP_REG_mask();
2502 }
2503
2504 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2505 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2506 Node* u = addp->fast_out(i);
2507 if (u->is_LoadStore()) {
2508 // On AArch64, LoadStoreNodes (i.e. compare and swap
2509 // instructions) only take register indirect as an operand, so
2510 // any attempt to use an AddPNode as an input to a LoadStoreNode
2511 // must fail.
2512 return false;
2513 }
2514 if (u->is_Mem()) {
2515 int opsize = u->as_Mem()->memory_size();
2516 assert(opsize > 0, "unexpected memory operand size");
2517 if (u->as_Mem()->memory_size() != (1<<shift)) {
2518 return false;
2519 }
2520 }
2521 }
2522 return true;
2523 }
2524
2525 // Convert BootTest condition to Assembler condition.
2526 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2527 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2528 Assembler::Condition result;
2529 switch(cond) {
2530 case BoolTest::eq:
2531 result = Assembler::EQ; break;
2532 case BoolTest::ne:
2533 result = Assembler::NE; break;
2534 case BoolTest::le:
2535 result = Assembler::LE; break;
2536 case BoolTest::ge:
2537 result = Assembler::GE; break;
2538 case BoolTest::lt:
2539 result = Assembler::LT; break;
2540 case BoolTest::gt:
2541 result = Assembler::GT; break;
2542 case BoolTest::ule:
2543 result = Assembler::LS; break;
2544 case BoolTest::uge:
2545 result = Assembler::HS; break;
2546 case BoolTest::ult:
2547 result = Assembler::LO; break;
2548 case BoolTest::ugt:
2549 result = Assembler::HI; break;
2550 case BoolTest::overflow:
2551 result = Assembler::VS; break;
2552 case BoolTest::no_overflow:
2553 result = Assembler::VC; break;
2554 default:
2555 ShouldNotReachHere();
2556 return Assembler::Condition(-1);
2557 }
2558
2559 // Check conversion
2560 if (cond & BoolTest::unsigned_compare) {
2561 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2562 } else {
2563 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2564 }
2565
2566 return result;
2567 }
2568
2569 // Binary src (Replicate con)
2570 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2571 if (n == nullptr || m == nullptr) {
2572 return false;
2573 }
2574
2575 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2576 return false;
2577 }
2578
2579 Node* imm_node = m->in(1);
2580 if (!imm_node->is_Con()) {
2581 return false;
2582 }
2583
2584 const Type* t = imm_node->bottom_type();
2585 if (!(t->isa_int() || t->isa_long())) {
2586 return false;
2587 }
2588
2589 switch (n->Opcode()) {
2590 case Op_AndV:
2591 case Op_OrV:
2592 case Op_XorV: {
2593 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2594 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2595 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2596 }
2597 case Op_AddVB:
2598 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2599 case Op_AddVS:
2600 case Op_AddVI:
2601 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2602 case Op_AddVL:
2603 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2604 default:
2605 return false;
2606 }
2607 }
2608
2609 // (XorV src (Replicate m1))
2610 // (XorVMask src (MaskAll m1))
2611 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2612 if (n != nullptr && m != nullptr) {
2613 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2614 VectorNode::is_all_ones_vector(m);
2615 }
2616 return false;
2617 }
2618
2619 // Should the matcher clone input 'm' of node 'n'?
2620 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2621 if (is_vshift_con_pattern(n, m) ||
2622 is_vector_bitwise_not_pattern(n, m) ||
2623 is_valid_sve_arith_imm_pattern(n, m)) {
2624 mstack.push(m, Visit);
2625 return true;
2626 }
2627 return false;
2628 }
2629
2630 // Should the Matcher clone shifts on addressing modes, expecting them
2631 // to be subsumed into complex addressing expressions or compute them
2632 // into registers?
2633 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2634 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2635 return true;
2636 }
2637
2638 Node *off = m->in(AddPNode::Offset);
2639 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2640 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2641 // Are there other uses besides address expressions?
2642 !is_visited(off)) {
2643 address_visited.set(off->_idx); // Flag as address_visited
2644 mstack.push(off->in(2), Visit);
2645 Node *conv = off->in(1);
2646 if (conv->Opcode() == Op_ConvI2L &&
2647 // Are there other uses besides address expressions?
2648 !is_visited(conv)) {
2649 address_visited.set(conv->_idx); // Flag as address_visited
2650 mstack.push(conv->in(1), Pre_Visit);
2651 } else {
2652 mstack.push(conv, Pre_Visit);
2653 }
2654 address_visited.test_set(m->_idx); // Flag as address_visited
2655 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2656 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2657 return true;
2658 } else if (off->Opcode() == Op_ConvI2L &&
2659 // Are there other uses besides address expressions?
2660 !is_visited(off)) {
2661 address_visited.test_set(m->_idx); // Flag as address_visited
2662 address_visited.set(off->_idx); // Flag as address_visited
2663 mstack.push(off->in(1), Pre_Visit);
2664 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2665 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2666 return true;
2667 }
2668 return false;
2669 }
2670
2671 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2672 { \
2673 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2674 guarantee(DISP == 0, "mode not permitted for volatile"); \
2675 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2676 __ INSN(REG, as_Register(BASE)); \
2677 }
2678
2679
2680 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2681 {
2682 Address::extend scale;
2683
2684 // Hooboy, this is fugly. We need a way to communicate to the
2685 // encoder that the index needs to be sign extended, so we have to
2686 // enumerate all the cases.
2687 switch (opcode) {
2688 case INDINDEXSCALEDI2L:
2689 case INDINDEXSCALEDI2LN:
2690 case INDINDEXI2L:
2691 case INDINDEXI2LN:
2692 scale = Address::sxtw(size);
2693 break;
2694 default:
2695 scale = Address::lsl(size);
2696 }
2697
2698 if (index == -1) {
2699 return Address(base, disp);
2700 } else {
2701 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2702 return Address(base, as_Register(index), scale);
2703 }
2704 }
2705
2706
2707 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2708 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2709 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2710 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2711 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2712
2713 // Used for all non-volatile memory accesses. The use of
2714 // $mem->opcode() to discover whether this pattern uses sign-extended
2715 // offsets is something of a kludge.
2716 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2717 Register reg, int opcode,
2718 Register base, int index, int scale, int disp,
2719 int size_in_memory)
2720 {
2721 Address addr = mem2address(opcode, base, index, scale, disp);
2722 if (addr.getMode() == Address::base_plus_offset) {
2723 /* Fix up any out-of-range offsets. */
2724 assert_different_registers(rscratch1, base);
2725 assert_different_registers(rscratch1, reg);
2726 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2727 }
2728 (masm->*insn)(reg, addr);
2729 }
2730
2731 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2732 FloatRegister reg, int opcode,
2733 Register base, int index, int size, int disp,
2734 int size_in_memory)
2735 {
2736 Address::extend scale;
2737
2738 switch (opcode) {
2739 case INDINDEXSCALEDI2L:
2740 case INDINDEXSCALEDI2LN:
2741 scale = Address::sxtw(size);
2742 break;
2743 default:
2744 scale = Address::lsl(size);
2745 }
2746
2747 if (index == -1) {
2748 /* If we get an out-of-range offset it is a bug in the compiler,
2749 so we assert here. */
2750 assert(Address::offset_ok_for_immed(disp, exact_log2(size_in_memory)), "c2 compiler bug");
2751 /* Fix up any out-of-range offsets. */
2752 assert_different_registers(rscratch1, base);
2753 Address addr = Address(base, disp);
2754 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2755 (masm->*insn)(reg, addr);
2756 } else {
2757 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2758 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2759 }
2760 }
2761
2762 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2763 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2764 int opcode, Register base, int index, int size, int disp)
2765 {
2766 if (index == -1) {
2767 (masm->*insn)(reg, T, Address(base, disp));
2768 } else {
2769 assert(disp == 0, "unsupported address mode");
2770 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2771 }
2772 }
2773
2774 %}
2775
2776
2777
2778 //----------ENCODING BLOCK-----------------------------------------------------
2779 // This block specifies the encoding classes used by the compiler to
2780 // output byte streams. Encoding classes are parameterized macros
2781 // used by Machine Instruction Nodes in order to generate the bit
2782 // encoding of the instruction. Operands specify their base encoding
2783 // interface with the interface keyword. There are currently
2784 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2785 // COND_INTER. REG_INTER causes an operand to generate a function
2786 // which returns its register number when queried. CONST_INTER causes
2787 // an operand to generate a function which returns the value of the
2788 // constant when queried. MEMORY_INTER causes an operand to generate
2789 // four functions which return the Base Register, the Index Register,
2790 // the Scale Value, and the Offset Value of the operand when queried.
2791 // COND_INTER causes an operand to generate six functions which return
2792 // the encoding code (ie - encoding bits for the instruction)
2793 // associated with each basic boolean condition for a conditional
2794 // instruction.
2795 //
2796 // Instructions specify two basic values for encoding. Again, a
2797 // function is available to check if the constant displacement is an
2798 // oop. They use the ins_encode keyword to specify their encoding
2799 // classes (which must be a sequence of enc_class names, and their
2800 // parameters, specified in the encoding block), and they use the
2801 // opcode keyword to specify, in order, their primary, secondary, and
2802 // tertiary opcode. Only the opcode sections which a particular
2803 // instruction needs for encoding need to be specified.
2804 encode %{
2805 // Build emit functions for each basic byte or larger field in the
2806 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2807 // from C++ code in the enc_class source block. Emit functions will
2808 // live in the main source block for now. In future, we can
2809 // generalize this by adding a syntax that specifies the sizes of
2810 // fields in an order, so that the adlc can build the emit functions
2811 // automagically
2812
2813 // catch all for unimplemented encodings
2814 enc_class enc_unimplemented %{
2815 __ unimplemented("C2 catch all");
2816 %}
2817
2818 // BEGIN Non-volatile memory access
2819
2820 // This encoding class is generated automatically from ad_encode.m4.
2821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2822 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2823 Register dst_reg = as_Register($dst$$reg);
2824 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2825 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2826 %}
2827
2828 // This encoding class is generated automatically from ad_encode.m4.
2829 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2830 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2831 Register dst_reg = as_Register($dst$$reg);
2832 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2833 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2834 %}
2835
2836 // This encoding class is generated automatically from ad_encode.m4.
2837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2838 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2839 Register dst_reg = as_Register($dst$$reg);
2840 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2841 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2842 %}
2843
2844 // This encoding class is generated automatically from ad_encode.m4.
2845 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2846 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2847 Register dst_reg = as_Register($dst$$reg);
2848 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2849 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2850 %}
2851
2852 // This encoding class is generated automatically from ad_encode.m4.
2853 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2854 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2855 Register dst_reg = as_Register($dst$$reg);
2856 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2857 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2858 %}
2859
2860 // This encoding class is generated automatically from ad_encode.m4.
2861 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2862 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2863 Register dst_reg = as_Register($dst$$reg);
2864 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2865 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2866 %}
2867
2868 // This encoding class is generated automatically from ad_encode.m4.
2869 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2870 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2871 Register dst_reg = as_Register($dst$$reg);
2872 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2873 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2874 %}
2875
2876 // This encoding class is generated automatically from ad_encode.m4.
2877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2878 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2879 Register dst_reg = as_Register($dst$$reg);
2880 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2881 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2882 %}
2883
2884 // This encoding class is generated automatically from ad_encode.m4.
2885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2886 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2887 Register dst_reg = as_Register($dst$$reg);
2888 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2889 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2890 %}
2891
2892 // This encoding class is generated automatically from ad_encode.m4.
2893 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2894 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2895 Register dst_reg = as_Register($dst$$reg);
2896 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2897 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2898 %}
2899
2900 // This encoding class is generated automatically from ad_encode.m4.
2901 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2902 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2903 Register dst_reg = as_Register($dst$$reg);
2904 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2905 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2906 %}
2907
2908 // This encoding class is generated automatically from ad_encode.m4.
2909 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2910 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2911 Register dst_reg = as_Register($dst$$reg);
2912 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2913 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2914 %}
2915
2916 // This encoding class is generated automatically from ad_encode.m4.
2917 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2918 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2919 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2920 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2921 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2922 %}
2923
2924 // This encoding class is generated automatically from ad_encode.m4.
2925 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2926 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2927 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2928 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2929 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2930 %}
2931
2932 // This encoding class is generated automatically from ad_encode.m4.
2933 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2934 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2935 Register src_reg = as_Register($src$$reg);
2936 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2937 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2938 %}
2939
2940 // This encoding class is generated automatically from ad_encode.m4.
2941 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2942 enc_class aarch64_enc_strb0(memory1 mem) %{
2943 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2944 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2945 %}
2946
2947 // This encoding class is generated automatically from ad_encode.m4.
2948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2949 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2950 Register src_reg = as_Register($src$$reg);
2951 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2952 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2953 %}
2954
2955 // This encoding class is generated automatically from ad_encode.m4.
2956 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2957 enc_class aarch64_enc_strh0(memory2 mem) %{
2958 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2959 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2960 %}
2961
2962 // This encoding class is generated automatically from ad_encode.m4.
2963 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2964 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2965 Register src_reg = as_Register($src$$reg);
2966 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2967 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2968 %}
2969
2970 // This encoding class is generated automatically from ad_encode.m4.
2971 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2972 enc_class aarch64_enc_strw0(memory4 mem) %{
2973 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
2974 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2975 %}
2976
2977 // This encoding class is generated automatically from ad_encode.m4.
2978 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2979 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2980 Register src_reg = as_Register($src$$reg);
2981 // we sometimes get asked to store the stack pointer into the
2982 // current thread -- we cannot do that directly on AArch64
2983 if (src_reg == r31_sp) {
2984 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2985 __ mov(rscratch2, sp);
2986 src_reg = rscratch2;
2987 }
2988 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
2989 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2990 %}
2991
2992 // This encoding class is generated automatically from ad_encode.m4.
2993 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2994 enc_class aarch64_enc_str0(memory8 mem) %{
2995 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
2996 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2997 %}
2998
2999 // This encoding class is generated automatically from ad_encode.m4.
3000 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3001 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
3002 FloatRegister src_reg = as_FloatRegister($src$$reg);
3003 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
3004 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
3005 %}
3006
3007 // This encoding class is generated automatically from ad_encode.m4.
3008 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3009 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3010 FloatRegister src_reg = as_FloatRegister($src$$reg);
3011 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3012 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3013 %}
3014
3015 // This encoding class is generated automatically from ad_encode.m4.
3016 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3017 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3018 __ membar(Assembler::StoreStore);
3019 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3020 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3021 %}
3022
3023 // END Non-volatile memory access
3024
3025 // Vector loads and stores
3026 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3027 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3028 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3029 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3030 %}
3031
3032 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3033 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3034 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3035 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3036 %}
3037
3038 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3039 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3040 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3041 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3042 %}
3043
3044 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3045 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3046 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3047 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3048 %}
3049
3050 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3051 FloatRegister src_reg = as_FloatRegister($src$$reg);
3052 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3053 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3054 %}
3055
3056 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3057 FloatRegister src_reg = as_FloatRegister($src$$reg);
3058 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3059 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3060 %}
3061
3062 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3063 FloatRegister src_reg = as_FloatRegister($src$$reg);
3064 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3065 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3066 %}
3067
3068 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3069 FloatRegister src_reg = as_FloatRegister($src$$reg);
3070 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3071 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3072 %}
3073
3074 // volatile loads and stores
3075
3076 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3077 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3078 rscratch1, stlrb);
3079 %}
3080
3081 enc_class aarch64_enc_stlrb0(memory mem) %{
3082 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3083 rscratch1, stlrb);
3084 %}
3085
3086 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3087 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3088 rscratch1, stlrh);
3089 %}
3090
3091 enc_class aarch64_enc_stlrh0(memory mem) %{
3092 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3093 rscratch1, stlrh);
3094 %}
3095
3096 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3097 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3098 rscratch1, stlrw);
3099 %}
3100
3101 enc_class aarch64_enc_stlrw0(memory mem) %{
3102 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3103 rscratch1, stlrw);
3104 %}
3105
3106 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3107 Register dst_reg = as_Register($dst$$reg);
3108 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3109 rscratch1, ldarb);
3110 __ sxtbw(dst_reg, dst_reg);
3111 %}
3112
3113 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3114 Register dst_reg = as_Register($dst$$reg);
3115 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3116 rscratch1, ldarb);
3117 __ sxtb(dst_reg, dst_reg);
3118 %}
3119
3120 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3121 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3122 rscratch1, ldarb);
3123 %}
3124
3125 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3126 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3127 rscratch1, ldarb);
3128 %}
3129
3130 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3131 Register dst_reg = as_Register($dst$$reg);
3132 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3133 rscratch1, ldarh);
3134 __ sxthw(dst_reg, dst_reg);
3135 %}
3136
3137 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3138 Register dst_reg = as_Register($dst$$reg);
3139 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3140 rscratch1, ldarh);
3141 __ sxth(dst_reg, dst_reg);
3142 %}
3143
3144 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3145 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3146 rscratch1, ldarh);
3147 %}
3148
3149 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3150 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3151 rscratch1, ldarh);
3152 %}
3153
3154 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3155 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3156 rscratch1, ldarw);
3157 %}
3158
3159 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3160 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3161 rscratch1, ldarw);
3162 %}
3163
3164 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3165 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3166 rscratch1, ldar);
3167 %}
3168
3169 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3170 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldarw);
3172 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3173 %}
3174
3175 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3176 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3177 rscratch1, ldar);
3178 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3179 %}
3180
3181 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3182 Register src_reg = as_Register($src$$reg);
3183 // we sometimes get asked to store the stack pointer into the
3184 // current thread -- we cannot do that directly on AArch64
3185 if (src_reg == r31_sp) {
3186 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3187 __ mov(rscratch2, sp);
3188 src_reg = rscratch2;
3189 }
3190 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3191 rscratch1, stlr);
3192 %}
3193
3194 enc_class aarch64_enc_stlr0(memory mem) %{
3195 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3196 rscratch1, stlr);
3197 %}
3198
3199 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3200 {
3201 FloatRegister src_reg = as_FloatRegister($src$$reg);
3202 __ fmovs(rscratch2, src_reg);
3203 }
3204 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3205 rscratch1, stlrw);
3206 %}
3207
3208 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3209 {
3210 FloatRegister src_reg = as_FloatRegister($src$$reg);
3211 __ fmovd(rscratch2, src_reg);
3212 }
3213 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3214 rscratch1, stlr);
3215 %}
3216
3217 // synchronized read/update encodings
3218
3219 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3220 Register dst_reg = as_Register($dst$$reg);
3221 Register base = as_Register($mem$$base);
3222 int index = $mem$$index;
3223 int scale = $mem$$scale;
3224 int disp = $mem$$disp;
3225 if (index == -1) {
3226 if (disp != 0) {
3227 __ lea(rscratch1, Address(base, disp));
3228 __ ldaxr(dst_reg, rscratch1);
3229 } else {
3230 // TODO
3231 // should we ever get anything other than this case?
3232 __ ldaxr(dst_reg, base);
3233 }
3234 } else {
3235 Register index_reg = as_Register(index);
3236 if (disp == 0) {
3237 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3238 __ ldaxr(dst_reg, rscratch1);
3239 } else {
3240 __ lea(rscratch1, Address(base, disp));
3241 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3242 __ ldaxr(dst_reg, rscratch1);
3243 }
3244 }
3245 %}
3246
3247 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3248 Register src_reg = as_Register($src$$reg);
3249 Register base = as_Register($mem$$base);
3250 int index = $mem$$index;
3251 int scale = $mem$$scale;
3252 int disp = $mem$$disp;
3253 if (index == -1) {
3254 if (disp != 0) {
3255 __ lea(rscratch2, Address(base, disp));
3256 __ stlxr(rscratch1, src_reg, rscratch2);
3257 } else {
3258 // TODO
3259 // should we ever get anything other than this case?
3260 __ stlxr(rscratch1, src_reg, base);
3261 }
3262 } else {
3263 Register index_reg = as_Register(index);
3264 if (disp == 0) {
3265 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3266 __ stlxr(rscratch1, src_reg, rscratch2);
3267 } else {
3268 __ lea(rscratch2, Address(base, disp));
3269 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3270 __ stlxr(rscratch1, src_reg, rscratch2);
3271 }
3272 }
3273 __ cmpw(rscratch1, zr);
3274 %}
3275
3276 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3277 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3278 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3279 Assembler::xword, /*acquire*/ false, /*release*/ true,
3280 /*weak*/ false, noreg);
3281 %}
3282
3283 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3284 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3285 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3286 Assembler::word, /*acquire*/ false, /*release*/ true,
3287 /*weak*/ false, noreg);
3288 %}
3289
3290 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3291 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3292 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3293 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3294 /*weak*/ false, noreg);
3295 %}
3296
3297 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3298 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3299 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3300 Assembler::byte, /*acquire*/ false, /*release*/ true,
3301 /*weak*/ false, noreg);
3302 %}
3303
3304
3305 // The only difference between aarch64_enc_cmpxchg and
3306 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3307 // CompareAndSwap sequence to serve as a barrier on acquiring a
3308 // lock.
3309 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3310 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3311 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3312 Assembler::xword, /*acquire*/ true, /*release*/ true,
3313 /*weak*/ false, noreg);
3314 %}
3315
3316 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3317 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3318 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3319 Assembler::word, /*acquire*/ true, /*release*/ true,
3320 /*weak*/ false, noreg);
3321 %}
3322
3323 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3324 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3325 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3326 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3327 /*weak*/ false, noreg);
3328 %}
3329
3330 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3331 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3332 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3333 Assembler::byte, /*acquire*/ true, /*release*/ true,
3334 /*weak*/ false, noreg);
3335 %}
3336
3337 // auxiliary used for CompareAndSwapX to set result register
3338 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3339 Register res_reg = as_Register($res$$reg);
3340 __ cset(res_reg, Assembler::EQ);
3341 %}
3342
3343 // prefetch encodings
3344
3345 enc_class aarch64_enc_prefetchw(memory mem) %{
3346 Register base = as_Register($mem$$base);
3347 int index = $mem$$index;
3348 int scale = $mem$$scale;
3349 int disp = $mem$$disp;
3350 if (index == -1) {
3351 __ prfm(Address(base, disp), PSTL1KEEP);
3352 } else {
3353 Register index_reg = as_Register(index);
3354 if (disp == 0) {
3355 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3356 } else {
3357 __ lea(rscratch1, Address(base, disp));
3358 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3359 }
3360 }
3361 %}
3362
3363 /// mov envcodings
3364
3365 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3366 uint32_t con = (uint32_t)$src$$constant;
3367 Register dst_reg = as_Register($dst$$reg);
3368 if (con == 0) {
3369 __ movw(dst_reg, zr);
3370 } else {
3371 __ movw(dst_reg, con);
3372 }
3373 %}
3374
3375 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3376 Register dst_reg = as_Register($dst$$reg);
3377 uint64_t con = (uint64_t)$src$$constant;
3378 if (con == 0) {
3379 __ mov(dst_reg, zr);
3380 } else {
3381 __ mov(dst_reg, con);
3382 }
3383 %}
3384
3385 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3386 Register dst_reg = as_Register($dst$$reg);
3387 address con = (address)$src$$constant;
3388 if (con == nullptr || con == (address)1) {
3389 ShouldNotReachHere();
3390 } else {
3391 relocInfo::relocType rtype = $src->constant_reloc();
3392 if (rtype == relocInfo::oop_type) {
3393 __ movoop(dst_reg, (jobject)con);
3394 } else if (rtype == relocInfo::metadata_type) {
3395 __ mov_metadata(dst_reg, (Metadata*)con);
3396 } else {
3397 assert(rtype == relocInfo::none, "unexpected reloc type");
3398 if (! __ is_valid_AArch64_address(con) ||
3399 con < (address)(uintptr_t)os::vm_page_size()) {
3400 __ mov(dst_reg, con);
3401 } else {
3402 uint64_t offset;
3403 __ adrp(dst_reg, con, offset);
3404 __ add(dst_reg, dst_reg, offset);
3405 }
3406 }
3407 }
3408 %}
3409
3410 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3411 Register dst_reg = as_Register($dst$$reg);
3412 __ mov(dst_reg, zr);
3413 %}
3414
3415 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3416 Register dst_reg = as_Register($dst$$reg);
3417 __ mov(dst_reg, (uint64_t)1);
3418 %}
3419
3420 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3421 __ load_byte_map_base($dst$$Register);
3422 %}
3423
3424 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3425 Register dst_reg = as_Register($dst$$reg);
3426 address con = (address)$src$$constant;
3427 if (con == nullptr) {
3428 ShouldNotReachHere();
3429 } else {
3430 relocInfo::relocType rtype = $src->constant_reloc();
3431 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3432 __ set_narrow_oop(dst_reg, (jobject)con);
3433 }
3434 %}
3435
3436 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3437 Register dst_reg = as_Register($dst$$reg);
3438 __ mov(dst_reg, zr);
3439 %}
3440
3441 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3442 Register dst_reg = as_Register($dst$$reg);
3443 address con = (address)$src$$constant;
3444 if (con == nullptr) {
3445 ShouldNotReachHere();
3446 } else {
3447 relocInfo::relocType rtype = $src->constant_reloc();
3448 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3449 __ set_narrow_klass(dst_reg, (Klass *)con);
3450 }
3451 %}
3452
3453 // arithmetic encodings
3454
3455 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3456 Register dst_reg = as_Register($dst$$reg);
3457 Register src_reg = as_Register($src1$$reg);
3458 int32_t con = (int32_t)$src2$$constant;
3459 // add has primary == 0, subtract has primary == 1
3460 if ($primary) { con = -con; }
3461 if (con < 0) {
3462 __ subw(dst_reg, src_reg, -con);
3463 } else {
3464 __ addw(dst_reg, src_reg, con);
3465 }
3466 %}
3467
3468 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3469 Register dst_reg = as_Register($dst$$reg);
3470 Register src_reg = as_Register($src1$$reg);
3471 int32_t con = (int32_t)$src2$$constant;
3472 // add has primary == 0, subtract has primary == 1
3473 if ($primary) { con = -con; }
3474 if (con < 0) {
3475 __ sub(dst_reg, src_reg, -con);
3476 } else {
3477 __ add(dst_reg, src_reg, con);
3478 }
3479 %}
3480
3481 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3482 Register dst_reg = as_Register($dst$$reg);
3483 Register src1_reg = as_Register($src1$$reg);
3484 Register src2_reg = as_Register($src2$$reg);
3485 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3486 %}
3487
3488 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3489 Register dst_reg = as_Register($dst$$reg);
3490 Register src1_reg = as_Register($src1$$reg);
3491 Register src2_reg = as_Register($src2$$reg);
3492 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3493 %}
3494
3495 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3496 Register dst_reg = as_Register($dst$$reg);
3497 Register src1_reg = as_Register($src1$$reg);
3498 Register src2_reg = as_Register($src2$$reg);
3499 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3500 %}
3501
3502 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3503 Register dst_reg = as_Register($dst$$reg);
3504 Register src1_reg = as_Register($src1$$reg);
3505 Register src2_reg = as_Register($src2$$reg);
3506 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3507 %}
3508
3509 // compare instruction encodings
3510
3511 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3512 Register reg1 = as_Register($src1$$reg);
3513 Register reg2 = as_Register($src2$$reg);
3514 __ cmpw(reg1, reg2);
3515 %}
3516
3517 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3518 Register reg = as_Register($src1$$reg);
3519 int32_t val = $src2$$constant;
3520 if (val >= 0) {
3521 __ subsw(zr, reg, val);
3522 } else {
3523 __ addsw(zr, reg, -val);
3524 }
3525 %}
3526
3527 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3528 Register reg1 = as_Register($src1$$reg);
3529 uint32_t val = (uint32_t)$src2$$constant;
3530 __ movw(rscratch1, val);
3531 __ cmpw(reg1, rscratch1);
3532 %}
3533
3534 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3535 Register reg1 = as_Register($src1$$reg);
3536 Register reg2 = as_Register($src2$$reg);
3537 __ cmp(reg1, reg2);
3538 %}
3539
3540 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3541 Register reg = as_Register($src1$$reg);
3542 int64_t val = $src2$$constant;
3543 if (val >= 0) {
3544 __ subs(zr, reg, val);
3545 } else if (val != -val) {
3546 __ adds(zr, reg, -val);
3547 } else {
3548 // aargh, Long.MIN_VALUE is a special case
3549 __ orr(rscratch1, zr, (uint64_t)val);
3550 __ subs(zr, reg, rscratch1);
3551 }
3552 %}
3553
3554 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3555 Register reg1 = as_Register($src1$$reg);
3556 uint64_t val = (uint64_t)$src2$$constant;
3557 __ mov(rscratch1, val);
3558 __ cmp(reg1, rscratch1);
3559 %}
3560
3561 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3562 Register reg1 = as_Register($src1$$reg);
3563 Register reg2 = as_Register($src2$$reg);
3564 __ cmp(reg1, reg2);
3565 %}
3566
3567 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3568 Register reg1 = as_Register($src1$$reg);
3569 Register reg2 = as_Register($src2$$reg);
3570 __ cmpw(reg1, reg2);
3571 %}
3572
3573 enc_class aarch64_enc_testp(iRegP src) %{
3574 Register reg = as_Register($src$$reg);
3575 __ cmp(reg, zr);
3576 %}
3577
3578 enc_class aarch64_enc_testn(iRegN src) %{
3579 Register reg = as_Register($src$$reg);
3580 __ cmpw(reg, zr);
3581 %}
3582
3583 enc_class aarch64_enc_b(label lbl) %{
3584 Label *L = $lbl$$label;
3585 __ b(*L);
3586 %}
3587
3588 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3589 Label *L = $lbl$$label;
3590 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3591 %}
3592
3593 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3594 Label *L = $lbl$$label;
3595 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3596 %}
3597
3598 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3599 %{
3600 Register sub_reg = as_Register($sub$$reg);
3601 Register super_reg = as_Register($super$$reg);
3602 Register temp_reg = as_Register($temp$$reg);
3603 Register result_reg = as_Register($result$$reg);
3604
3605 Label miss;
3606 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3607 nullptr, &miss,
3608 /*set_cond_codes:*/ true);
3609 if ($primary) {
3610 __ mov(result_reg, zr);
3611 }
3612 __ bind(miss);
3613 %}
3614
3615 enc_class aarch64_enc_java_static_call(method meth) %{
3616 address addr = (address)$meth$$method;
3617 address call;
3618 if (!_method) {
3619 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3620 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3621 if (call == nullptr) {
3622 ciEnv::current()->record_failure("CodeCache is full");
3623 return;
3624 }
3625 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3626 // The NOP here is purely to ensure that eliding a call to
3627 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3628 __ nop();
3629 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3630 } else {
3631 int method_index = resolved_method_index(masm);
3632 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3633 : static_call_Relocation::spec(method_index);
3634 call = __ trampoline_call(Address(addr, rspec));
3635 if (call == nullptr) {
3636 ciEnv::current()->record_failure("CodeCache is full");
3637 return;
3638 }
3639 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3640 // Calls of the same statically bound method can share
3641 // a stub to the interpreter.
3642 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3643 } else {
3644 // Emit stub for static call
3645 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3646 if (stub == nullptr) {
3647 ciEnv::current()->record_failure("CodeCache is full");
3648 return;
3649 }
3650 }
3651 }
3652
3653 __ post_call_nop();
3654
3655 // Only non uncommon_trap calls need to reinitialize ptrue.
3656 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3657 __ reinitialize_ptrue();
3658 }
3659 %}
3660
3661 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3662 int method_index = resolved_method_index(masm);
3663 address call = __ ic_call((address)$meth$$method, method_index);
3664 if (call == nullptr) {
3665 ciEnv::current()->record_failure("CodeCache is full");
3666 return;
3667 }
3668 __ post_call_nop();
3669 if (Compile::current()->max_vector_size() > 0) {
3670 __ reinitialize_ptrue();
3671 }
3672 %}
3673
3674 enc_class aarch64_enc_call_epilog() %{
3675 if (VerifyStackAtCalls) {
3676 // Check that stack depth is unchanged: find majik cookie on stack
3677 __ call_Unimplemented();
3678 }
3679 %}
3680
3681 enc_class aarch64_enc_java_to_runtime(method meth) %{
3682 // some calls to generated routines (arraycopy code) are scheduled
3683 // by C2 as runtime calls. if so we can call them using a br (they
3684 // will be in a reachable segment) otherwise we have to use a blr
3685 // which loads the absolute address into a register.
3686 address entry = (address)$meth$$method;
3687 CodeBlob *cb = CodeCache::find_blob(entry);
3688 if (cb) {
3689 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3690 if (call == nullptr) {
3691 ciEnv::current()->record_failure("CodeCache is full");
3692 return;
3693 }
3694 __ post_call_nop();
3695 } else {
3696 Label retaddr;
3697 __ adr(rscratch2, retaddr);
3698 __ lea(rscratch1, RuntimeAddress(entry));
3699 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3700 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3701 __ blr(rscratch1);
3702 __ bind(retaddr);
3703 __ post_call_nop();
3704 __ add(sp, sp, 2 * wordSize);
3705 }
3706 if (Compile::current()->max_vector_size() > 0) {
3707 __ reinitialize_ptrue();
3708 }
3709 %}
3710
3711 enc_class aarch64_enc_rethrow() %{
3712 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3713 %}
3714
3715 enc_class aarch64_enc_ret() %{
3716 #ifdef ASSERT
3717 if (Compile::current()->max_vector_size() > 0) {
3718 __ verify_ptrue();
3719 }
3720 #endif
3721 __ ret(lr);
3722 %}
3723
3724 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3725 Register target_reg = as_Register($jump_target$$reg);
3726 __ br(target_reg);
3727 %}
3728
3729 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3730 Register target_reg = as_Register($jump_target$$reg);
3731 // exception oop should be in r0
3732 // ret addr has been popped into lr
3733 // callee expects it in r3
3734 __ mov(r3, lr);
3735 __ br(target_reg);
3736 %}
3737
3738 %}
3739
3740 //----------FRAME--------------------------------------------------------------
3741 // Definition of frame structure and management information.
3742 //
3743 // S T A C K L A Y O U T Allocators stack-slot number
3744 // | (to get allocators register number
3745 // G Owned by | | v add OptoReg::stack0())
3746 // r CALLER | |
3747 // o | +--------+ pad to even-align allocators stack-slot
3748 // w V | pad0 | numbers; owned by CALLER
3749 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3750 // h ^ | in | 5
3751 // | | args | 4 Holes in incoming args owned by SELF
3752 // | | | | 3
3753 // | | +--------+
3754 // V | | old out| Empty on Intel, window on Sparc
3755 // | old |preserve| Must be even aligned.
3756 // | SP-+--------+----> Matcher::_old_SP, even aligned
3757 // | | in | 3 area for Intel ret address
3758 // Owned by |preserve| Empty on Sparc.
3759 // SELF +--------+
3760 // | | pad2 | 2 pad to align old SP
3761 // | +--------+ 1
3762 // | | locks | 0
3763 // | +--------+----> OptoReg::stack0(), even aligned
3764 // | | pad1 | 11 pad to align new SP
3765 // | +--------+
3766 // | | | 10
3767 // | | spills | 9 spills
3768 // V | | 8 (pad0 slot for callee)
3769 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3770 // ^ | out | 7
3771 // | | args | 6 Holes in outgoing args owned by CALLEE
3772 // Owned by +--------+
3773 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3774 // | new |preserve| Must be even-aligned.
3775 // | SP-+--------+----> Matcher::_new_SP, even aligned
3776 // | | |
3777 //
3778 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3779 // known from SELF's arguments and the Java calling convention.
3780 // Region 6-7 is determined per call site.
3781 // Note 2: If the calling convention leaves holes in the incoming argument
3782 // area, those holes are owned by SELF. Holes in the outgoing area
3783 // are owned by the CALLEE. Holes should not be necessary in the
3784 // incoming area, as the Java calling convention is completely under
3785 // the control of the AD file. Doubles can be sorted and packed to
3786 // avoid holes. Holes in the outgoing arguments may be necessary for
3787 // varargs C calling conventions.
3788 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3789 // even aligned with pad0 as needed.
3790 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3791 // (the latter is true on Intel but is it false on AArch64?)
3792 // region 6-11 is even aligned; it may be padded out more so that
3793 // the region from SP to FP meets the minimum stack alignment.
3794 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3795 // alignment. Region 11, pad1, may be dynamically extended so that
3796 // SP meets the minimum alignment.
3797
3798 frame %{
3799 // These three registers define part of the calling convention
3800 // between compiled code and the interpreter.
3801
3802 // Inline Cache Register or Method for I2C.
3803 inline_cache_reg(R12);
3804
3805 // Number of stack slots consumed by locking an object
3806 sync_stack_slots(2);
3807
3808 // Compiled code's Frame Pointer
3809 frame_pointer(R31);
3810
3811 // Interpreter stores its frame pointer in a register which is
3812 // stored to the stack by I2CAdaptors.
3813 // I2CAdaptors convert from interpreted java to compiled java.
3814 interpreter_frame_pointer(R29);
3815
3816 // Stack alignment requirement
3817 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3818
3819 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3820 // for calls to C. Supports the var-args backing area for register parms.
3821 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3822
3823 // The after-PROLOG location of the return address. Location of
3824 // return address specifies a type (REG or STACK) and a number
3825 // representing the register number (i.e. - use a register name) or
3826 // stack slot.
3827 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3828 // Otherwise, it is above the locks and verification slot and alignment word
3829 // TODO this may well be correct but need to check why that - 2 is there
3830 // ppc port uses 0 but we definitely need to allow for fixed_slots
3831 // which folds in the space used for monitors
3832 return_addr(STACK - 2 +
3833 align_up((Compile::current()->in_preserve_stack_slots() +
3834 Compile::current()->fixed_slots()),
3835 stack_alignment_in_slots()));
3836
3837 // Location of compiled Java return values. Same as C for now.
3838 return_value
3839 %{
3840 // TODO do we allow ideal_reg == Op_RegN???
3841 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3842 "only return normal values");
3843
3844 static const int lo[Op_RegL + 1] = { // enum name
3845 0, // Op_Node
3846 0, // Op_Set
3847 R0_num, // Op_RegN
3848 R0_num, // Op_RegI
3849 R0_num, // Op_RegP
3850 V0_num, // Op_RegF
3851 V0_num, // Op_RegD
3852 R0_num // Op_RegL
3853 };
3854
3855 static const int hi[Op_RegL + 1] = { // enum name
3856 0, // Op_Node
3857 0, // Op_Set
3858 OptoReg::Bad, // Op_RegN
3859 OptoReg::Bad, // Op_RegI
3860 R0_H_num, // Op_RegP
3861 OptoReg::Bad, // Op_RegF
3862 V0_H_num, // Op_RegD
3863 R0_H_num // Op_RegL
3864 };
3865
3866 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3867 %}
3868 %}
3869
3870 //----------ATTRIBUTES---------------------------------------------------------
3871 //----------Operand Attributes-------------------------------------------------
3872 op_attrib op_cost(1); // Required cost attribute
3873
3874 //----------Instruction Attributes---------------------------------------------
3875 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3876 ins_attrib ins_size(32); // Required size attribute (in bits)
3877 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3878 // a non-matching short branch variant
3879 // of some long branch?
3880 ins_attrib ins_alignment(4); // Required alignment attribute (must
3881 // be a power of 2) specifies the
3882 // alignment that some part of the
3883 // instruction (not necessarily the
3884 // start) requires. If > 1, a
3885 // compute_padding() function must be
3886 // provided for the instruction
3887
3888 //----------OPERANDS-----------------------------------------------------------
3889 // Operand definitions must precede instruction definitions for correct parsing
3890 // in the ADLC because operands constitute user defined types which are used in
3891 // instruction definitions.
3892
3893 //----------Simple Operands----------------------------------------------------
3894
3895 // Integer operands 32 bit
3896 // 32 bit immediate
3897 operand immI()
3898 %{
3899 match(ConI);
3900
3901 op_cost(0);
3902 format %{ %}
3903 interface(CONST_INTER);
3904 %}
3905
3906 // 32 bit zero
3907 operand immI0()
3908 %{
3909 predicate(n->get_int() == 0);
3910 match(ConI);
3911
3912 op_cost(0);
3913 format %{ %}
3914 interface(CONST_INTER);
3915 %}
3916
3917 // 32 bit unit increment
3918 operand immI_1()
3919 %{
3920 predicate(n->get_int() == 1);
3921 match(ConI);
3922
3923 op_cost(0);
3924 format %{ %}
3925 interface(CONST_INTER);
3926 %}
3927
3928 // 32 bit unit decrement
3929 operand immI_M1()
3930 %{
3931 predicate(n->get_int() == -1);
3932 match(ConI);
3933
3934 op_cost(0);
3935 format %{ %}
3936 interface(CONST_INTER);
3937 %}
3938
3939 // Shift values for add/sub extension shift
3940 operand immIExt()
3941 %{
3942 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3943 match(ConI);
3944
3945 op_cost(0);
3946 format %{ %}
3947 interface(CONST_INTER);
3948 %}
3949
3950 operand immI_gt_1()
3951 %{
3952 predicate(n->get_int() > 1);
3953 match(ConI);
3954
3955 op_cost(0);
3956 format %{ %}
3957 interface(CONST_INTER);
3958 %}
3959
3960 operand immI_le_4()
3961 %{
3962 predicate(n->get_int() <= 4);
3963 match(ConI);
3964
3965 op_cost(0);
3966 format %{ %}
3967 interface(CONST_INTER);
3968 %}
3969
3970 operand immI_16()
3971 %{
3972 predicate(n->get_int() == 16);
3973 match(ConI);
3974
3975 op_cost(0);
3976 format %{ %}
3977 interface(CONST_INTER);
3978 %}
3979
3980 operand immI_24()
3981 %{
3982 predicate(n->get_int() == 24);
3983 match(ConI);
3984
3985 op_cost(0);
3986 format %{ %}
3987 interface(CONST_INTER);
3988 %}
3989
3990 operand immI_32()
3991 %{
3992 predicate(n->get_int() == 32);
3993 match(ConI);
3994
3995 op_cost(0);
3996 format %{ %}
3997 interface(CONST_INTER);
3998 %}
3999
4000 operand immI_48()
4001 %{
4002 predicate(n->get_int() == 48);
4003 match(ConI);
4004
4005 op_cost(0);
4006 format %{ %}
4007 interface(CONST_INTER);
4008 %}
4009
4010 operand immI_56()
4011 %{
4012 predicate(n->get_int() == 56);
4013 match(ConI);
4014
4015 op_cost(0);
4016 format %{ %}
4017 interface(CONST_INTER);
4018 %}
4019
4020 operand immI_255()
4021 %{
4022 predicate(n->get_int() == 255);
4023 match(ConI);
4024
4025 op_cost(0);
4026 format %{ %}
4027 interface(CONST_INTER);
4028 %}
4029
4030 operand immI_65535()
4031 %{
4032 predicate(n->get_int() == 65535);
4033 match(ConI);
4034
4035 op_cost(0);
4036 format %{ %}
4037 interface(CONST_INTER);
4038 %}
4039
4040 operand immI_positive()
4041 %{
4042 predicate(n->get_int() > 0);
4043 match(ConI);
4044
4045 op_cost(0);
4046 format %{ %}
4047 interface(CONST_INTER);
4048 %}
4049
4050 // BoolTest condition for signed compare
4051 operand immI_cmp_cond()
4052 %{
4053 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4054 match(ConI);
4055
4056 op_cost(0);
4057 format %{ %}
4058 interface(CONST_INTER);
4059 %}
4060
4061 // BoolTest condition for unsigned compare
4062 operand immI_cmpU_cond()
4063 %{
4064 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4065 match(ConI);
4066
4067 op_cost(0);
4068 format %{ %}
4069 interface(CONST_INTER);
4070 %}
4071
4072 operand immL_255()
4073 %{
4074 predicate(n->get_long() == 255L);
4075 match(ConL);
4076
4077 op_cost(0);
4078 format %{ %}
4079 interface(CONST_INTER);
4080 %}
4081
4082 operand immL_65535()
4083 %{
4084 predicate(n->get_long() == 65535L);
4085 match(ConL);
4086
4087 op_cost(0);
4088 format %{ %}
4089 interface(CONST_INTER);
4090 %}
4091
4092 operand immL_4294967295()
4093 %{
4094 predicate(n->get_long() == 4294967295L);
4095 match(ConL);
4096
4097 op_cost(0);
4098 format %{ %}
4099 interface(CONST_INTER);
4100 %}
4101
4102 operand immL_bitmask()
4103 %{
4104 predicate((n->get_long() != 0)
4105 && ((n->get_long() & 0xc000000000000000l) == 0)
4106 && is_power_of_2(n->get_long() + 1));
4107 match(ConL);
4108
4109 op_cost(0);
4110 format %{ %}
4111 interface(CONST_INTER);
4112 %}
4113
4114 operand immI_bitmask()
4115 %{
4116 predicate((n->get_int() != 0)
4117 && ((n->get_int() & 0xc0000000) == 0)
4118 && is_power_of_2(n->get_int() + 1));
4119 match(ConI);
4120
4121 op_cost(0);
4122 format %{ %}
4123 interface(CONST_INTER);
4124 %}
4125
4126 operand immL_positive_bitmaskI()
4127 %{
4128 predicate((n->get_long() != 0)
4129 && ((julong)n->get_long() < 0x80000000ULL)
4130 && is_power_of_2(n->get_long() + 1));
4131 match(ConL);
4132
4133 op_cost(0);
4134 format %{ %}
4135 interface(CONST_INTER);
4136 %}
4137
4138 // Scale values for scaled offset addressing modes (up to long but not quad)
4139 operand immIScale()
4140 %{
4141 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4142 match(ConI);
4143
4144 op_cost(0);
4145 format %{ %}
4146 interface(CONST_INTER);
4147 %}
4148
4149 // 5 bit signed integer
4150 operand immI5()
4151 %{
4152 predicate(Assembler::is_simm(n->get_int(), 5));
4153 match(ConI);
4154
4155 op_cost(0);
4156 format %{ %}
4157 interface(CONST_INTER);
4158 %}
4159
4160 // 7 bit unsigned integer
4161 operand immIU7()
4162 %{
4163 predicate(Assembler::is_uimm(n->get_int(), 7));
4164 match(ConI);
4165
4166 op_cost(0);
4167 format %{ %}
4168 interface(CONST_INTER);
4169 %}
4170
4171 // Offset for scaled or unscaled immediate loads and stores
4172 operand immIOffset()
4173 %{
4174 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4175 match(ConI);
4176
4177 op_cost(0);
4178 format %{ %}
4179 interface(CONST_INTER);
4180 %}
4181
4182 operand immIOffset1()
4183 %{
4184 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4185 match(ConI);
4186
4187 op_cost(0);
4188 format %{ %}
4189 interface(CONST_INTER);
4190 %}
4191
4192 operand immIOffset2()
4193 %{
4194 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4195 match(ConI);
4196
4197 op_cost(0);
4198 format %{ %}
4199 interface(CONST_INTER);
4200 %}
4201
4202 operand immIOffset4()
4203 %{
4204 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4205 match(ConI);
4206
4207 op_cost(0);
4208 format %{ %}
4209 interface(CONST_INTER);
4210 %}
4211
4212 operand immIOffset8()
4213 %{
4214 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4215 match(ConI);
4216
4217 op_cost(0);
4218 format %{ %}
4219 interface(CONST_INTER);
4220 %}
4221
4222 operand immIOffset16()
4223 %{
4224 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4225 match(ConI);
4226
4227 op_cost(0);
4228 format %{ %}
4229 interface(CONST_INTER);
4230 %}
4231
4232 operand immLoffset()
4233 %{
4234 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4235 match(ConL);
4236
4237 op_cost(0);
4238 format %{ %}
4239 interface(CONST_INTER);
4240 %}
4241
4242 operand immLoffset1()
4243 %{
4244 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4245 match(ConL);
4246
4247 op_cost(0);
4248 format %{ %}
4249 interface(CONST_INTER);
4250 %}
4251
4252 operand immLoffset2()
4253 %{
4254 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4255 match(ConL);
4256
4257 op_cost(0);
4258 format %{ %}
4259 interface(CONST_INTER);
4260 %}
4261
4262 operand immLoffset4()
4263 %{
4264 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4265 match(ConL);
4266
4267 op_cost(0);
4268 format %{ %}
4269 interface(CONST_INTER);
4270 %}
4271
4272 operand immLoffset8()
4273 %{
4274 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4275 match(ConL);
4276
4277 op_cost(0);
4278 format %{ %}
4279 interface(CONST_INTER);
4280 %}
4281
4282 operand immLoffset16()
4283 %{
4284 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4285 match(ConL);
4286
4287 op_cost(0);
4288 format %{ %}
4289 interface(CONST_INTER);
4290 %}
4291
4292 // 5 bit signed long integer
4293 operand immL5()
4294 %{
4295 predicate(Assembler::is_simm(n->get_long(), 5));
4296 match(ConL);
4297
4298 op_cost(0);
4299 format %{ %}
4300 interface(CONST_INTER);
4301 %}
4302
4303 // 7 bit unsigned long integer
4304 operand immLU7()
4305 %{
4306 predicate(Assembler::is_uimm(n->get_long(), 7));
4307 match(ConL);
4308
4309 op_cost(0);
4310 format %{ %}
4311 interface(CONST_INTER);
4312 %}
4313
4314 // 8 bit signed value.
4315 operand immI8()
4316 %{
4317 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4318 match(ConI);
4319
4320 op_cost(0);
4321 format %{ %}
4322 interface(CONST_INTER);
4323 %}
4324
4325 // 8 bit signed value (simm8), or #simm8 LSL 8.
4326 operand immI8_shift8()
4327 %{
4328 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4329 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4330 match(ConI);
4331
4332 op_cost(0);
4333 format %{ %}
4334 interface(CONST_INTER);
4335 %}
4336
4337 // 8 bit signed value (simm8), or #simm8 LSL 8.
4338 operand immL8_shift8()
4339 %{
4340 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4341 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4342 match(ConL);
4343
4344 op_cost(0);
4345 format %{ %}
4346 interface(CONST_INTER);
4347 %}
4348
4349 // 8 bit integer valid for vector add sub immediate
4350 operand immBAddSubV()
4351 %{
4352 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4353 match(ConI);
4354
4355 op_cost(0);
4356 format %{ %}
4357 interface(CONST_INTER);
4358 %}
4359
4360 // 32 bit integer valid for add sub immediate
4361 operand immIAddSub()
4362 %{
4363 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4364 match(ConI);
4365 op_cost(0);
4366 format %{ %}
4367 interface(CONST_INTER);
4368 %}
4369
4370 // 32 bit integer valid for vector add sub immediate
4371 operand immIAddSubV()
4372 %{
4373 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4374 match(ConI);
4375
4376 op_cost(0);
4377 format %{ %}
4378 interface(CONST_INTER);
4379 %}
4380
4381 // 32 bit unsigned integer valid for logical immediate
4382
4383 operand immBLog()
4384 %{
4385 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4386 match(ConI);
4387
4388 op_cost(0);
4389 format %{ %}
4390 interface(CONST_INTER);
4391 %}
4392
4393 operand immSLog()
4394 %{
4395 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4396 match(ConI);
4397
4398 op_cost(0);
4399 format %{ %}
4400 interface(CONST_INTER);
4401 %}
4402
4403 operand immILog()
4404 %{
4405 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4406 match(ConI);
4407
4408 op_cost(0);
4409 format %{ %}
4410 interface(CONST_INTER);
4411 %}
4412
4413 // Integer operands 64 bit
4414 // 64 bit immediate
4415 operand immL()
4416 %{
4417 match(ConL);
4418
4419 op_cost(0);
4420 format %{ %}
4421 interface(CONST_INTER);
4422 %}
4423
4424 // 64 bit zero
4425 operand immL0()
4426 %{
4427 predicate(n->get_long() == 0);
4428 match(ConL);
4429
4430 op_cost(0);
4431 format %{ %}
4432 interface(CONST_INTER);
4433 %}
4434
4435 // 64 bit unit decrement
4436 operand immL_M1()
4437 %{
4438 predicate(n->get_long() == -1);
4439 match(ConL);
4440
4441 op_cost(0);
4442 format %{ %}
4443 interface(CONST_INTER);
4444 %}
4445
4446 // 64 bit integer valid for add sub immediate
4447 operand immLAddSub()
4448 %{
4449 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4450 match(ConL);
4451 op_cost(0);
4452 format %{ %}
4453 interface(CONST_INTER);
4454 %}
4455
4456 // 64 bit integer valid for addv subv immediate
4457 operand immLAddSubV()
4458 %{
4459 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4460 match(ConL);
4461
4462 op_cost(0);
4463 format %{ %}
4464 interface(CONST_INTER);
4465 %}
4466
4467 // 64 bit integer valid for logical immediate
4468 operand immLLog()
4469 %{
4470 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4471 match(ConL);
4472 op_cost(0);
4473 format %{ %}
4474 interface(CONST_INTER);
4475 %}
4476
4477 // Long Immediate: low 32-bit mask
4478 operand immL_32bits()
4479 %{
4480 predicate(n->get_long() == 0xFFFFFFFFL);
4481 match(ConL);
4482 op_cost(0);
4483 format %{ %}
4484 interface(CONST_INTER);
4485 %}
4486
4487 // Pointer operands
4488 // Pointer Immediate
4489 operand immP()
4490 %{
4491 match(ConP);
4492
4493 op_cost(0);
4494 format %{ %}
4495 interface(CONST_INTER);
4496 %}
4497
4498 // nullptr Pointer Immediate
4499 operand immP0()
4500 %{
4501 predicate(n->get_ptr() == 0);
4502 match(ConP);
4503
4504 op_cost(0);
4505 format %{ %}
4506 interface(CONST_INTER);
4507 %}
4508
4509 // Pointer Immediate One
4510 // this is used in object initialization (initial object header)
4511 operand immP_1()
4512 %{
4513 predicate(n->get_ptr() == 1);
4514 match(ConP);
4515
4516 op_cost(0);
4517 format %{ %}
4518 interface(CONST_INTER);
4519 %}
4520
4521 // Card Table Byte Map Base
4522 operand immByteMapBase()
4523 %{
4524 // Get base of card map
4525 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4526 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4527 match(ConP);
4528
4529 op_cost(0);
4530 format %{ %}
4531 interface(CONST_INTER);
4532 %}
4533
4534 // Float and Double operands
4535 // Double Immediate
4536 operand immD()
4537 %{
4538 match(ConD);
4539 op_cost(0);
4540 format %{ %}
4541 interface(CONST_INTER);
4542 %}
4543
4544 // Double Immediate: +0.0d
4545 operand immD0()
4546 %{
4547 predicate(jlong_cast(n->getd()) == 0);
4548 match(ConD);
4549
4550 op_cost(0);
4551 format %{ %}
4552 interface(CONST_INTER);
4553 %}
4554
4555 // constant 'double +0.0'.
4556 operand immDPacked()
4557 %{
4558 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4559 match(ConD);
4560 op_cost(0);
4561 format %{ %}
4562 interface(CONST_INTER);
4563 %}
4564
4565 // Float Immediate
4566 operand immF()
4567 %{
4568 match(ConF);
4569 op_cost(0);
4570 format %{ %}
4571 interface(CONST_INTER);
4572 %}
4573
4574 // Float Immediate: +0.0f.
4575 operand immF0()
4576 %{
4577 predicate(jint_cast(n->getf()) == 0);
4578 match(ConF);
4579
4580 op_cost(0);
4581 format %{ %}
4582 interface(CONST_INTER);
4583 %}
4584
4585 //
4586 operand immFPacked()
4587 %{
4588 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4589 match(ConF);
4590 op_cost(0);
4591 format %{ %}
4592 interface(CONST_INTER);
4593 %}
4594
4595 // Narrow pointer operands
4596 // Narrow Pointer Immediate
4597 operand immN()
4598 %{
4599 match(ConN);
4600
4601 op_cost(0);
4602 format %{ %}
4603 interface(CONST_INTER);
4604 %}
4605
4606 // Narrow nullptr Pointer Immediate
4607 operand immN0()
4608 %{
4609 predicate(n->get_narrowcon() == 0);
4610 match(ConN);
4611
4612 op_cost(0);
4613 format %{ %}
4614 interface(CONST_INTER);
4615 %}
4616
4617 operand immNKlass()
4618 %{
4619 match(ConNKlass);
4620
4621 op_cost(0);
4622 format %{ %}
4623 interface(CONST_INTER);
4624 %}
4625
4626 // Integer 32 bit Register Operands
4627 // Integer 32 bitRegister (excludes SP)
4628 operand iRegI()
4629 %{
4630 constraint(ALLOC_IN_RC(any_reg32));
4631 match(RegI);
4632 match(iRegINoSp);
4633 op_cost(0);
4634 format %{ %}
4635 interface(REG_INTER);
4636 %}
4637
4638 // Integer 32 bit Register not Special
4639 operand iRegINoSp()
4640 %{
4641 constraint(ALLOC_IN_RC(no_special_reg32));
4642 match(RegI);
4643 op_cost(0);
4644 format %{ %}
4645 interface(REG_INTER);
4646 %}
4647
4648 // Integer 64 bit Register Operands
4649 // Integer 64 bit Register (includes SP)
4650 operand iRegL()
4651 %{
4652 constraint(ALLOC_IN_RC(any_reg));
4653 match(RegL);
4654 match(iRegLNoSp);
4655 op_cost(0);
4656 format %{ %}
4657 interface(REG_INTER);
4658 %}
4659
4660 // Integer 64 bit Register not Special
4661 operand iRegLNoSp()
4662 %{
4663 constraint(ALLOC_IN_RC(no_special_reg));
4664 match(RegL);
4665 match(iRegL_R0);
4666 format %{ %}
4667 interface(REG_INTER);
4668 %}
4669
4670 // Pointer Register Operands
4671 // Pointer Register
4672 operand iRegP()
4673 %{
4674 constraint(ALLOC_IN_RC(ptr_reg));
4675 match(RegP);
4676 match(iRegPNoSp);
4677 match(iRegP_R0);
4678 //match(iRegP_R2);
4679 //match(iRegP_R4);
4680 match(iRegP_R5);
4681 match(thread_RegP);
4682 op_cost(0);
4683 format %{ %}
4684 interface(REG_INTER);
4685 %}
4686
4687 // Pointer 64 bit Register not Special
4688 operand iRegPNoSp()
4689 %{
4690 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4691 match(RegP);
4692 // match(iRegP);
4693 // match(iRegP_R0);
4694 // match(iRegP_R2);
4695 // match(iRegP_R4);
4696 // match(iRegP_R5);
4697 // match(thread_RegP);
4698 op_cost(0);
4699 format %{ %}
4700 interface(REG_INTER);
4701 %}
4702
4703 // This operand is not allowed to use rfp even if
4704 // rfp is not used to hold the frame pointer.
4705 operand iRegPNoSpNoRfp()
4706 %{
4707 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4708 match(RegP);
4709 match(iRegPNoSp);
4710 op_cost(0);
4711 format %{ %}
4712 interface(REG_INTER);
4713 %}
4714
4715 // Pointer 64 bit Register R0 only
4716 operand iRegP_R0()
4717 %{
4718 constraint(ALLOC_IN_RC(r0_reg));
4719 match(RegP);
4720 // match(iRegP);
4721 match(iRegPNoSp);
4722 op_cost(0);
4723 format %{ %}
4724 interface(REG_INTER);
4725 %}
4726
4727 // Pointer 64 bit Register R1 only
4728 operand iRegP_R1()
4729 %{
4730 constraint(ALLOC_IN_RC(r1_reg));
4731 match(RegP);
4732 // match(iRegP);
4733 match(iRegPNoSp);
4734 op_cost(0);
4735 format %{ %}
4736 interface(REG_INTER);
4737 %}
4738
4739 // Pointer 64 bit Register R2 only
4740 operand iRegP_R2()
4741 %{
4742 constraint(ALLOC_IN_RC(r2_reg));
4743 match(RegP);
4744 // match(iRegP);
4745 match(iRegPNoSp);
4746 op_cost(0);
4747 format %{ %}
4748 interface(REG_INTER);
4749 %}
4750
4751 // Pointer 64 bit Register R3 only
4752 operand iRegP_R3()
4753 %{
4754 constraint(ALLOC_IN_RC(r3_reg));
4755 match(RegP);
4756 // match(iRegP);
4757 match(iRegPNoSp);
4758 op_cost(0);
4759 format %{ %}
4760 interface(REG_INTER);
4761 %}
4762
4763 // Pointer 64 bit Register R4 only
4764 operand iRegP_R4()
4765 %{
4766 constraint(ALLOC_IN_RC(r4_reg));
4767 match(RegP);
4768 // match(iRegP);
4769 match(iRegPNoSp);
4770 op_cost(0);
4771 format %{ %}
4772 interface(REG_INTER);
4773 %}
4774
4775 // Pointer 64 bit Register R5 only
4776 operand iRegP_R5()
4777 %{
4778 constraint(ALLOC_IN_RC(r5_reg));
4779 match(RegP);
4780 // match(iRegP);
4781 match(iRegPNoSp);
4782 op_cost(0);
4783 format %{ %}
4784 interface(REG_INTER);
4785 %}
4786
4787 // Pointer 64 bit Register R10 only
4788 operand iRegP_R10()
4789 %{
4790 constraint(ALLOC_IN_RC(r10_reg));
4791 match(RegP);
4792 // match(iRegP);
4793 match(iRegPNoSp);
4794 op_cost(0);
4795 format %{ %}
4796 interface(REG_INTER);
4797 %}
4798
4799 // Long 64 bit Register R0 only
4800 operand iRegL_R0()
4801 %{
4802 constraint(ALLOC_IN_RC(r0_reg));
4803 match(RegL);
4804 match(iRegLNoSp);
4805 op_cost(0);
4806 format %{ %}
4807 interface(REG_INTER);
4808 %}
4809
4810 // Long 64 bit Register R11 only
4811 operand iRegL_R11()
4812 %{
4813 constraint(ALLOC_IN_RC(r11_reg));
4814 match(RegL);
4815 match(iRegLNoSp);
4816 op_cost(0);
4817 format %{ %}
4818 interface(REG_INTER);
4819 %}
4820
4821 // Register R0 only
4822 operand iRegI_R0()
4823 %{
4824 constraint(ALLOC_IN_RC(int_r0_reg));
4825 match(RegI);
4826 match(iRegINoSp);
4827 op_cost(0);
4828 format %{ %}
4829 interface(REG_INTER);
4830 %}
4831
4832 // Register R2 only
4833 operand iRegI_R2()
4834 %{
4835 constraint(ALLOC_IN_RC(int_r2_reg));
4836 match(RegI);
4837 match(iRegINoSp);
4838 op_cost(0);
4839 format %{ %}
4840 interface(REG_INTER);
4841 %}
4842
4843 // Register R3 only
4844 operand iRegI_R3()
4845 %{
4846 constraint(ALLOC_IN_RC(int_r3_reg));
4847 match(RegI);
4848 match(iRegINoSp);
4849 op_cost(0);
4850 format %{ %}
4851 interface(REG_INTER);
4852 %}
4853
4854
4855 // Register R4 only
4856 operand iRegI_R4()
4857 %{
4858 constraint(ALLOC_IN_RC(int_r4_reg));
4859 match(RegI);
4860 match(iRegINoSp);
4861 op_cost(0);
4862 format %{ %}
4863 interface(REG_INTER);
4864 %}
4865
4866
4867 // Pointer Register Operands
4868 // Narrow Pointer Register
4869 operand iRegN()
4870 %{
4871 constraint(ALLOC_IN_RC(any_reg32));
4872 match(RegN);
4873 match(iRegNNoSp);
4874 op_cost(0);
4875 format %{ %}
4876 interface(REG_INTER);
4877 %}
4878
4879 // Integer 64 bit Register not Special
4880 operand iRegNNoSp()
4881 %{
4882 constraint(ALLOC_IN_RC(no_special_reg32));
4883 match(RegN);
4884 op_cost(0);
4885 format %{ %}
4886 interface(REG_INTER);
4887 %}
4888
4889 // Float Register
4890 // Float register operands
4891 operand vRegF()
4892 %{
4893 constraint(ALLOC_IN_RC(float_reg));
4894 match(RegF);
4895
4896 op_cost(0);
4897 format %{ %}
4898 interface(REG_INTER);
4899 %}
4900
4901 // Double Register
4902 // Double register operands
4903 operand vRegD()
4904 %{
4905 constraint(ALLOC_IN_RC(double_reg));
4906 match(RegD);
4907
4908 op_cost(0);
4909 format %{ %}
4910 interface(REG_INTER);
4911 %}
4912
4913 // Generic vector class. This will be used for
4914 // all vector operands, including NEON and SVE.
4915 operand vReg()
4916 %{
4917 constraint(ALLOC_IN_RC(dynamic));
4918 match(VecA);
4919 match(VecD);
4920 match(VecX);
4921
4922 op_cost(0);
4923 format %{ %}
4924 interface(REG_INTER);
4925 %}
4926
4927 operand vecA()
4928 %{
4929 constraint(ALLOC_IN_RC(vectora_reg));
4930 match(VecA);
4931
4932 op_cost(0);
4933 format %{ %}
4934 interface(REG_INTER);
4935 %}
4936
4937 operand vecD()
4938 %{
4939 constraint(ALLOC_IN_RC(vectord_reg));
4940 match(VecD);
4941
4942 op_cost(0);
4943 format %{ %}
4944 interface(REG_INTER);
4945 %}
4946
4947 operand vecX()
4948 %{
4949 constraint(ALLOC_IN_RC(vectorx_reg));
4950 match(VecX);
4951
4952 op_cost(0);
4953 format %{ %}
4954 interface(REG_INTER);
4955 %}
4956
4957 operand vRegD_V0()
4958 %{
4959 constraint(ALLOC_IN_RC(v0_reg));
4960 match(RegD);
4961 op_cost(0);
4962 format %{ %}
4963 interface(REG_INTER);
4964 %}
4965
4966 operand vRegD_V1()
4967 %{
4968 constraint(ALLOC_IN_RC(v1_reg));
4969 match(RegD);
4970 op_cost(0);
4971 format %{ %}
4972 interface(REG_INTER);
4973 %}
4974
4975 operand vRegD_V2()
4976 %{
4977 constraint(ALLOC_IN_RC(v2_reg));
4978 match(RegD);
4979 op_cost(0);
4980 format %{ %}
4981 interface(REG_INTER);
4982 %}
4983
4984 operand vRegD_V3()
4985 %{
4986 constraint(ALLOC_IN_RC(v3_reg));
4987 match(RegD);
4988 op_cost(0);
4989 format %{ %}
4990 interface(REG_INTER);
4991 %}
4992
4993 operand vRegD_V4()
4994 %{
4995 constraint(ALLOC_IN_RC(v4_reg));
4996 match(RegD);
4997 op_cost(0);
4998 format %{ %}
4999 interface(REG_INTER);
5000 %}
5001
5002 operand vRegD_V5()
5003 %{
5004 constraint(ALLOC_IN_RC(v5_reg));
5005 match(RegD);
5006 op_cost(0);
5007 format %{ %}
5008 interface(REG_INTER);
5009 %}
5010
5011 operand vRegD_V6()
5012 %{
5013 constraint(ALLOC_IN_RC(v6_reg));
5014 match(RegD);
5015 op_cost(0);
5016 format %{ %}
5017 interface(REG_INTER);
5018 %}
5019
5020 operand vRegD_V7()
5021 %{
5022 constraint(ALLOC_IN_RC(v7_reg));
5023 match(RegD);
5024 op_cost(0);
5025 format %{ %}
5026 interface(REG_INTER);
5027 %}
5028
5029 operand pReg()
5030 %{
5031 constraint(ALLOC_IN_RC(pr_reg));
5032 match(RegVectMask);
5033 match(pRegGov);
5034 op_cost(0);
5035 format %{ %}
5036 interface(REG_INTER);
5037 %}
5038
5039 operand pRegGov()
5040 %{
5041 constraint(ALLOC_IN_RC(gov_pr));
5042 match(RegVectMask);
5043 match(pReg);
5044 op_cost(0);
5045 format %{ %}
5046 interface(REG_INTER);
5047 %}
5048
5049 operand pRegGov_P0()
5050 %{
5051 constraint(ALLOC_IN_RC(p0_reg));
5052 match(RegVectMask);
5053 op_cost(0);
5054 format %{ %}
5055 interface(REG_INTER);
5056 %}
5057
5058 operand pRegGov_P1()
5059 %{
5060 constraint(ALLOC_IN_RC(p1_reg));
5061 match(RegVectMask);
5062 op_cost(0);
5063 format %{ %}
5064 interface(REG_INTER);
5065 %}
5066
5067 // Flags register, used as output of signed compare instructions
5068
5069 // note that on AArch64 we also use this register as the output for
5070 // for floating point compare instructions (CmpF CmpD). this ensures
5071 // that ordered inequality tests use GT, GE, LT or LE none of which
5072 // pass through cases where the result is unordered i.e. one or both
5073 // inputs to the compare is a NaN. this means that the ideal code can
5074 // replace e.g. a GT with an LE and not end up capturing the NaN case
5075 // (where the comparison should always fail). EQ and NE tests are
5076 // always generated in ideal code so that unordered folds into the NE
5077 // case, matching the behaviour of AArch64 NE.
5078 //
5079 // This differs from x86 where the outputs of FP compares use a
5080 // special FP flags registers and where compares based on this
5081 // register are distinguished into ordered inequalities (cmpOpUCF) and
5082 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5083 // to explicitly handle the unordered case in branches. x86 also has
5084 // to include extra CMoveX rules to accept a cmpOpUCF input.
5085
5086 operand rFlagsReg()
5087 %{
5088 constraint(ALLOC_IN_RC(int_flags));
5089 match(RegFlags);
5090
5091 op_cost(0);
5092 format %{ "RFLAGS" %}
5093 interface(REG_INTER);
5094 %}
5095
5096 // Flags register, used as output of unsigned compare instructions
5097 operand rFlagsRegU()
5098 %{
5099 constraint(ALLOC_IN_RC(int_flags));
5100 match(RegFlags);
5101
5102 op_cost(0);
5103 format %{ "RFLAGSU" %}
5104 interface(REG_INTER);
5105 %}
5106
5107 // Special Registers
5108
5109 // Method Register
5110 operand inline_cache_RegP(iRegP reg)
5111 %{
5112 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5113 match(reg);
5114 match(iRegPNoSp);
5115 op_cost(0);
5116 format %{ %}
5117 interface(REG_INTER);
5118 %}
5119
5120 // Thread Register
5121 operand thread_RegP(iRegP reg)
5122 %{
5123 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5124 match(reg);
5125 op_cost(0);
5126 format %{ %}
5127 interface(REG_INTER);
5128 %}
5129
5130 //----------Memory Operands----------------------------------------------------
5131
5132 operand indirect(iRegP reg)
5133 %{
5134 constraint(ALLOC_IN_RC(ptr_reg));
5135 match(reg);
5136 op_cost(0);
5137 format %{ "[$reg]" %}
5138 interface(MEMORY_INTER) %{
5139 base($reg);
5140 index(0xffffffff);
5141 scale(0x0);
5142 disp(0x0);
5143 %}
5144 %}
5145
5146 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5147 %{
5148 constraint(ALLOC_IN_RC(ptr_reg));
5149 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5150 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5151 op_cost(0);
5152 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5153 interface(MEMORY_INTER) %{
5154 base($reg);
5155 index($ireg);
5156 scale($scale);
5157 disp(0x0);
5158 %}
5159 %}
5160
5161 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5162 %{
5163 constraint(ALLOC_IN_RC(ptr_reg));
5164 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5165 match(AddP reg (LShiftL lreg scale));
5166 op_cost(0);
5167 format %{ "$reg, $lreg lsl($scale)" %}
5168 interface(MEMORY_INTER) %{
5169 base($reg);
5170 index($lreg);
5171 scale($scale);
5172 disp(0x0);
5173 %}
5174 %}
5175
5176 operand indIndexI2L(iRegP reg, iRegI ireg)
5177 %{
5178 constraint(ALLOC_IN_RC(ptr_reg));
5179 match(AddP reg (ConvI2L ireg));
5180 op_cost(0);
5181 format %{ "$reg, $ireg, 0, I2L" %}
5182 interface(MEMORY_INTER) %{
5183 base($reg);
5184 index($ireg);
5185 scale(0x0);
5186 disp(0x0);
5187 %}
5188 %}
5189
5190 operand indIndex(iRegP reg, iRegL lreg)
5191 %{
5192 constraint(ALLOC_IN_RC(ptr_reg));
5193 match(AddP reg lreg);
5194 op_cost(0);
5195 format %{ "$reg, $lreg" %}
5196 interface(MEMORY_INTER) %{
5197 base($reg);
5198 index($lreg);
5199 scale(0x0);
5200 disp(0x0);
5201 %}
5202 %}
5203
5204 operand indOffI1(iRegP reg, immIOffset1 off)
5205 %{
5206 constraint(ALLOC_IN_RC(ptr_reg));
5207 match(AddP reg off);
5208 op_cost(0);
5209 format %{ "[$reg, $off]" %}
5210 interface(MEMORY_INTER) %{
5211 base($reg);
5212 index(0xffffffff);
5213 scale(0x0);
5214 disp($off);
5215 %}
5216 %}
5217
5218 operand indOffI2(iRegP reg, immIOffset2 off)
5219 %{
5220 constraint(ALLOC_IN_RC(ptr_reg));
5221 match(AddP reg off);
5222 op_cost(0);
5223 format %{ "[$reg, $off]" %}
5224 interface(MEMORY_INTER) %{
5225 base($reg);
5226 index(0xffffffff);
5227 scale(0x0);
5228 disp($off);
5229 %}
5230 %}
5231
5232 operand indOffI4(iRegP reg, immIOffset4 off)
5233 %{
5234 constraint(ALLOC_IN_RC(ptr_reg));
5235 match(AddP reg off);
5236 op_cost(0);
5237 format %{ "[$reg, $off]" %}
5238 interface(MEMORY_INTER) %{
5239 base($reg);
5240 index(0xffffffff);
5241 scale(0x0);
5242 disp($off);
5243 %}
5244 %}
5245
5246 operand indOffI8(iRegP reg, immIOffset8 off)
5247 %{
5248 constraint(ALLOC_IN_RC(ptr_reg));
5249 match(AddP reg off);
5250 op_cost(0);
5251 format %{ "[$reg, $off]" %}
5252 interface(MEMORY_INTER) %{
5253 base($reg);
5254 index(0xffffffff);
5255 scale(0x0);
5256 disp($off);
5257 %}
5258 %}
5259
5260 operand indOffI16(iRegP reg, immIOffset16 off)
5261 %{
5262 constraint(ALLOC_IN_RC(ptr_reg));
5263 match(AddP reg off);
5264 op_cost(0);
5265 format %{ "[$reg, $off]" %}
5266 interface(MEMORY_INTER) %{
5267 base($reg);
5268 index(0xffffffff);
5269 scale(0x0);
5270 disp($off);
5271 %}
5272 %}
5273
5274 operand indOffL1(iRegP reg, immLoffset1 off)
5275 %{
5276 constraint(ALLOC_IN_RC(ptr_reg));
5277 match(AddP reg off);
5278 op_cost(0);
5279 format %{ "[$reg, $off]" %}
5280 interface(MEMORY_INTER) %{
5281 base($reg);
5282 index(0xffffffff);
5283 scale(0x0);
5284 disp($off);
5285 %}
5286 %}
5287
5288 operand indOffL2(iRegP reg, immLoffset2 off)
5289 %{
5290 constraint(ALLOC_IN_RC(ptr_reg));
5291 match(AddP reg off);
5292 op_cost(0);
5293 format %{ "[$reg, $off]" %}
5294 interface(MEMORY_INTER) %{
5295 base($reg);
5296 index(0xffffffff);
5297 scale(0x0);
5298 disp($off);
5299 %}
5300 %}
5301
5302 operand indOffL4(iRegP reg, immLoffset4 off)
5303 %{
5304 constraint(ALLOC_IN_RC(ptr_reg));
5305 match(AddP reg off);
5306 op_cost(0);
5307 format %{ "[$reg, $off]" %}
5308 interface(MEMORY_INTER) %{
5309 base($reg);
5310 index(0xffffffff);
5311 scale(0x0);
5312 disp($off);
5313 %}
5314 %}
5315
5316 operand indOffL8(iRegP reg, immLoffset8 off)
5317 %{
5318 constraint(ALLOC_IN_RC(ptr_reg));
5319 match(AddP reg off);
5320 op_cost(0);
5321 format %{ "[$reg, $off]" %}
5322 interface(MEMORY_INTER) %{
5323 base($reg);
5324 index(0xffffffff);
5325 scale(0x0);
5326 disp($off);
5327 %}
5328 %}
5329
5330 operand indOffL16(iRegP reg, immLoffset16 off)
5331 %{
5332 constraint(ALLOC_IN_RC(ptr_reg));
5333 match(AddP reg off);
5334 op_cost(0);
5335 format %{ "[$reg, $off]" %}
5336 interface(MEMORY_INTER) %{
5337 base($reg);
5338 index(0xffffffff);
5339 scale(0x0);
5340 disp($off);
5341 %}
5342 %}
5343
5344 operand indirectN(iRegN reg)
5345 %{
5346 predicate(CompressedOops::shift() == 0);
5347 constraint(ALLOC_IN_RC(ptr_reg));
5348 match(DecodeN reg);
5349 op_cost(0);
5350 format %{ "[$reg]\t# narrow" %}
5351 interface(MEMORY_INTER) %{
5352 base($reg);
5353 index(0xffffffff);
5354 scale(0x0);
5355 disp(0x0);
5356 %}
5357 %}
5358
5359 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5360 %{
5361 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5362 constraint(ALLOC_IN_RC(ptr_reg));
5363 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5364 op_cost(0);
5365 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5366 interface(MEMORY_INTER) %{
5367 base($reg);
5368 index($ireg);
5369 scale($scale);
5370 disp(0x0);
5371 %}
5372 %}
5373
5374 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5375 %{
5376 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5377 constraint(ALLOC_IN_RC(ptr_reg));
5378 match(AddP (DecodeN reg) (LShiftL lreg scale));
5379 op_cost(0);
5380 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5381 interface(MEMORY_INTER) %{
5382 base($reg);
5383 index($lreg);
5384 scale($scale);
5385 disp(0x0);
5386 %}
5387 %}
5388
5389 operand indIndexI2LN(iRegN reg, iRegI ireg)
5390 %{
5391 predicate(CompressedOops::shift() == 0);
5392 constraint(ALLOC_IN_RC(ptr_reg));
5393 match(AddP (DecodeN reg) (ConvI2L ireg));
5394 op_cost(0);
5395 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5396 interface(MEMORY_INTER) %{
5397 base($reg);
5398 index($ireg);
5399 scale(0x0);
5400 disp(0x0);
5401 %}
5402 %}
5403
5404 operand indIndexN(iRegN reg, iRegL lreg)
5405 %{
5406 predicate(CompressedOops::shift() == 0);
5407 constraint(ALLOC_IN_RC(ptr_reg));
5408 match(AddP (DecodeN reg) lreg);
5409 op_cost(0);
5410 format %{ "$reg, $lreg\t# narrow" %}
5411 interface(MEMORY_INTER) %{
5412 base($reg);
5413 index($lreg);
5414 scale(0x0);
5415 disp(0x0);
5416 %}
5417 %}
5418
5419 operand indOffIN(iRegN reg, immIOffset off)
5420 %{
5421 predicate(CompressedOops::shift() == 0);
5422 constraint(ALLOC_IN_RC(ptr_reg));
5423 match(AddP (DecodeN reg) off);
5424 op_cost(0);
5425 format %{ "[$reg, $off]\t# narrow" %}
5426 interface(MEMORY_INTER) %{
5427 base($reg);
5428 index(0xffffffff);
5429 scale(0x0);
5430 disp($off);
5431 %}
5432 %}
5433
5434 operand indOffLN(iRegN reg, immLoffset off)
5435 %{
5436 predicate(CompressedOops::shift() == 0);
5437 constraint(ALLOC_IN_RC(ptr_reg));
5438 match(AddP (DecodeN reg) off);
5439 op_cost(0);
5440 format %{ "[$reg, $off]\t# narrow" %}
5441 interface(MEMORY_INTER) %{
5442 base($reg);
5443 index(0xffffffff);
5444 scale(0x0);
5445 disp($off);
5446 %}
5447 %}
5448
5449
5450 //----------Special Memory Operands--------------------------------------------
5451 // Stack Slot Operand - This operand is used for loading and storing temporary
5452 // values on the stack where a match requires a value to
5453 // flow through memory.
5454 operand stackSlotP(sRegP reg)
5455 %{
5456 constraint(ALLOC_IN_RC(stack_slots));
5457 op_cost(100);
5458 // No match rule because this operand is only generated in matching
5459 // match(RegP);
5460 format %{ "[$reg]" %}
5461 interface(MEMORY_INTER) %{
5462 base(0x1e); // RSP
5463 index(0x0); // No Index
5464 scale(0x0); // No Scale
5465 disp($reg); // Stack Offset
5466 %}
5467 %}
5468
5469 operand stackSlotI(sRegI reg)
5470 %{
5471 constraint(ALLOC_IN_RC(stack_slots));
5472 // No match rule because this operand is only generated in matching
5473 // match(RegI);
5474 format %{ "[$reg]" %}
5475 interface(MEMORY_INTER) %{
5476 base(0x1e); // RSP
5477 index(0x0); // No Index
5478 scale(0x0); // No Scale
5479 disp($reg); // Stack Offset
5480 %}
5481 %}
5482
5483 operand stackSlotF(sRegF reg)
5484 %{
5485 constraint(ALLOC_IN_RC(stack_slots));
5486 // No match rule because this operand is only generated in matching
5487 // match(RegF);
5488 format %{ "[$reg]" %}
5489 interface(MEMORY_INTER) %{
5490 base(0x1e); // RSP
5491 index(0x0); // No Index
5492 scale(0x0); // No Scale
5493 disp($reg); // Stack Offset
5494 %}
5495 %}
5496
5497 operand stackSlotD(sRegD reg)
5498 %{
5499 constraint(ALLOC_IN_RC(stack_slots));
5500 // No match rule because this operand is only generated in matching
5501 // match(RegD);
5502 format %{ "[$reg]" %}
5503 interface(MEMORY_INTER) %{
5504 base(0x1e); // RSP
5505 index(0x0); // No Index
5506 scale(0x0); // No Scale
5507 disp($reg); // Stack Offset
5508 %}
5509 %}
5510
5511 operand stackSlotL(sRegL reg)
5512 %{
5513 constraint(ALLOC_IN_RC(stack_slots));
5514 // No match rule because this operand is only generated in matching
5515 // match(RegL);
5516 format %{ "[$reg]" %}
5517 interface(MEMORY_INTER) %{
5518 base(0x1e); // RSP
5519 index(0x0); // No Index
5520 scale(0x0); // No Scale
5521 disp($reg); // Stack Offset
5522 %}
5523 %}
5524
5525 // Operands for expressing Control Flow
5526 // NOTE: Label is a predefined operand which should not be redefined in
5527 // the AD file. It is generically handled within the ADLC.
5528
5529 //----------Conditional Branch Operands----------------------------------------
5530 // Comparison Op - This is the operation of the comparison, and is limited to
5531 // the following set of codes:
5532 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5533 //
5534 // Other attributes of the comparison, such as unsignedness, are specified
5535 // by the comparison instruction that sets a condition code flags register.
5536 // That result is represented by a flags operand whose subtype is appropriate
5537 // to the unsignedness (etc.) of the comparison.
5538 //
5539 // Later, the instruction which matches both the Comparison Op (a Bool) and
5540 // the flags (produced by the Cmp) specifies the coding of the comparison op
5541 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5542
5543 // used for signed integral comparisons and fp comparisons
5544
5545 operand cmpOp()
5546 %{
5547 match(Bool);
5548
5549 format %{ "" %}
5550 interface(COND_INTER) %{
5551 equal(0x0, "eq");
5552 not_equal(0x1, "ne");
5553 less(0xb, "lt");
5554 greater_equal(0xa, "ge");
5555 less_equal(0xd, "le");
5556 greater(0xc, "gt");
5557 overflow(0x6, "vs");
5558 no_overflow(0x7, "vc");
5559 %}
5560 %}
5561
5562 // used for unsigned integral comparisons
5563
5564 operand cmpOpU()
5565 %{
5566 match(Bool);
5567
5568 format %{ "" %}
5569 interface(COND_INTER) %{
5570 equal(0x0, "eq");
5571 not_equal(0x1, "ne");
5572 less(0x3, "lo");
5573 greater_equal(0x2, "hs");
5574 less_equal(0x9, "ls");
5575 greater(0x8, "hi");
5576 overflow(0x6, "vs");
5577 no_overflow(0x7, "vc");
5578 %}
5579 %}
5580
5581 // used for certain integral comparisons which can be
5582 // converted to cbxx or tbxx instructions
5583
5584 operand cmpOpEqNe()
5585 %{
5586 match(Bool);
5587 op_cost(0);
5588 predicate(n->as_Bool()->_test._test == BoolTest::ne
5589 || n->as_Bool()->_test._test == BoolTest::eq);
5590
5591 format %{ "" %}
5592 interface(COND_INTER) %{
5593 equal(0x0, "eq");
5594 not_equal(0x1, "ne");
5595 less(0xb, "lt");
5596 greater_equal(0xa, "ge");
5597 less_equal(0xd, "le");
5598 greater(0xc, "gt");
5599 overflow(0x6, "vs");
5600 no_overflow(0x7, "vc");
5601 %}
5602 %}
5603
5604 // used for certain integral comparisons which can be
5605 // converted to cbxx or tbxx instructions
5606
5607 operand cmpOpLtGe()
5608 %{
5609 match(Bool);
5610 op_cost(0);
5611
5612 predicate(n->as_Bool()->_test._test == BoolTest::lt
5613 || n->as_Bool()->_test._test == BoolTest::ge);
5614
5615 format %{ "" %}
5616 interface(COND_INTER) %{
5617 equal(0x0, "eq");
5618 not_equal(0x1, "ne");
5619 less(0xb, "lt");
5620 greater_equal(0xa, "ge");
5621 less_equal(0xd, "le");
5622 greater(0xc, "gt");
5623 overflow(0x6, "vs");
5624 no_overflow(0x7, "vc");
5625 %}
5626 %}
5627
5628 // used for certain unsigned integral comparisons which can be
5629 // converted to cbxx or tbxx instructions
5630
5631 operand cmpOpUEqNeLeGt()
5632 %{
5633 match(Bool);
5634 op_cost(0);
5635
5636 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5637 n->as_Bool()->_test._test == BoolTest::ne ||
5638 n->as_Bool()->_test._test == BoolTest::le ||
5639 n->as_Bool()->_test._test == BoolTest::gt);
5640
5641 format %{ "" %}
5642 interface(COND_INTER) %{
5643 equal(0x0, "eq");
5644 not_equal(0x1, "ne");
5645 less(0x3, "lo");
5646 greater_equal(0x2, "hs");
5647 less_equal(0x9, "ls");
5648 greater(0x8, "hi");
5649 overflow(0x6, "vs");
5650 no_overflow(0x7, "vc");
5651 %}
5652 %}
5653
5654 // Special operand allowing long args to int ops to be truncated for free
5655
5656 operand iRegL2I(iRegL reg) %{
5657
5658 op_cost(0);
5659
5660 match(ConvL2I reg);
5661
5662 format %{ "l2i($reg)" %}
5663
5664 interface(REG_INTER)
5665 %}
5666
5667 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5668 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5669 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5670 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5671
5672 //----------OPERAND CLASSES----------------------------------------------------
5673 // Operand Classes are groups of operands that are used as to simplify
5674 // instruction definitions by not requiring the AD writer to specify
5675 // separate instructions for every form of operand when the
5676 // instruction accepts multiple operand types with the same basic
5677 // encoding and format. The classic case of this is memory operands.
5678
5679 // memory is used to define read/write location for load/store
5680 // instruction defs. we can turn a memory op into an Address
5681
5682 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5683 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
5684
5685 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5686 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN);
5687
5688 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5689 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
5690
5691 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5692 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
5693
5694 // All of the memory operands. For the pipeline description.
5695 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5696 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5697 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN);
5698
5699
5700 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5701 // operations. it allows the src to be either an iRegI or a (ConvL2I
5702 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5703 // can be elided because the 32-bit instruction will just employ the
5704 // lower 32 bits anyway.
5705 //
5706 // n.b. this does not elide all L2I conversions. if the truncated
5707 // value is consumed by more than one operation then the ConvL2I
5708 // cannot be bundled into the consuming nodes so an l2i gets planted
5709 // (actually a movw $dst $src) and the downstream instructions consume
5710 // the result of the l2i as an iRegI input. That's a shame since the
5711 // movw is actually redundant but its not too costly.
5712
5713 opclass iRegIorL2I(iRegI, iRegL2I);
5714
5715 //----------PIPELINE-----------------------------------------------------------
5716 // Rules which define the behavior of the target architectures pipeline.
5717
5718 // For specific pipelines, eg A53, define the stages of that pipeline
5719 //pipe_desc(ISS, EX1, EX2, WR);
5720 #define ISS S0
5721 #define EX1 S1
5722 #define EX2 S2
5723 #define WR S3
5724
5725 // Integer ALU reg operation
5726 pipeline %{
5727
5728 attributes %{
5729 // ARM instructions are of fixed length
5730 fixed_size_instructions; // Fixed size instructions TODO does
5731 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5732 // ARM instructions come in 32-bit word units
5733 instruction_unit_size = 4; // An instruction is 4 bytes long
5734 instruction_fetch_unit_size = 64; // The processor fetches one line
5735 instruction_fetch_units = 1; // of 64 bytes
5736
5737 // List of nop instructions
5738 nops( MachNop );
5739 %}
5740
5741 // We don't use an actual pipeline model so don't care about resources
5742 // or description. we do use pipeline classes to introduce fixed
5743 // latencies
5744
5745 //----------RESOURCES----------------------------------------------------------
5746 // Resources are the functional units available to the machine
5747
5748 resources( INS0, INS1, INS01 = INS0 | INS1,
5749 ALU0, ALU1, ALU = ALU0 | ALU1,
5750 MAC,
5751 DIV,
5752 BRANCH,
5753 LDST,
5754 NEON_FP);
5755
5756 //----------PIPELINE DESCRIPTION-----------------------------------------------
5757 // Pipeline Description specifies the stages in the machine's pipeline
5758
5759 // Define the pipeline as a generic 6 stage pipeline
5760 pipe_desc(S0, S1, S2, S3, S4, S5);
5761
5762 //----------PIPELINE CLASSES---------------------------------------------------
5763 // Pipeline Classes describe the stages in which input and output are
5764 // referenced by the hardware pipeline.
5765
5766 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5767 %{
5768 single_instruction;
5769 src1 : S1(read);
5770 src2 : S2(read);
5771 dst : S5(write);
5772 INS01 : ISS;
5773 NEON_FP : S5;
5774 %}
5775
5776 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5777 %{
5778 single_instruction;
5779 src1 : S1(read);
5780 src2 : S2(read);
5781 dst : S5(write);
5782 INS01 : ISS;
5783 NEON_FP : S5;
5784 %}
5785
5786 pipe_class fp_uop_s(vRegF dst, vRegF src)
5787 %{
5788 single_instruction;
5789 src : S1(read);
5790 dst : S5(write);
5791 INS01 : ISS;
5792 NEON_FP : S5;
5793 %}
5794
5795 pipe_class fp_uop_d(vRegD dst, vRegD src)
5796 %{
5797 single_instruction;
5798 src : S1(read);
5799 dst : S5(write);
5800 INS01 : ISS;
5801 NEON_FP : S5;
5802 %}
5803
5804 pipe_class fp_d2f(vRegF dst, vRegD src)
5805 %{
5806 single_instruction;
5807 src : S1(read);
5808 dst : S5(write);
5809 INS01 : ISS;
5810 NEON_FP : S5;
5811 %}
5812
5813 pipe_class fp_f2d(vRegD dst, vRegF src)
5814 %{
5815 single_instruction;
5816 src : S1(read);
5817 dst : S5(write);
5818 INS01 : ISS;
5819 NEON_FP : S5;
5820 %}
5821
5822 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5823 %{
5824 single_instruction;
5825 src : S1(read);
5826 dst : S5(write);
5827 INS01 : ISS;
5828 NEON_FP : S5;
5829 %}
5830
5831 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5832 %{
5833 single_instruction;
5834 src : S1(read);
5835 dst : S5(write);
5836 INS01 : ISS;
5837 NEON_FP : S5;
5838 %}
5839
5840 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5841 %{
5842 single_instruction;
5843 src : S1(read);
5844 dst : S5(write);
5845 INS01 : ISS;
5846 NEON_FP : S5;
5847 %}
5848
5849 pipe_class fp_l2f(vRegF dst, iRegL src)
5850 %{
5851 single_instruction;
5852 src : S1(read);
5853 dst : S5(write);
5854 INS01 : ISS;
5855 NEON_FP : S5;
5856 %}
5857
5858 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5859 %{
5860 single_instruction;
5861 src : S1(read);
5862 dst : S5(write);
5863 INS01 : ISS;
5864 NEON_FP : S5;
5865 %}
5866
5867 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5868 %{
5869 single_instruction;
5870 src : S1(read);
5871 dst : S5(write);
5872 INS01 : ISS;
5873 NEON_FP : S5;
5874 %}
5875
5876 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5877 %{
5878 single_instruction;
5879 src : S1(read);
5880 dst : S5(write);
5881 INS01 : ISS;
5882 NEON_FP : S5;
5883 %}
5884
5885 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5886 %{
5887 single_instruction;
5888 src : S1(read);
5889 dst : S5(write);
5890 INS01 : ISS;
5891 NEON_FP : S5;
5892 %}
5893
5894 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5895 %{
5896 single_instruction;
5897 src1 : S1(read);
5898 src2 : S2(read);
5899 dst : S5(write);
5900 INS0 : ISS;
5901 NEON_FP : S5;
5902 %}
5903
5904 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5905 %{
5906 single_instruction;
5907 src1 : S1(read);
5908 src2 : S2(read);
5909 dst : S5(write);
5910 INS0 : ISS;
5911 NEON_FP : S5;
5912 %}
5913
5914 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5915 %{
5916 single_instruction;
5917 cr : S1(read);
5918 src1 : S1(read);
5919 src2 : S1(read);
5920 dst : S3(write);
5921 INS01 : ISS;
5922 NEON_FP : S3;
5923 %}
5924
5925 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
5926 %{
5927 single_instruction;
5928 cr : S1(read);
5929 src1 : S1(read);
5930 src2 : S1(read);
5931 dst : S3(write);
5932 INS01 : ISS;
5933 NEON_FP : S3;
5934 %}
5935
5936 pipe_class fp_imm_s(vRegF dst)
5937 %{
5938 single_instruction;
5939 dst : S3(write);
5940 INS01 : ISS;
5941 NEON_FP : S3;
5942 %}
5943
5944 pipe_class fp_imm_d(vRegD dst)
5945 %{
5946 single_instruction;
5947 dst : S3(write);
5948 INS01 : ISS;
5949 NEON_FP : S3;
5950 %}
5951
5952 pipe_class fp_load_constant_s(vRegF dst)
5953 %{
5954 single_instruction;
5955 dst : S4(write);
5956 INS01 : ISS;
5957 NEON_FP : S4;
5958 %}
5959
5960 pipe_class fp_load_constant_d(vRegD dst)
5961 %{
5962 single_instruction;
5963 dst : S4(write);
5964 INS01 : ISS;
5965 NEON_FP : S4;
5966 %}
5967
5968 //------- Integer ALU operations --------------------------
5969
5970 // Integer ALU reg-reg operation
5971 // Operands needed in EX1, result generated in EX2
5972 // Eg. ADD x0, x1, x2
5973 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
5974 %{
5975 single_instruction;
5976 dst : EX2(write);
5977 src1 : EX1(read);
5978 src2 : EX1(read);
5979 INS01 : ISS; // Dual issue as instruction 0 or 1
5980 ALU : EX2;
5981 %}
5982
5983 // Integer ALU reg-reg operation with constant shift
5984 // Shifted register must be available in LATE_ISS instead of EX1
5985 // Eg. ADD x0, x1, x2, LSL #2
5986 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
5987 %{
5988 single_instruction;
5989 dst : EX2(write);
5990 src1 : EX1(read);
5991 src2 : ISS(read);
5992 INS01 : ISS;
5993 ALU : EX2;
5994 %}
5995
5996 // Integer ALU reg operation with constant shift
5997 // Eg. LSL x0, x1, #shift
5998 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
5999 %{
6000 single_instruction;
6001 dst : EX2(write);
6002 src1 : ISS(read);
6003 INS01 : ISS;
6004 ALU : EX2;
6005 %}
6006
6007 // Integer ALU reg-reg operation with variable shift
6008 // Both operands must be available in LATE_ISS instead of EX1
6009 // Result is available in EX1 instead of EX2
6010 // Eg. LSLV x0, x1, x2
6011 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6012 %{
6013 single_instruction;
6014 dst : EX1(write);
6015 src1 : ISS(read);
6016 src2 : ISS(read);
6017 INS01 : ISS;
6018 ALU : EX1;
6019 %}
6020
6021 // Integer ALU reg-reg operation with extract
6022 // As for _vshift above, but result generated in EX2
6023 // Eg. EXTR x0, x1, x2, #N
6024 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6025 %{
6026 single_instruction;
6027 dst : EX2(write);
6028 src1 : ISS(read);
6029 src2 : ISS(read);
6030 INS1 : ISS; // Can only dual issue as Instruction 1
6031 ALU : EX1;
6032 %}
6033
6034 // Integer ALU reg operation
6035 // Eg. NEG x0, x1
6036 pipe_class ialu_reg(iRegI dst, iRegI src)
6037 %{
6038 single_instruction;
6039 dst : EX2(write);
6040 src : EX1(read);
6041 INS01 : ISS;
6042 ALU : EX2;
6043 %}
6044
6045 // Integer ALU reg mmediate operation
6046 // Eg. ADD x0, x1, #N
6047 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6048 %{
6049 single_instruction;
6050 dst : EX2(write);
6051 src1 : EX1(read);
6052 INS01 : ISS;
6053 ALU : EX2;
6054 %}
6055
6056 // Integer ALU immediate operation (no source operands)
6057 // Eg. MOV x0, #N
6058 pipe_class ialu_imm(iRegI dst)
6059 %{
6060 single_instruction;
6061 dst : EX1(write);
6062 INS01 : ISS;
6063 ALU : EX1;
6064 %}
6065
6066 //------- Compare operation -------------------------------
6067
6068 // Compare reg-reg
6069 // Eg. CMP x0, x1
6070 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6071 %{
6072 single_instruction;
6073 // fixed_latency(16);
6074 cr : EX2(write);
6075 op1 : EX1(read);
6076 op2 : EX1(read);
6077 INS01 : ISS;
6078 ALU : EX2;
6079 %}
6080
6081 // Compare reg-reg
6082 // Eg. CMP x0, #N
6083 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6084 %{
6085 single_instruction;
6086 // fixed_latency(16);
6087 cr : EX2(write);
6088 op1 : EX1(read);
6089 INS01 : ISS;
6090 ALU : EX2;
6091 %}
6092
6093 //------- Conditional instructions ------------------------
6094
6095 // Conditional no operands
6096 // Eg. CSINC x0, zr, zr, <cond>
6097 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6098 %{
6099 single_instruction;
6100 cr : EX1(read);
6101 dst : EX2(write);
6102 INS01 : ISS;
6103 ALU : EX2;
6104 %}
6105
6106 // Conditional 2 operand
6107 // EG. CSEL X0, X1, X2, <cond>
6108 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6109 %{
6110 single_instruction;
6111 cr : EX1(read);
6112 src1 : EX1(read);
6113 src2 : EX1(read);
6114 dst : EX2(write);
6115 INS01 : ISS;
6116 ALU : EX2;
6117 %}
6118
6119 // Conditional 2 operand
6120 // EG. CSEL X0, X1, X2, <cond>
6121 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6122 %{
6123 single_instruction;
6124 cr : EX1(read);
6125 src : EX1(read);
6126 dst : EX2(write);
6127 INS01 : ISS;
6128 ALU : EX2;
6129 %}
6130
6131 //------- Multiply pipeline operations --------------------
6132
6133 // Multiply reg-reg
6134 // Eg. MUL w0, w1, w2
6135 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6136 %{
6137 single_instruction;
6138 dst : WR(write);
6139 src1 : ISS(read);
6140 src2 : ISS(read);
6141 INS01 : ISS;
6142 MAC : WR;
6143 %}
6144
6145 // Multiply accumulate
6146 // Eg. MADD w0, w1, w2, w3
6147 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6148 %{
6149 single_instruction;
6150 dst : WR(write);
6151 src1 : ISS(read);
6152 src2 : ISS(read);
6153 src3 : ISS(read);
6154 INS01 : ISS;
6155 MAC : WR;
6156 %}
6157
6158 // Eg. MUL w0, w1, w2
6159 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6160 %{
6161 single_instruction;
6162 fixed_latency(3); // Maximum latency for 64 bit mul
6163 dst : WR(write);
6164 src1 : ISS(read);
6165 src2 : ISS(read);
6166 INS01 : ISS;
6167 MAC : WR;
6168 %}
6169
6170 // Multiply accumulate
6171 // Eg. MADD w0, w1, w2, w3
6172 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6173 %{
6174 single_instruction;
6175 fixed_latency(3); // Maximum latency for 64 bit mul
6176 dst : WR(write);
6177 src1 : ISS(read);
6178 src2 : ISS(read);
6179 src3 : ISS(read);
6180 INS01 : ISS;
6181 MAC : WR;
6182 %}
6183
6184 //------- Divide pipeline operations --------------------
6185
6186 // Eg. SDIV w0, w1, w2
6187 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6188 %{
6189 single_instruction;
6190 fixed_latency(8); // Maximum latency for 32 bit divide
6191 dst : WR(write);
6192 src1 : ISS(read);
6193 src2 : ISS(read);
6194 INS0 : ISS; // Can only dual issue as instruction 0
6195 DIV : WR;
6196 %}
6197
6198 // Eg. SDIV x0, x1, x2
6199 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6200 %{
6201 single_instruction;
6202 fixed_latency(16); // Maximum latency for 64 bit divide
6203 dst : WR(write);
6204 src1 : ISS(read);
6205 src2 : ISS(read);
6206 INS0 : ISS; // Can only dual issue as instruction 0
6207 DIV : WR;
6208 %}
6209
6210 //------- Load pipeline operations ------------------------
6211
6212 // Load - prefetch
6213 // Eg. PFRM <mem>
6214 pipe_class iload_prefetch(memory mem)
6215 %{
6216 single_instruction;
6217 mem : ISS(read);
6218 INS01 : ISS;
6219 LDST : WR;
6220 %}
6221
6222 // Load - reg, mem
6223 // Eg. LDR x0, <mem>
6224 pipe_class iload_reg_mem(iRegI dst, memory mem)
6225 %{
6226 single_instruction;
6227 dst : WR(write);
6228 mem : ISS(read);
6229 INS01 : ISS;
6230 LDST : WR;
6231 %}
6232
6233 // Load - reg, reg
6234 // Eg. LDR x0, [sp, x1]
6235 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6236 %{
6237 single_instruction;
6238 dst : WR(write);
6239 src : ISS(read);
6240 INS01 : ISS;
6241 LDST : WR;
6242 %}
6243
6244 //------- Store pipeline operations -----------------------
6245
6246 // Store - zr, mem
6247 // Eg. STR zr, <mem>
6248 pipe_class istore_mem(memory mem)
6249 %{
6250 single_instruction;
6251 mem : ISS(read);
6252 INS01 : ISS;
6253 LDST : WR;
6254 %}
6255
6256 // Store - reg, mem
6257 // Eg. STR x0, <mem>
6258 pipe_class istore_reg_mem(iRegI src, memory mem)
6259 %{
6260 single_instruction;
6261 mem : ISS(read);
6262 src : EX2(read);
6263 INS01 : ISS;
6264 LDST : WR;
6265 %}
6266
6267 // Store - reg, reg
6268 // Eg. STR x0, [sp, x1]
6269 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6270 %{
6271 single_instruction;
6272 dst : ISS(read);
6273 src : EX2(read);
6274 INS01 : ISS;
6275 LDST : WR;
6276 %}
6277
6278 //------- Store pipeline operations -----------------------
6279
6280 // Branch
6281 pipe_class pipe_branch()
6282 %{
6283 single_instruction;
6284 INS01 : ISS;
6285 BRANCH : EX1;
6286 %}
6287
6288 // Conditional branch
6289 pipe_class pipe_branch_cond(rFlagsReg cr)
6290 %{
6291 single_instruction;
6292 cr : EX1(read);
6293 INS01 : ISS;
6294 BRANCH : EX1;
6295 %}
6296
6297 // Compare & Branch
6298 // EG. CBZ/CBNZ
6299 pipe_class pipe_cmp_branch(iRegI op1)
6300 %{
6301 single_instruction;
6302 op1 : EX1(read);
6303 INS01 : ISS;
6304 BRANCH : EX1;
6305 %}
6306
6307 //------- Synchronisation operations ----------------------
6308
6309 // Any operation requiring serialization.
6310 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6311 pipe_class pipe_serial()
6312 %{
6313 single_instruction;
6314 force_serialization;
6315 fixed_latency(16);
6316 INS01 : ISS(2); // Cannot dual issue with any other instruction
6317 LDST : WR;
6318 %}
6319
6320 // Generic big/slow expanded idiom - also serialized
6321 pipe_class pipe_slow()
6322 %{
6323 instruction_count(10);
6324 multiple_bundles;
6325 force_serialization;
6326 fixed_latency(16);
6327 INS01 : ISS(2); // Cannot dual issue with any other instruction
6328 LDST : WR;
6329 %}
6330
6331 // Empty pipeline class
6332 pipe_class pipe_class_empty()
6333 %{
6334 single_instruction;
6335 fixed_latency(0);
6336 %}
6337
6338 // Default pipeline class.
6339 pipe_class pipe_class_default()
6340 %{
6341 single_instruction;
6342 fixed_latency(2);
6343 %}
6344
6345 // Pipeline class for compares.
6346 pipe_class pipe_class_compare()
6347 %{
6348 single_instruction;
6349 fixed_latency(16);
6350 %}
6351
6352 // Pipeline class for memory operations.
6353 pipe_class pipe_class_memory()
6354 %{
6355 single_instruction;
6356 fixed_latency(16);
6357 %}
6358
6359 // Pipeline class for call.
6360 pipe_class pipe_class_call()
6361 %{
6362 single_instruction;
6363 fixed_latency(100);
6364 %}
6365
6366 // Define the class for the Nop node.
6367 define %{
6368 MachNop = pipe_class_empty;
6369 %}
6370
6371 %}
6372 //----------INSTRUCTIONS-------------------------------------------------------
6373 //
6374 // match -- States which machine-independent subtree may be replaced
6375 // by this instruction.
6376 // ins_cost -- The estimated cost of this instruction is used by instruction
6377 // selection to identify a minimum cost tree of machine
6378 // instructions that matches a tree of machine-independent
6379 // instructions.
6380 // format -- A string providing the disassembly for this instruction.
6381 // The value of an instruction's operand may be inserted
6382 // by referring to it with a '$' prefix.
6383 // opcode -- Three instruction opcodes may be provided. These are referred
6384 // to within an encode class as $primary, $secondary, and $tertiary
6385 // rrspectively. The primary opcode is commonly used to
6386 // indicate the type of machine instruction, while secondary
6387 // and tertiary are often used for prefix options or addressing
6388 // modes.
6389 // ins_encode -- A list of encode classes with parameters. The encode class
6390 // name must have been defined in an 'enc_class' specification
6391 // in the encode section of the architecture description.
6392
6393 // ============================================================================
6394 // Memory (Load/Store) Instructions
6395
6396 // Load Instructions
6397
6398 // Load Byte (8 bit signed)
6399 instruct loadB(iRegINoSp dst, memory1 mem)
6400 %{
6401 match(Set dst (LoadB mem));
6402 predicate(!needs_acquiring_load(n));
6403
6404 ins_cost(4 * INSN_COST);
6405 format %{ "ldrsbw $dst, $mem\t# byte" %}
6406
6407 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6408
6409 ins_pipe(iload_reg_mem);
6410 %}
6411
6412 // Load Byte (8 bit signed) into long
6413 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6414 %{
6415 match(Set dst (ConvI2L (LoadB mem)));
6416 predicate(!needs_acquiring_load(n->in(1)));
6417
6418 ins_cost(4 * INSN_COST);
6419 format %{ "ldrsb $dst, $mem\t# byte" %}
6420
6421 ins_encode(aarch64_enc_ldrsb(dst, mem));
6422
6423 ins_pipe(iload_reg_mem);
6424 %}
6425
6426 // Load Byte (8 bit unsigned)
6427 instruct loadUB(iRegINoSp dst, memory1 mem)
6428 %{
6429 match(Set dst (LoadUB mem));
6430 predicate(!needs_acquiring_load(n));
6431
6432 ins_cost(4 * INSN_COST);
6433 format %{ "ldrbw $dst, $mem\t# byte" %}
6434
6435 ins_encode(aarch64_enc_ldrb(dst, mem));
6436
6437 ins_pipe(iload_reg_mem);
6438 %}
6439
6440 // Load Byte (8 bit unsigned) into long
6441 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6442 %{
6443 match(Set dst (ConvI2L (LoadUB mem)));
6444 predicate(!needs_acquiring_load(n->in(1)));
6445
6446 ins_cost(4 * INSN_COST);
6447 format %{ "ldrb $dst, $mem\t# byte" %}
6448
6449 ins_encode(aarch64_enc_ldrb(dst, mem));
6450
6451 ins_pipe(iload_reg_mem);
6452 %}
6453
6454 // Load Short (16 bit signed)
6455 instruct loadS(iRegINoSp dst, memory2 mem)
6456 %{
6457 match(Set dst (LoadS mem));
6458 predicate(!needs_acquiring_load(n));
6459
6460 ins_cost(4 * INSN_COST);
6461 format %{ "ldrshw $dst, $mem\t# short" %}
6462
6463 ins_encode(aarch64_enc_ldrshw(dst, mem));
6464
6465 ins_pipe(iload_reg_mem);
6466 %}
6467
6468 // Load Short (16 bit signed) into long
6469 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6470 %{
6471 match(Set dst (ConvI2L (LoadS mem)));
6472 predicate(!needs_acquiring_load(n->in(1)));
6473
6474 ins_cost(4 * INSN_COST);
6475 format %{ "ldrsh $dst, $mem\t# short" %}
6476
6477 ins_encode(aarch64_enc_ldrsh(dst, mem));
6478
6479 ins_pipe(iload_reg_mem);
6480 %}
6481
6482 // Load Char (16 bit unsigned)
6483 instruct loadUS(iRegINoSp dst, memory2 mem)
6484 %{
6485 match(Set dst (LoadUS mem));
6486 predicate(!needs_acquiring_load(n));
6487
6488 ins_cost(4 * INSN_COST);
6489 format %{ "ldrh $dst, $mem\t# short" %}
6490
6491 ins_encode(aarch64_enc_ldrh(dst, mem));
6492
6493 ins_pipe(iload_reg_mem);
6494 %}
6495
6496 // Load Short/Char (16 bit unsigned) into long
6497 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6498 %{
6499 match(Set dst (ConvI2L (LoadUS mem)));
6500 predicate(!needs_acquiring_load(n->in(1)));
6501
6502 ins_cost(4 * INSN_COST);
6503 format %{ "ldrh $dst, $mem\t# short" %}
6504
6505 ins_encode(aarch64_enc_ldrh(dst, mem));
6506
6507 ins_pipe(iload_reg_mem);
6508 %}
6509
6510 // Load Integer (32 bit signed)
6511 instruct loadI(iRegINoSp dst, memory4 mem)
6512 %{
6513 match(Set dst (LoadI mem));
6514 predicate(!needs_acquiring_load(n));
6515
6516 ins_cost(4 * INSN_COST);
6517 format %{ "ldrw $dst, $mem\t# int" %}
6518
6519 ins_encode(aarch64_enc_ldrw(dst, mem));
6520
6521 ins_pipe(iload_reg_mem);
6522 %}
6523
6524 // Load Integer (32 bit signed) into long
6525 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6526 %{
6527 match(Set dst (ConvI2L (LoadI mem)));
6528 predicate(!needs_acquiring_load(n->in(1)));
6529
6530 ins_cost(4 * INSN_COST);
6531 format %{ "ldrsw $dst, $mem\t# int" %}
6532
6533 ins_encode(aarch64_enc_ldrsw(dst, mem));
6534
6535 ins_pipe(iload_reg_mem);
6536 %}
6537
6538 // Load Integer (32 bit unsigned) into long
6539 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6540 %{
6541 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6542 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6543
6544 ins_cost(4 * INSN_COST);
6545 format %{ "ldrw $dst, $mem\t# int" %}
6546
6547 ins_encode(aarch64_enc_ldrw(dst, mem));
6548
6549 ins_pipe(iload_reg_mem);
6550 %}
6551
6552 // Load Long (64 bit signed)
6553 instruct loadL(iRegLNoSp dst, memory8 mem)
6554 %{
6555 match(Set dst (LoadL mem));
6556 predicate(!needs_acquiring_load(n));
6557
6558 ins_cost(4 * INSN_COST);
6559 format %{ "ldr $dst, $mem\t# int" %}
6560
6561 ins_encode(aarch64_enc_ldr(dst, mem));
6562
6563 ins_pipe(iload_reg_mem);
6564 %}
6565
6566 // Load Range
6567 instruct loadRange(iRegINoSp dst, memory4 mem)
6568 %{
6569 match(Set dst (LoadRange mem));
6570
6571 ins_cost(4 * INSN_COST);
6572 format %{ "ldrw $dst, $mem\t# range" %}
6573
6574 ins_encode(aarch64_enc_ldrw(dst, mem));
6575
6576 ins_pipe(iload_reg_mem);
6577 %}
6578
6579 // Load Pointer
6580 instruct loadP(iRegPNoSp dst, memory8 mem)
6581 %{
6582 match(Set dst (LoadP mem));
6583 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6584
6585 ins_cost(4 * INSN_COST);
6586 format %{ "ldr $dst, $mem\t# ptr" %}
6587
6588 ins_encode(aarch64_enc_ldr(dst, mem));
6589
6590 ins_pipe(iload_reg_mem);
6591 %}
6592
6593 // Load Compressed Pointer
6594 instruct loadN(iRegNNoSp dst, memory4 mem)
6595 %{
6596 match(Set dst (LoadN mem));
6597 predicate(!needs_acquiring_load(n));
6598
6599 ins_cost(4 * INSN_COST);
6600 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6601
6602 ins_encode(aarch64_enc_ldrw(dst, mem));
6603
6604 ins_pipe(iload_reg_mem);
6605 %}
6606
6607 // Load Klass Pointer
6608 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6609 %{
6610 match(Set dst (LoadKlass mem));
6611 predicate(!needs_acquiring_load(n));
6612
6613 ins_cost(4 * INSN_COST);
6614 format %{ "ldr $dst, $mem\t# class" %}
6615
6616 ins_encode(aarch64_enc_ldr(dst, mem));
6617
6618 ins_pipe(iload_reg_mem);
6619 %}
6620
6621 // Load Narrow Klass Pointer
6622 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6623 %{
6624 match(Set dst (LoadNKlass mem));
6625 predicate(!needs_acquiring_load(n) && !UseCompactObjectHeaders);
6626
6627 ins_cost(4 * INSN_COST);
6628 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6629
6630 ins_encode(aarch64_enc_ldrw(dst, mem));
6631
6632 ins_pipe(iload_reg_mem);
6633 %}
6634
6635 instruct loadNKlassCompactHeaders(iRegNNoSp dst, memory4 mem, rFlagsReg cr)
6636 %{
6637 match(Set dst (LoadNKlass mem));
6638 effect(KILL cr);
6639 predicate(!needs_acquiring_load(n) && UseCompactObjectHeaders);
6640
6641 ins_cost(4 * INSN_COST);
6642 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6643 ins_encode %{
6644 __ load_nklass_compact($dst$$Register, $mem$$base$$Register, $mem$$index$$Register, $mem$$scale, $mem$$disp);
6645 %}
6646 ins_pipe(pipe_slow);
6647 %}
6648
6649 // Load Float
6650 instruct loadF(vRegF dst, memory4 mem)
6651 %{
6652 match(Set dst (LoadF mem));
6653 predicate(!needs_acquiring_load(n));
6654
6655 ins_cost(4 * INSN_COST);
6656 format %{ "ldrs $dst, $mem\t# float" %}
6657
6658 ins_encode( aarch64_enc_ldrs(dst, mem) );
6659
6660 ins_pipe(pipe_class_memory);
6661 %}
6662
6663 // Load Double
6664 instruct loadD(vRegD dst, memory8 mem)
6665 %{
6666 match(Set dst (LoadD mem));
6667 predicate(!needs_acquiring_load(n));
6668
6669 ins_cost(4 * INSN_COST);
6670 format %{ "ldrd $dst, $mem\t# double" %}
6671
6672 ins_encode( aarch64_enc_ldrd(dst, mem) );
6673
6674 ins_pipe(pipe_class_memory);
6675 %}
6676
6677
6678 // Load Int Constant
6679 instruct loadConI(iRegINoSp dst, immI src)
6680 %{
6681 match(Set dst src);
6682
6683 ins_cost(INSN_COST);
6684 format %{ "mov $dst, $src\t# int" %}
6685
6686 ins_encode( aarch64_enc_movw_imm(dst, src) );
6687
6688 ins_pipe(ialu_imm);
6689 %}
6690
6691 // Load Long Constant
6692 instruct loadConL(iRegLNoSp dst, immL src)
6693 %{
6694 match(Set dst src);
6695
6696 ins_cost(INSN_COST);
6697 format %{ "mov $dst, $src\t# long" %}
6698
6699 ins_encode( aarch64_enc_mov_imm(dst, src) );
6700
6701 ins_pipe(ialu_imm);
6702 %}
6703
6704 // Load Pointer Constant
6705
6706 instruct loadConP(iRegPNoSp dst, immP con)
6707 %{
6708 match(Set dst con);
6709
6710 ins_cost(INSN_COST * 4);
6711 format %{
6712 "mov $dst, $con\t# ptr\n\t"
6713 %}
6714
6715 ins_encode(aarch64_enc_mov_p(dst, con));
6716
6717 ins_pipe(ialu_imm);
6718 %}
6719
6720 // Load Null Pointer Constant
6721
6722 instruct loadConP0(iRegPNoSp dst, immP0 con)
6723 %{
6724 match(Set dst con);
6725
6726 ins_cost(INSN_COST);
6727 format %{ "mov $dst, $con\t# nullptr ptr" %}
6728
6729 ins_encode(aarch64_enc_mov_p0(dst, con));
6730
6731 ins_pipe(ialu_imm);
6732 %}
6733
6734 // Load Pointer Constant One
6735
6736 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6737 %{
6738 match(Set dst con);
6739
6740 ins_cost(INSN_COST);
6741 format %{ "mov $dst, $con\t# nullptr ptr" %}
6742
6743 ins_encode(aarch64_enc_mov_p1(dst, con));
6744
6745 ins_pipe(ialu_imm);
6746 %}
6747
6748 // Load Byte Map Base Constant
6749
6750 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6751 %{
6752 match(Set dst con);
6753
6754 ins_cost(INSN_COST);
6755 format %{ "adr $dst, $con\t# Byte Map Base" %}
6756
6757 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6758
6759 ins_pipe(ialu_imm);
6760 %}
6761
6762 // Load Narrow Pointer Constant
6763
6764 instruct loadConN(iRegNNoSp dst, immN con)
6765 %{
6766 match(Set dst con);
6767
6768 ins_cost(INSN_COST * 4);
6769 format %{ "mov $dst, $con\t# compressed ptr" %}
6770
6771 ins_encode(aarch64_enc_mov_n(dst, con));
6772
6773 ins_pipe(ialu_imm);
6774 %}
6775
6776 // Load Narrow Null Pointer Constant
6777
6778 instruct loadConN0(iRegNNoSp dst, immN0 con)
6779 %{
6780 match(Set dst con);
6781
6782 ins_cost(INSN_COST);
6783 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6784
6785 ins_encode(aarch64_enc_mov_n0(dst, con));
6786
6787 ins_pipe(ialu_imm);
6788 %}
6789
6790 // Load Narrow Klass Constant
6791
6792 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6793 %{
6794 match(Set dst con);
6795
6796 ins_cost(INSN_COST);
6797 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6798
6799 ins_encode(aarch64_enc_mov_nk(dst, con));
6800
6801 ins_pipe(ialu_imm);
6802 %}
6803
6804 // Load Packed Float Constant
6805
6806 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6807 match(Set dst con);
6808 ins_cost(INSN_COST * 4);
6809 format %{ "fmovs $dst, $con"%}
6810 ins_encode %{
6811 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6812 %}
6813
6814 ins_pipe(fp_imm_s);
6815 %}
6816
6817 // Load Float Constant
6818
6819 instruct loadConF(vRegF dst, immF con) %{
6820 match(Set dst con);
6821
6822 ins_cost(INSN_COST * 4);
6823
6824 format %{
6825 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6826 %}
6827
6828 ins_encode %{
6829 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6830 %}
6831
6832 ins_pipe(fp_load_constant_s);
6833 %}
6834
6835 // Load Packed Double Constant
6836
6837 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6838 match(Set dst con);
6839 ins_cost(INSN_COST);
6840 format %{ "fmovd $dst, $con"%}
6841 ins_encode %{
6842 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6843 %}
6844
6845 ins_pipe(fp_imm_d);
6846 %}
6847
6848 // Load Double Constant
6849
6850 instruct loadConD(vRegD dst, immD con) %{
6851 match(Set dst con);
6852
6853 ins_cost(INSN_COST * 5);
6854 format %{
6855 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6856 %}
6857
6858 ins_encode %{
6859 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6860 %}
6861
6862 ins_pipe(fp_load_constant_d);
6863 %}
6864
6865 // Store Instructions
6866
6867 // Store CMS card-mark Immediate
6868 instruct storeimmCM0(immI0 zero, memory1 mem)
6869 %{
6870 match(Set mem (StoreCM mem zero));
6871
6872 ins_cost(INSN_COST);
6873 format %{ "storestore (elided)\n\t"
6874 "strb zr, $mem\t# byte" %}
6875
6876 ins_encode(aarch64_enc_strb0(mem));
6877
6878 ins_pipe(istore_mem);
6879 %}
6880
6881 // Store CMS card-mark Immediate with intervening StoreStore
6882 // needed when using CMS with no conditional card marking
6883 instruct storeimmCM0_ordered(immI0 zero, memory1 mem)
6884 %{
6885 match(Set mem (StoreCM mem zero));
6886
6887 ins_cost(INSN_COST * 2);
6888 format %{ "storestore\n\t"
6889 "dmb ishst"
6890 "\n\tstrb zr, $mem\t# byte" %}
6891
6892 ins_encode(aarch64_enc_strb0_ordered(mem));
6893
6894 ins_pipe(istore_mem);
6895 %}
6896
6897 // Store Byte
6898 instruct storeB(iRegIorL2I src, memory1 mem)
6899 %{
6900 match(Set mem (StoreB mem src));
6901 predicate(!needs_releasing_store(n));
6902
6903 ins_cost(INSN_COST);
6904 format %{ "strb $src, $mem\t# byte" %}
6905
6906 ins_encode(aarch64_enc_strb(src, mem));
6907
6908 ins_pipe(istore_reg_mem);
6909 %}
6910
6911
6912 instruct storeimmB0(immI0 zero, memory1 mem)
6913 %{
6914 match(Set mem (StoreB mem zero));
6915 predicate(!needs_releasing_store(n));
6916
6917 ins_cost(INSN_COST);
6918 format %{ "strb rscractch2, $mem\t# byte" %}
6919
6920 ins_encode(aarch64_enc_strb0(mem));
6921
6922 ins_pipe(istore_mem);
6923 %}
6924
6925 // Store Char/Short
6926 instruct storeC(iRegIorL2I src, memory2 mem)
6927 %{
6928 match(Set mem (StoreC mem src));
6929 predicate(!needs_releasing_store(n));
6930
6931 ins_cost(INSN_COST);
6932 format %{ "strh $src, $mem\t# short" %}
6933
6934 ins_encode(aarch64_enc_strh(src, mem));
6935
6936 ins_pipe(istore_reg_mem);
6937 %}
6938
6939 instruct storeimmC0(immI0 zero, memory2 mem)
6940 %{
6941 match(Set mem (StoreC mem zero));
6942 predicate(!needs_releasing_store(n));
6943
6944 ins_cost(INSN_COST);
6945 format %{ "strh zr, $mem\t# short" %}
6946
6947 ins_encode(aarch64_enc_strh0(mem));
6948
6949 ins_pipe(istore_mem);
6950 %}
6951
6952 // Store Integer
6953
6954 instruct storeI(iRegIorL2I src, memory4 mem)
6955 %{
6956 match(Set mem(StoreI mem src));
6957 predicate(!needs_releasing_store(n));
6958
6959 ins_cost(INSN_COST);
6960 format %{ "strw $src, $mem\t# int" %}
6961
6962 ins_encode(aarch64_enc_strw(src, mem));
6963
6964 ins_pipe(istore_reg_mem);
6965 %}
6966
6967 instruct storeimmI0(immI0 zero, memory4 mem)
6968 %{
6969 match(Set mem(StoreI mem zero));
6970 predicate(!needs_releasing_store(n));
6971
6972 ins_cost(INSN_COST);
6973 format %{ "strw zr, $mem\t# int" %}
6974
6975 ins_encode(aarch64_enc_strw0(mem));
6976
6977 ins_pipe(istore_mem);
6978 %}
6979
6980 // Store Long (64 bit signed)
6981 instruct storeL(iRegL src, memory8 mem)
6982 %{
6983 match(Set mem (StoreL mem src));
6984 predicate(!needs_releasing_store(n));
6985
6986 ins_cost(INSN_COST);
6987 format %{ "str $src, $mem\t# int" %}
6988
6989 ins_encode(aarch64_enc_str(src, mem));
6990
6991 ins_pipe(istore_reg_mem);
6992 %}
6993
6994 // Store Long (64 bit signed)
6995 instruct storeimmL0(immL0 zero, memory8 mem)
6996 %{
6997 match(Set mem (StoreL mem zero));
6998 predicate(!needs_releasing_store(n));
6999
7000 ins_cost(INSN_COST);
7001 format %{ "str zr, $mem\t# int" %}
7002
7003 ins_encode(aarch64_enc_str0(mem));
7004
7005 ins_pipe(istore_mem);
7006 %}
7007
7008 // Store Pointer
7009 instruct storeP(iRegP src, memory8 mem)
7010 %{
7011 match(Set mem (StoreP mem src));
7012 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7013
7014 ins_cost(INSN_COST);
7015 format %{ "str $src, $mem\t# ptr" %}
7016
7017 ins_encode(aarch64_enc_str(src, mem));
7018
7019 ins_pipe(istore_reg_mem);
7020 %}
7021
7022 // Store Pointer
7023 instruct storeimmP0(immP0 zero, memory8 mem)
7024 %{
7025 match(Set mem (StoreP mem zero));
7026 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7027
7028 ins_cost(INSN_COST);
7029 format %{ "str zr, $mem\t# ptr" %}
7030
7031 ins_encode(aarch64_enc_str0(mem));
7032
7033 ins_pipe(istore_mem);
7034 %}
7035
7036 // Store Compressed Pointer
7037 instruct storeN(iRegN src, memory4 mem)
7038 %{
7039 match(Set mem (StoreN mem src));
7040 predicate(!needs_releasing_store(n));
7041
7042 ins_cost(INSN_COST);
7043 format %{ "strw $src, $mem\t# compressed ptr" %}
7044
7045 ins_encode(aarch64_enc_strw(src, mem));
7046
7047 ins_pipe(istore_reg_mem);
7048 %}
7049
7050 instruct storeImmN0(immN0 zero, memory4 mem)
7051 %{
7052 match(Set mem (StoreN mem zero));
7053 predicate(!needs_releasing_store(n));
7054
7055 ins_cost(INSN_COST);
7056 format %{ "strw zr, $mem\t# compressed ptr" %}
7057
7058 ins_encode(aarch64_enc_strw0(mem));
7059
7060 ins_pipe(istore_mem);
7061 %}
7062
7063 // Store Float
7064 instruct storeF(vRegF src, memory4 mem)
7065 %{
7066 match(Set mem (StoreF mem src));
7067 predicate(!needs_releasing_store(n));
7068
7069 ins_cost(INSN_COST);
7070 format %{ "strs $src, $mem\t# float" %}
7071
7072 ins_encode( aarch64_enc_strs(src, mem) );
7073
7074 ins_pipe(pipe_class_memory);
7075 %}
7076
7077 // TODO
7078 // implement storeImmF0 and storeFImmPacked
7079
7080 // Store Double
7081 instruct storeD(vRegD src, memory8 mem)
7082 %{
7083 match(Set mem (StoreD mem src));
7084 predicate(!needs_releasing_store(n));
7085
7086 ins_cost(INSN_COST);
7087 format %{ "strd $src, $mem\t# double" %}
7088
7089 ins_encode( aarch64_enc_strd(src, mem) );
7090
7091 ins_pipe(pipe_class_memory);
7092 %}
7093
7094 // Store Compressed Klass Pointer
7095 instruct storeNKlass(iRegN src, memory4 mem)
7096 %{
7097 predicate(!needs_releasing_store(n));
7098 match(Set mem (StoreNKlass mem src));
7099
7100 ins_cost(INSN_COST);
7101 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7102
7103 ins_encode(aarch64_enc_strw(src, mem));
7104
7105 ins_pipe(istore_reg_mem);
7106 %}
7107
7108 // TODO
7109 // implement storeImmD0 and storeDImmPacked
7110
7111 // prefetch instructions
7112 // Must be safe to execute with invalid address (cannot fault).
7113
7114 instruct prefetchalloc( memory8 mem ) %{
7115 match(PrefetchAllocation mem);
7116
7117 ins_cost(INSN_COST);
7118 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7119
7120 ins_encode( aarch64_enc_prefetchw(mem) );
7121
7122 ins_pipe(iload_prefetch);
7123 %}
7124
7125 // ---------------- volatile loads and stores ----------------
7126
7127 // Load Byte (8 bit signed)
7128 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7129 %{
7130 match(Set dst (LoadB mem));
7131
7132 ins_cost(VOLATILE_REF_COST);
7133 format %{ "ldarsb $dst, $mem\t# byte" %}
7134
7135 ins_encode(aarch64_enc_ldarsb(dst, mem));
7136
7137 ins_pipe(pipe_serial);
7138 %}
7139
7140 // Load Byte (8 bit signed) into long
7141 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7142 %{
7143 match(Set dst (ConvI2L (LoadB mem)));
7144
7145 ins_cost(VOLATILE_REF_COST);
7146 format %{ "ldarsb $dst, $mem\t# byte" %}
7147
7148 ins_encode(aarch64_enc_ldarsb(dst, mem));
7149
7150 ins_pipe(pipe_serial);
7151 %}
7152
7153 // Load Byte (8 bit unsigned)
7154 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7155 %{
7156 match(Set dst (LoadUB mem));
7157
7158 ins_cost(VOLATILE_REF_COST);
7159 format %{ "ldarb $dst, $mem\t# byte" %}
7160
7161 ins_encode(aarch64_enc_ldarb(dst, mem));
7162
7163 ins_pipe(pipe_serial);
7164 %}
7165
7166 // Load Byte (8 bit unsigned) into long
7167 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7168 %{
7169 match(Set dst (ConvI2L (LoadUB mem)));
7170
7171 ins_cost(VOLATILE_REF_COST);
7172 format %{ "ldarb $dst, $mem\t# byte" %}
7173
7174 ins_encode(aarch64_enc_ldarb(dst, mem));
7175
7176 ins_pipe(pipe_serial);
7177 %}
7178
7179 // Load Short (16 bit signed)
7180 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7181 %{
7182 match(Set dst (LoadS mem));
7183
7184 ins_cost(VOLATILE_REF_COST);
7185 format %{ "ldarshw $dst, $mem\t# short" %}
7186
7187 ins_encode(aarch64_enc_ldarshw(dst, mem));
7188
7189 ins_pipe(pipe_serial);
7190 %}
7191
7192 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7193 %{
7194 match(Set dst (LoadUS mem));
7195
7196 ins_cost(VOLATILE_REF_COST);
7197 format %{ "ldarhw $dst, $mem\t# short" %}
7198
7199 ins_encode(aarch64_enc_ldarhw(dst, mem));
7200
7201 ins_pipe(pipe_serial);
7202 %}
7203
7204 // Load Short/Char (16 bit unsigned) into long
7205 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7206 %{
7207 match(Set dst (ConvI2L (LoadUS mem)));
7208
7209 ins_cost(VOLATILE_REF_COST);
7210 format %{ "ldarh $dst, $mem\t# short" %}
7211
7212 ins_encode(aarch64_enc_ldarh(dst, mem));
7213
7214 ins_pipe(pipe_serial);
7215 %}
7216
7217 // Load Short/Char (16 bit signed) into long
7218 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7219 %{
7220 match(Set dst (ConvI2L (LoadS mem)));
7221
7222 ins_cost(VOLATILE_REF_COST);
7223 format %{ "ldarh $dst, $mem\t# short" %}
7224
7225 ins_encode(aarch64_enc_ldarsh(dst, mem));
7226
7227 ins_pipe(pipe_serial);
7228 %}
7229
7230 // Load Integer (32 bit signed)
7231 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7232 %{
7233 match(Set dst (LoadI mem));
7234
7235 ins_cost(VOLATILE_REF_COST);
7236 format %{ "ldarw $dst, $mem\t# int" %}
7237
7238 ins_encode(aarch64_enc_ldarw(dst, mem));
7239
7240 ins_pipe(pipe_serial);
7241 %}
7242
7243 // Load Integer (32 bit unsigned) into long
7244 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7245 %{
7246 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7247
7248 ins_cost(VOLATILE_REF_COST);
7249 format %{ "ldarw $dst, $mem\t# int" %}
7250
7251 ins_encode(aarch64_enc_ldarw(dst, mem));
7252
7253 ins_pipe(pipe_serial);
7254 %}
7255
7256 // Load Long (64 bit signed)
7257 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7258 %{
7259 match(Set dst (LoadL mem));
7260
7261 ins_cost(VOLATILE_REF_COST);
7262 format %{ "ldar $dst, $mem\t# int" %}
7263
7264 ins_encode(aarch64_enc_ldar(dst, mem));
7265
7266 ins_pipe(pipe_serial);
7267 %}
7268
7269 // Load Pointer
7270 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7271 %{
7272 match(Set dst (LoadP mem));
7273 predicate(n->as_Load()->barrier_data() == 0);
7274
7275 ins_cost(VOLATILE_REF_COST);
7276 format %{ "ldar $dst, $mem\t# ptr" %}
7277
7278 ins_encode(aarch64_enc_ldar(dst, mem));
7279
7280 ins_pipe(pipe_serial);
7281 %}
7282
7283 // Load Compressed Pointer
7284 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7285 %{
7286 match(Set dst (LoadN mem));
7287
7288 ins_cost(VOLATILE_REF_COST);
7289 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7290
7291 ins_encode(aarch64_enc_ldarw(dst, mem));
7292
7293 ins_pipe(pipe_serial);
7294 %}
7295
7296 // Load Float
7297 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7298 %{
7299 match(Set dst (LoadF mem));
7300
7301 ins_cost(VOLATILE_REF_COST);
7302 format %{ "ldars $dst, $mem\t# float" %}
7303
7304 ins_encode( aarch64_enc_fldars(dst, mem) );
7305
7306 ins_pipe(pipe_serial);
7307 %}
7308
7309 // Load Double
7310 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7311 %{
7312 match(Set dst (LoadD mem));
7313
7314 ins_cost(VOLATILE_REF_COST);
7315 format %{ "ldard $dst, $mem\t# double" %}
7316
7317 ins_encode( aarch64_enc_fldard(dst, mem) );
7318
7319 ins_pipe(pipe_serial);
7320 %}
7321
7322 // Store Byte
7323 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7324 %{
7325 match(Set mem (StoreB mem src));
7326
7327 ins_cost(VOLATILE_REF_COST);
7328 format %{ "stlrb $src, $mem\t# byte" %}
7329
7330 ins_encode(aarch64_enc_stlrb(src, mem));
7331
7332 ins_pipe(pipe_class_memory);
7333 %}
7334
7335 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7336 %{
7337 match(Set mem (StoreB mem zero));
7338
7339 ins_cost(VOLATILE_REF_COST);
7340 format %{ "stlrb zr, $mem\t# byte" %}
7341
7342 ins_encode(aarch64_enc_stlrb0(mem));
7343
7344 ins_pipe(pipe_class_memory);
7345 %}
7346
7347 // Store Char/Short
7348 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7349 %{
7350 match(Set mem (StoreC mem src));
7351
7352 ins_cost(VOLATILE_REF_COST);
7353 format %{ "stlrh $src, $mem\t# short" %}
7354
7355 ins_encode(aarch64_enc_stlrh(src, mem));
7356
7357 ins_pipe(pipe_class_memory);
7358 %}
7359
7360 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7361 %{
7362 match(Set mem (StoreC mem zero));
7363
7364 ins_cost(VOLATILE_REF_COST);
7365 format %{ "stlrh zr, $mem\t# short" %}
7366
7367 ins_encode(aarch64_enc_stlrh0(mem));
7368
7369 ins_pipe(pipe_class_memory);
7370 %}
7371
7372 // Store Integer
7373
7374 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7375 %{
7376 match(Set mem(StoreI mem src));
7377
7378 ins_cost(VOLATILE_REF_COST);
7379 format %{ "stlrw $src, $mem\t# int" %}
7380
7381 ins_encode(aarch64_enc_stlrw(src, mem));
7382
7383 ins_pipe(pipe_class_memory);
7384 %}
7385
7386 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7387 %{
7388 match(Set mem(StoreI mem zero));
7389
7390 ins_cost(VOLATILE_REF_COST);
7391 format %{ "stlrw zr, $mem\t# int" %}
7392
7393 ins_encode(aarch64_enc_stlrw0(mem));
7394
7395 ins_pipe(pipe_class_memory);
7396 %}
7397
7398 // Store Long (64 bit signed)
7399 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7400 %{
7401 match(Set mem (StoreL mem src));
7402
7403 ins_cost(VOLATILE_REF_COST);
7404 format %{ "stlr $src, $mem\t# int" %}
7405
7406 ins_encode(aarch64_enc_stlr(src, mem));
7407
7408 ins_pipe(pipe_class_memory);
7409 %}
7410
7411 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7412 %{
7413 match(Set mem (StoreL mem zero));
7414
7415 ins_cost(VOLATILE_REF_COST);
7416 format %{ "stlr zr, $mem\t# int" %}
7417
7418 ins_encode(aarch64_enc_stlr0(mem));
7419
7420 ins_pipe(pipe_class_memory);
7421 %}
7422
7423 // Store Pointer
7424 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7425 %{
7426 match(Set mem (StoreP mem src));
7427 predicate(n->as_Store()->barrier_data() == 0);
7428
7429 ins_cost(VOLATILE_REF_COST);
7430 format %{ "stlr $src, $mem\t# ptr" %}
7431
7432 ins_encode(aarch64_enc_stlr(src, mem));
7433
7434 ins_pipe(pipe_class_memory);
7435 %}
7436
7437 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7438 %{
7439 match(Set mem (StoreP mem zero));
7440 predicate(n->as_Store()->barrier_data() == 0);
7441
7442 ins_cost(VOLATILE_REF_COST);
7443 format %{ "stlr zr, $mem\t# ptr" %}
7444
7445 ins_encode(aarch64_enc_stlr0(mem));
7446
7447 ins_pipe(pipe_class_memory);
7448 %}
7449
7450 // Store Compressed Pointer
7451 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7452 %{
7453 match(Set mem (StoreN mem src));
7454
7455 ins_cost(VOLATILE_REF_COST);
7456 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7457
7458 ins_encode(aarch64_enc_stlrw(src, mem));
7459
7460 ins_pipe(pipe_class_memory);
7461 %}
7462
7463 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7464 %{
7465 match(Set mem (StoreN mem zero));
7466
7467 ins_cost(VOLATILE_REF_COST);
7468 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7469
7470 ins_encode(aarch64_enc_stlrw0(mem));
7471
7472 ins_pipe(pipe_class_memory);
7473 %}
7474
7475 // Store Float
7476 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7477 %{
7478 match(Set mem (StoreF mem src));
7479
7480 ins_cost(VOLATILE_REF_COST);
7481 format %{ "stlrs $src, $mem\t# float" %}
7482
7483 ins_encode( aarch64_enc_fstlrs(src, mem) );
7484
7485 ins_pipe(pipe_class_memory);
7486 %}
7487
7488 // TODO
7489 // implement storeImmF0 and storeFImmPacked
7490
7491 // Store Double
7492 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7493 %{
7494 match(Set mem (StoreD mem src));
7495
7496 ins_cost(VOLATILE_REF_COST);
7497 format %{ "stlrd $src, $mem\t# double" %}
7498
7499 ins_encode( aarch64_enc_fstlrd(src, mem) );
7500
7501 ins_pipe(pipe_class_memory);
7502 %}
7503
7504 // ---------------- end of volatile loads and stores ----------------
7505
7506 instruct cacheWB(indirect addr)
7507 %{
7508 predicate(VM_Version::supports_data_cache_line_flush());
7509 match(CacheWB addr);
7510
7511 ins_cost(100);
7512 format %{"cache wb $addr" %}
7513 ins_encode %{
7514 assert($addr->index_position() < 0, "should be");
7515 assert($addr$$disp == 0, "should be");
7516 __ cache_wb(Address($addr$$base$$Register, 0));
7517 %}
7518 ins_pipe(pipe_slow); // XXX
7519 %}
7520
7521 instruct cacheWBPreSync()
7522 %{
7523 predicate(VM_Version::supports_data_cache_line_flush());
7524 match(CacheWBPreSync);
7525
7526 ins_cost(100);
7527 format %{"cache wb presync" %}
7528 ins_encode %{
7529 __ cache_wbsync(true);
7530 %}
7531 ins_pipe(pipe_slow); // XXX
7532 %}
7533
7534 instruct cacheWBPostSync()
7535 %{
7536 predicate(VM_Version::supports_data_cache_line_flush());
7537 match(CacheWBPostSync);
7538
7539 ins_cost(100);
7540 format %{"cache wb postsync" %}
7541 ins_encode %{
7542 __ cache_wbsync(false);
7543 %}
7544 ins_pipe(pipe_slow); // XXX
7545 %}
7546
7547 // ============================================================================
7548 // BSWAP Instructions
7549
7550 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7551 match(Set dst (ReverseBytesI src));
7552
7553 ins_cost(INSN_COST);
7554 format %{ "revw $dst, $src" %}
7555
7556 ins_encode %{
7557 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7558 %}
7559
7560 ins_pipe(ialu_reg);
7561 %}
7562
7563 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7564 match(Set dst (ReverseBytesL src));
7565
7566 ins_cost(INSN_COST);
7567 format %{ "rev $dst, $src" %}
7568
7569 ins_encode %{
7570 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7571 %}
7572
7573 ins_pipe(ialu_reg);
7574 %}
7575
7576 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7577 match(Set dst (ReverseBytesUS src));
7578
7579 ins_cost(INSN_COST);
7580 format %{ "rev16w $dst, $src" %}
7581
7582 ins_encode %{
7583 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7584 %}
7585
7586 ins_pipe(ialu_reg);
7587 %}
7588
7589 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7590 match(Set dst (ReverseBytesS src));
7591
7592 ins_cost(INSN_COST);
7593 format %{ "rev16w $dst, $src\n\t"
7594 "sbfmw $dst, $dst, #0, #15" %}
7595
7596 ins_encode %{
7597 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7598 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7599 %}
7600
7601 ins_pipe(ialu_reg);
7602 %}
7603
7604 // ============================================================================
7605 // Zero Count Instructions
7606
7607 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7608 match(Set dst (CountLeadingZerosI src));
7609
7610 ins_cost(INSN_COST);
7611 format %{ "clzw $dst, $src" %}
7612 ins_encode %{
7613 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7614 %}
7615
7616 ins_pipe(ialu_reg);
7617 %}
7618
7619 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7620 match(Set dst (CountLeadingZerosL src));
7621
7622 ins_cost(INSN_COST);
7623 format %{ "clz $dst, $src" %}
7624 ins_encode %{
7625 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7626 %}
7627
7628 ins_pipe(ialu_reg);
7629 %}
7630
7631 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7632 match(Set dst (CountTrailingZerosI src));
7633
7634 ins_cost(INSN_COST * 2);
7635 format %{ "rbitw $dst, $src\n\t"
7636 "clzw $dst, $dst" %}
7637 ins_encode %{
7638 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7639 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7640 %}
7641
7642 ins_pipe(ialu_reg);
7643 %}
7644
7645 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7646 match(Set dst (CountTrailingZerosL src));
7647
7648 ins_cost(INSN_COST * 2);
7649 format %{ "rbit $dst, $src\n\t"
7650 "clz $dst, $dst" %}
7651 ins_encode %{
7652 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7653 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7654 %}
7655
7656 ins_pipe(ialu_reg);
7657 %}
7658
7659 //---------- Population Count Instructions -------------------------------------
7660 //
7661
7662 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7663 match(Set dst (PopCountI src));
7664 effect(TEMP tmp);
7665 ins_cost(INSN_COST * 13);
7666
7667 format %{ "movw $src, $src\n\t"
7668 "mov $tmp, $src\t# vector (1D)\n\t"
7669 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7670 "addv $tmp, $tmp\t# vector (8B)\n\t"
7671 "mov $dst, $tmp\t# vector (1D)" %}
7672 ins_encode %{
7673 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7674 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7675 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7676 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7677 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7678 %}
7679
7680 ins_pipe(pipe_class_default);
7681 %}
7682
7683 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7684 match(Set dst (PopCountI (LoadI mem)));
7685 effect(TEMP tmp);
7686 ins_cost(INSN_COST * 13);
7687
7688 format %{ "ldrs $tmp, $mem\n\t"
7689 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7690 "addv $tmp, $tmp\t# vector (8B)\n\t"
7691 "mov $dst, $tmp\t# vector (1D)" %}
7692 ins_encode %{
7693 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7694 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7695 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7696 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7697 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7698 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7699 %}
7700
7701 ins_pipe(pipe_class_default);
7702 %}
7703
7704 // Note: Long.bitCount(long) returns an int.
7705 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7706 match(Set dst (PopCountL src));
7707 effect(TEMP tmp);
7708 ins_cost(INSN_COST * 13);
7709
7710 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7711 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7712 "addv $tmp, $tmp\t# vector (8B)\n\t"
7713 "mov $dst, $tmp\t# vector (1D)" %}
7714 ins_encode %{
7715 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7716 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7717 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7718 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7719 %}
7720
7721 ins_pipe(pipe_class_default);
7722 %}
7723
7724 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7725 match(Set dst (PopCountL (LoadL mem)));
7726 effect(TEMP tmp);
7727 ins_cost(INSN_COST * 13);
7728
7729 format %{ "ldrd $tmp, $mem\n\t"
7730 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7731 "addv $tmp, $tmp\t# vector (8B)\n\t"
7732 "mov $dst, $tmp\t# vector (1D)" %}
7733 ins_encode %{
7734 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7735 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7736 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7737 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7738 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7739 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7740 %}
7741
7742 ins_pipe(pipe_class_default);
7743 %}
7744
7745 // ============================================================================
7746 // VerifyVectorAlignment Instruction
7747
7748 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7749 match(Set addr (VerifyVectorAlignment addr mask));
7750 effect(KILL cr);
7751 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7752 ins_encode %{
7753 Label Lskip;
7754 // check if masked bits of addr are zero
7755 __ tst($addr$$Register, $mask$$constant);
7756 __ br(Assembler::EQ, Lskip);
7757 __ stop("verify_vector_alignment found a misaligned vector memory access");
7758 __ bind(Lskip);
7759 %}
7760 ins_pipe(pipe_slow);
7761 %}
7762
7763 // ============================================================================
7764 // MemBar Instruction
7765
7766 instruct load_fence() %{
7767 match(LoadFence);
7768 ins_cost(VOLATILE_REF_COST);
7769
7770 format %{ "load_fence" %}
7771
7772 ins_encode %{
7773 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7774 %}
7775 ins_pipe(pipe_serial);
7776 %}
7777
7778 instruct unnecessary_membar_acquire() %{
7779 predicate(unnecessary_acquire(n));
7780 match(MemBarAcquire);
7781 ins_cost(0);
7782
7783 format %{ "membar_acquire (elided)" %}
7784
7785 ins_encode %{
7786 __ block_comment("membar_acquire (elided)");
7787 %}
7788
7789 ins_pipe(pipe_class_empty);
7790 %}
7791
7792 instruct membar_acquire() %{
7793 match(MemBarAcquire);
7794 ins_cost(VOLATILE_REF_COST);
7795
7796 format %{ "membar_acquire\n\t"
7797 "dmb ishld" %}
7798
7799 ins_encode %{
7800 __ block_comment("membar_acquire");
7801 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7802 %}
7803
7804 ins_pipe(pipe_serial);
7805 %}
7806
7807
7808 instruct membar_acquire_lock() %{
7809 match(MemBarAcquireLock);
7810 ins_cost(VOLATILE_REF_COST);
7811
7812 format %{ "membar_acquire_lock (elided)" %}
7813
7814 ins_encode %{
7815 __ block_comment("membar_acquire_lock (elided)");
7816 %}
7817
7818 ins_pipe(pipe_serial);
7819 %}
7820
7821 instruct store_fence() %{
7822 match(StoreFence);
7823 ins_cost(VOLATILE_REF_COST);
7824
7825 format %{ "store_fence" %}
7826
7827 ins_encode %{
7828 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7829 %}
7830 ins_pipe(pipe_serial);
7831 %}
7832
7833 instruct unnecessary_membar_release() %{
7834 predicate(unnecessary_release(n));
7835 match(MemBarRelease);
7836 ins_cost(0);
7837
7838 format %{ "membar_release (elided)" %}
7839
7840 ins_encode %{
7841 __ block_comment("membar_release (elided)");
7842 %}
7843 ins_pipe(pipe_serial);
7844 %}
7845
7846 instruct membar_release() %{
7847 match(MemBarRelease);
7848 ins_cost(VOLATILE_REF_COST);
7849
7850 format %{ "membar_release\n\t"
7851 "dmb ishst\n\tdmb ishld" %}
7852
7853 ins_encode %{
7854 __ block_comment("membar_release");
7855 // These will be merged if AlwaysMergeDMB is enabled.
7856 __ membar(Assembler::StoreStore);
7857 __ membar(Assembler::LoadStore);
7858 %}
7859 ins_pipe(pipe_serial);
7860 %}
7861
7862 instruct membar_storestore() %{
7863 match(MemBarStoreStore);
7864 match(StoreStoreFence);
7865 ins_cost(VOLATILE_REF_COST);
7866
7867 format %{ "MEMBAR-store-store" %}
7868
7869 ins_encode %{
7870 __ membar(Assembler::StoreStore);
7871 %}
7872 ins_pipe(pipe_serial);
7873 %}
7874
7875 instruct membar_release_lock() %{
7876 match(MemBarReleaseLock);
7877 ins_cost(VOLATILE_REF_COST);
7878
7879 format %{ "membar_release_lock (elided)" %}
7880
7881 ins_encode %{
7882 __ block_comment("membar_release_lock (elided)");
7883 %}
7884
7885 ins_pipe(pipe_serial);
7886 %}
7887
7888 instruct unnecessary_membar_volatile() %{
7889 predicate(unnecessary_volatile(n));
7890 match(MemBarVolatile);
7891 ins_cost(0);
7892
7893 format %{ "membar_volatile (elided)" %}
7894
7895 ins_encode %{
7896 __ block_comment("membar_volatile (elided)");
7897 %}
7898
7899 ins_pipe(pipe_serial);
7900 %}
7901
7902 instruct membar_volatile() %{
7903 match(MemBarVolatile);
7904 ins_cost(VOLATILE_REF_COST*100);
7905
7906 format %{ "membar_volatile\n\t"
7907 "dmb ish"%}
7908
7909 ins_encode %{
7910 __ block_comment("membar_volatile");
7911 __ membar(Assembler::StoreLoad);
7912 %}
7913
7914 ins_pipe(pipe_serial);
7915 %}
7916
7917 // ============================================================================
7918 // Cast/Convert Instructions
7919
7920 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7921 match(Set dst (CastX2P src));
7922
7923 ins_cost(INSN_COST);
7924 format %{ "mov $dst, $src\t# long -> ptr" %}
7925
7926 ins_encode %{
7927 if ($dst$$reg != $src$$reg) {
7928 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7929 }
7930 %}
7931
7932 ins_pipe(ialu_reg);
7933 %}
7934
7935 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7936 match(Set dst (CastP2X src));
7937
7938 ins_cost(INSN_COST);
7939 format %{ "mov $dst, $src\t# ptr -> long" %}
7940
7941 ins_encode %{
7942 if ($dst$$reg != $src$$reg) {
7943 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7944 }
7945 %}
7946
7947 ins_pipe(ialu_reg);
7948 %}
7949
7950 // Convert oop into int for vectors alignment masking
7951 instruct convP2I(iRegINoSp dst, iRegP src) %{
7952 match(Set dst (ConvL2I (CastP2X src)));
7953
7954 ins_cost(INSN_COST);
7955 format %{ "movw $dst, $src\t# ptr -> int" %}
7956 ins_encode %{
7957 __ movw($dst$$Register, $src$$Register);
7958 %}
7959
7960 ins_pipe(ialu_reg);
7961 %}
7962
7963 // Convert compressed oop into int for vectors alignment masking
7964 // in case of 32bit oops (heap < 4Gb).
7965 instruct convN2I(iRegINoSp dst, iRegN src)
7966 %{
7967 predicate(CompressedOops::shift() == 0);
7968 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
7969
7970 ins_cost(INSN_COST);
7971 format %{ "mov dst, $src\t# compressed ptr -> int" %}
7972 ins_encode %{
7973 __ movw($dst$$Register, $src$$Register);
7974 %}
7975
7976 ins_pipe(ialu_reg);
7977 %}
7978
7979
7980 // Convert oop pointer into compressed form
7981 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7982 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
7983 match(Set dst (EncodeP src));
7984 effect(KILL cr);
7985 ins_cost(INSN_COST * 3);
7986 format %{ "encode_heap_oop $dst, $src" %}
7987 ins_encode %{
7988 Register s = $src$$Register;
7989 Register d = $dst$$Register;
7990 __ encode_heap_oop(d, s);
7991 %}
7992 ins_pipe(ialu_reg);
7993 %}
7994
7995 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7996 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
7997 match(Set dst (EncodeP src));
7998 ins_cost(INSN_COST * 3);
7999 format %{ "encode_heap_oop_not_null $dst, $src" %}
8000 ins_encode %{
8001 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
8002 %}
8003 ins_pipe(ialu_reg);
8004 %}
8005
8006 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8007 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8008 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8009 match(Set dst (DecodeN src));
8010 ins_cost(INSN_COST * 3);
8011 format %{ "decode_heap_oop $dst, $src" %}
8012 ins_encode %{
8013 Register s = $src$$Register;
8014 Register d = $dst$$Register;
8015 __ decode_heap_oop(d, s);
8016 %}
8017 ins_pipe(ialu_reg);
8018 %}
8019
8020 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8021 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8022 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8023 match(Set dst (DecodeN src));
8024 ins_cost(INSN_COST * 3);
8025 format %{ "decode_heap_oop_not_null $dst, $src" %}
8026 ins_encode %{
8027 Register s = $src$$Register;
8028 Register d = $dst$$Register;
8029 __ decode_heap_oop_not_null(d, s);
8030 %}
8031 ins_pipe(ialu_reg);
8032 %}
8033
8034 // n.b. AArch64 implementations of encode_klass_not_null and
8035 // decode_klass_not_null do not modify the flags register so, unlike
8036 // Intel, we don't kill CR as a side effect here
8037
8038 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8039 match(Set dst (EncodePKlass src));
8040
8041 ins_cost(INSN_COST * 3);
8042 format %{ "encode_klass_not_null $dst,$src" %}
8043
8044 ins_encode %{
8045 Register src_reg = as_Register($src$$reg);
8046 Register dst_reg = as_Register($dst$$reg);
8047 __ encode_klass_not_null(dst_reg, src_reg);
8048 %}
8049
8050 ins_pipe(ialu_reg);
8051 %}
8052
8053 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8054 match(Set dst (DecodeNKlass src));
8055
8056 ins_cost(INSN_COST * 3);
8057 format %{ "decode_klass_not_null $dst,$src" %}
8058
8059 ins_encode %{
8060 Register src_reg = as_Register($src$$reg);
8061 Register dst_reg = as_Register($dst$$reg);
8062 if (dst_reg != src_reg) {
8063 __ decode_klass_not_null(dst_reg, src_reg);
8064 } else {
8065 __ decode_klass_not_null(dst_reg);
8066 }
8067 %}
8068
8069 ins_pipe(ialu_reg);
8070 %}
8071
8072 instruct checkCastPP(iRegPNoSp dst)
8073 %{
8074 match(Set dst (CheckCastPP dst));
8075
8076 size(0);
8077 format %{ "# checkcastPP of $dst" %}
8078 ins_encode(/* empty encoding */);
8079 ins_pipe(pipe_class_empty);
8080 %}
8081
8082 instruct castPP(iRegPNoSp dst)
8083 %{
8084 match(Set dst (CastPP dst));
8085
8086 size(0);
8087 format %{ "# castPP of $dst" %}
8088 ins_encode(/* empty encoding */);
8089 ins_pipe(pipe_class_empty);
8090 %}
8091
8092 instruct castII(iRegI dst)
8093 %{
8094 match(Set dst (CastII dst));
8095
8096 size(0);
8097 format %{ "# castII of $dst" %}
8098 ins_encode(/* empty encoding */);
8099 ins_cost(0);
8100 ins_pipe(pipe_class_empty);
8101 %}
8102
8103 instruct castLL(iRegL dst)
8104 %{
8105 match(Set dst (CastLL dst));
8106
8107 size(0);
8108 format %{ "# castLL of $dst" %}
8109 ins_encode(/* empty encoding */);
8110 ins_cost(0);
8111 ins_pipe(pipe_class_empty);
8112 %}
8113
8114 instruct castFF(vRegF dst)
8115 %{
8116 match(Set dst (CastFF dst));
8117
8118 size(0);
8119 format %{ "# castFF of $dst" %}
8120 ins_encode(/* empty encoding */);
8121 ins_cost(0);
8122 ins_pipe(pipe_class_empty);
8123 %}
8124
8125 instruct castDD(vRegD dst)
8126 %{
8127 match(Set dst (CastDD dst));
8128
8129 size(0);
8130 format %{ "# castDD of $dst" %}
8131 ins_encode(/* empty encoding */);
8132 ins_cost(0);
8133 ins_pipe(pipe_class_empty);
8134 %}
8135
8136 instruct castVV(vReg dst)
8137 %{
8138 match(Set dst (CastVV dst));
8139
8140 size(0);
8141 format %{ "# castVV of $dst" %}
8142 ins_encode(/* empty encoding */);
8143 ins_cost(0);
8144 ins_pipe(pipe_class_empty);
8145 %}
8146
8147 instruct castVVMask(pRegGov dst)
8148 %{
8149 match(Set dst (CastVV dst));
8150
8151 size(0);
8152 format %{ "# castVV of $dst" %}
8153 ins_encode(/* empty encoding */);
8154 ins_cost(0);
8155 ins_pipe(pipe_class_empty);
8156 %}
8157
8158 // ============================================================================
8159 // Atomic operation instructions
8160 //
8161
8162 // standard CompareAndSwapX when we are using barriers
8163 // these have higher priority than the rules selected by a predicate
8164
8165 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8166 // can't match them
8167
8168 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8169
8170 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8171 ins_cost(2 * VOLATILE_REF_COST);
8172
8173 effect(KILL cr);
8174
8175 format %{
8176 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8177 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8178 %}
8179
8180 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8181 aarch64_enc_cset_eq(res));
8182
8183 ins_pipe(pipe_slow);
8184 %}
8185
8186 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8187
8188 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8189 ins_cost(2 * VOLATILE_REF_COST);
8190
8191 effect(KILL cr);
8192
8193 format %{
8194 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8195 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8196 %}
8197
8198 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8199 aarch64_enc_cset_eq(res));
8200
8201 ins_pipe(pipe_slow);
8202 %}
8203
8204 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8205
8206 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8207 ins_cost(2 * VOLATILE_REF_COST);
8208
8209 effect(KILL cr);
8210
8211 format %{
8212 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8213 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8214 %}
8215
8216 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8217 aarch64_enc_cset_eq(res));
8218
8219 ins_pipe(pipe_slow);
8220 %}
8221
8222 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8223
8224 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8225 ins_cost(2 * VOLATILE_REF_COST);
8226
8227 effect(KILL cr);
8228
8229 format %{
8230 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8231 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8232 %}
8233
8234 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8235 aarch64_enc_cset_eq(res));
8236
8237 ins_pipe(pipe_slow);
8238 %}
8239
8240 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8241
8242 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8243 predicate(n->as_LoadStore()->barrier_data() == 0);
8244 ins_cost(2 * VOLATILE_REF_COST);
8245
8246 effect(KILL cr);
8247
8248 format %{
8249 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8250 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8251 %}
8252
8253 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8254 aarch64_enc_cset_eq(res));
8255
8256 ins_pipe(pipe_slow);
8257 %}
8258
8259 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8260
8261 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8262 ins_cost(2 * VOLATILE_REF_COST);
8263
8264 effect(KILL cr);
8265
8266 format %{
8267 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8268 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8269 %}
8270
8271 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8272 aarch64_enc_cset_eq(res));
8273
8274 ins_pipe(pipe_slow);
8275 %}
8276
8277 // alternative CompareAndSwapX when we are eliding barriers
8278
8279 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8280
8281 predicate(needs_acquiring_load_exclusive(n));
8282 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8283 ins_cost(VOLATILE_REF_COST);
8284
8285 effect(KILL cr);
8286
8287 format %{
8288 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8289 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8290 %}
8291
8292 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8293 aarch64_enc_cset_eq(res));
8294
8295 ins_pipe(pipe_slow);
8296 %}
8297
8298 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8299
8300 predicate(needs_acquiring_load_exclusive(n));
8301 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8302 ins_cost(VOLATILE_REF_COST);
8303
8304 effect(KILL cr);
8305
8306 format %{
8307 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8308 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8309 %}
8310
8311 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8312 aarch64_enc_cset_eq(res));
8313
8314 ins_pipe(pipe_slow);
8315 %}
8316
8317 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8318
8319 predicate(needs_acquiring_load_exclusive(n));
8320 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8321 ins_cost(VOLATILE_REF_COST);
8322
8323 effect(KILL cr);
8324
8325 format %{
8326 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8327 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8328 %}
8329
8330 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8331 aarch64_enc_cset_eq(res));
8332
8333 ins_pipe(pipe_slow);
8334 %}
8335
8336 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8337
8338 predicate(needs_acquiring_load_exclusive(n));
8339 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8340 ins_cost(VOLATILE_REF_COST);
8341
8342 effect(KILL cr);
8343
8344 format %{
8345 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8346 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8347 %}
8348
8349 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8350 aarch64_enc_cset_eq(res));
8351
8352 ins_pipe(pipe_slow);
8353 %}
8354
8355 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8356
8357 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8358 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8359 ins_cost(VOLATILE_REF_COST);
8360
8361 effect(KILL cr);
8362
8363 format %{
8364 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8365 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8366 %}
8367
8368 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8369 aarch64_enc_cset_eq(res));
8370
8371 ins_pipe(pipe_slow);
8372 %}
8373
8374 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8375
8376 predicate(needs_acquiring_load_exclusive(n));
8377 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8378 ins_cost(VOLATILE_REF_COST);
8379
8380 effect(KILL cr);
8381
8382 format %{
8383 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8384 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8385 %}
8386
8387 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8388 aarch64_enc_cset_eq(res));
8389
8390 ins_pipe(pipe_slow);
8391 %}
8392
8393
8394 // ---------------------------------------------------------------------
8395
8396 // BEGIN This section of the file is automatically generated. Do not edit --------------
8397
8398 // Sundry CAS operations. Note that release is always true,
8399 // regardless of the memory ordering of the CAS. This is because we
8400 // need the volatile case to be sequentially consistent but there is
8401 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8402 // can't check the type of memory ordering here, so we always emit a
8403 // STLXR.
8404
8405 // This section is generated from cas.m4
8406
8407
8408 // This pattern is generated automatically from cas.m4.
8409 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8410 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8411 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8412 ins_cost(2 * VOLATILE_REF_COST);
8413 effect(TEMP_DEF res, KILL cr);
8414 format %{
8415 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8416 %}
8417 ins_encode %{
8418 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8419 Assembler::byte, /*acquire*/ false, /*release*/ true,
8420 /*weak*/ false, $res$$Register);
8421 __ sxtbw($res$$Register, $res$$Register);
8422 %}
8423 ins_pipe(pipe_slow);
8424 %}
8425
8426 // This pattern is generated automatically from cas.m4.
8427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8428 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8429 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8430 ins_cost(2 * VOLATILE_REF_COST);
8431 effect(TEMP_DEF res, KILL cr);
8432 format %{
8433 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8434 %}
8435 ins_encode %{
8436 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8437 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8438 /*weak*/ false, $res$$Register);
8439 __ sxthw($res$$Register, $res$$Register);
8440 %}
8441 ins_pipe(pipe_slow);
8442 %}
8443
8444 // This pattern is generated automatically from cas.m4.
8445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8446 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8447 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8448 ins_cost(2 * VOLATILE_REF_COST);
8449 effect(TEMP_DEF res, KILL cr);
8450 format %{
8451 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8452 %}
8453 ins_encode %{
8454 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8455 Assembler::word, /*acquire*/ false, /*release*/ true,
8456 /*weak*/ false, $res$$Register);
8457 %}
8458 ins_pipe(pipe_slow);
8459 %}
8460
8461 // This pattern is generated automatically from cas.m4.
8462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8463 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8464 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8465 ins_cost(2 * VOLATILE_REF_COST);
8466 effect(TEMP_DEF res, KILL cr);
8467 format %{
8468 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8469 %}
8470 ins_encode %{
8471 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8472 Assembler::xword, /*acquire*/ false, /*release*/ true,
8473 /*weak*/ false, $res$$Register);
8474 %}
8475 ins_pipe(pipe_slow);
8476 %}
8477
8478 // This pattern is generated automatically from cas.m4.
8479 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8480 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8481 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8482 ins_cost(2 * VOLATILE_REF_COST);
8483 effect(TEMP_DEF res, KILL cr);
8484 format %{
8485 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8486 %}
8487 ins_encode %{
8488 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8489 Assembler::word, /*acquire*/ false, /*release*/ true,
8490 /*weak*/ false, $res$$Register);
8491 %}
8492 ins_pipe(pipe_slow);
8493 %}
8494
8495 // This pattern is generated automatically from cas.m4.
8496 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8497 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8498 predicate(n->as_LoadStore()->barrier_data() == 0);
8499 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8500 ins_cost(2 * VOLATILE_REF_COST);
8501 effect(TEMP_DEF res, KILL cr);
8502 format %{
8503 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8504 %}
8505 ins_encode %{
8506 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8507 Assembler::xword, /*acquire*/ false, /*release*/ true,
8508 /*weak*/ false, $res$$Register);
8509 %}
8510 ins_pipe(pipe_slow);
8511 %}
8512
8513 // This pattern is generated automatically from cas.m4.
8514 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8515 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8516 predicate(needs_acquiring_load_exclusive(n));
8517 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8518 ins_cost(VOLATILE_REF_COST);
8519 effect(TEMP_DEF res, KILL cr);
8520 format %{
8521 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8522 %}
8523 ins_encode %{
8524 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8525 Assembler::byte, /*acquire*/ true, /*release*/ true,
8526 /*weak*/ false, $res$$Register);
8527 __ sxtbw($res$$Register, $res$$Register);
8528 %}
8529 ins_pipe(pipe_slow);
8530 %}
8531
8532 // This pattern is generated automatically from cas.m4.
8533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8534 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8535 predicate(needs_acquiring_load_exclusive(n));
8536 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8537 ins_cost(VOLATILE_REF_COST);
8538 effect(TEMP_DEF res, KILL cr);
8539 format %{
8540 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8541 %}
8542 ins_encode %{
8543 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8544 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8545 /*weak*/ false, $res$$Register);
8546 __ sxthw($res$$Register, $res$$Register);
8547 %}
8548 ins_pipe(pipe_slow);
8549 %}
8550
8551 // This pattern is generated automatically from cas.m4.
8552 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8553 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8554 predicate(needs_acquiring_load_exclusive(n));
8555 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8556 ins_cost(VOLATILE_REF_COST);
8557 effect(TEMP_DEF res, KILL cr);
8558 format %{
8559 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8560 %}
8561 ins_encode %{
8562 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8563 Assembler::word, /*acquire*/ true, /*release*/ true,
8564 /*weak*/ false, $res$$Register);
8565 %}
8566 ins_pipe(pipe_slow);
8567 %}
8568
8569 // This pattern is generated automatically from cas.m4.
8570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8571 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8572 predicate(needs_acquiring_load_exclusive(n));
8573 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8574 ins_cost(VOLATILE_REF_COST);
8575 effect(TEMP_DEF res, KILL cr);
8576 format %{
8577 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8578 %}
8579 ins_encode %{
8580 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8581 Assembler::xword, /*acquire*/ true, /*release*/ true,
8582 /*weak*/ false, $res$$Register);
8583 %}
8584 ins_pipe(pipe_slow);
8585 %}
8586
8587 // This pattern is generated automatically from cas.m4.
8588 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8589 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8590 predicate(needs_acquiring_load_exclusive(n));
8591 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8592 ins_cost(VOLATILE_REF_COST);
8593 effect(TEMP_DEF res, KILL cr);
8594 format %{
8595 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8596 %}
8597 ins_encode %{
8598 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8599 Assembler::word, /*acquire*/ true, /*release*/ true,
8600 /*weak*/ false, $res$$Register);
8601 %}
8602 ins_pipe(pipe_slow);
8603 %}
8604
8605 // This pattern is generated automatically from cas.m4.
8606 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8607 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8608 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8609 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8610 ins_cost(VOLATILE_REF_COST);
8611 effect(TEMP_DEF res, KILL cr);
8612 format %{
8613 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8614 %}
8615 ins_encode %{
8616 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8617 Assembler::xword, /*acquire*/ true, /*release*/ true,
8618 /*weak*/ false, $res$$Register);
8619 %}
8620 ins_pipe(pipe_slow);
8621 %}
8622
8623 // This pattern is generated automatically from cas.m4.
8624 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8625 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8626 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8627 ins_cost(2 * VOLATILE_REF_COST);
8628 effect(KILL cr);
8629 format %{
8630 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8631 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8632 %}
8633 ins_encode %{
8634 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8635 Assembler::byte, /*acquire*/ false, /*release*/ true,
8636 /*weak*/ true, noreg);
8637 __ csetw($res$$Register, Assembler::EQ);
8638 %}
8639 ins_pipe(pipe_slow);
8640 %}
8641
8642 // This pattern is generated automatically from cas.m4.
8643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8644 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8645 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8646 ins_cost(2 * VOLATILE_REF_COST);
8647 effect(KILL cr);
8648 format %{
8649 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8650 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8651 %}
8652 ins_encode %{
8653 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8654 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8655 /*weak*/ true, noreg);
8656 __ csetw($res$$Register, Assembler::EQ);
8657 %}
8658 ins_pipe(pipe_slow);
8659 %}
8660
8661 // This pattern is generated automatically from cas.m4.
8662 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8663 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8664 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8665 ins_cost(2 * VOLATILE_REF_COST);
8666 effect(KILL cr);
8667 format %{
8668 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8669 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8670 %}
8671 ins_encode %{
8672 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8673 Assembler::word, /*acquire*/ false, /*release*/ true,
8674 /*weak*/ true, noreg);
8675 __ csetw($res$$Register, Assembler::EQ);
8676 %}
8677 ins_pipe(pipe_slow);
8678 %}
8679
8680 // This pattern is generated automatically from cas.m4.
8681 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8682 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8683 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8684 ins_cost(2 * VOLATILE_REF_COST);
8685 effect(KILL cr);
8686 format %{
8687 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8688 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8689 %}
8690 ins_encode %{
8691 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8692 Assembler::xword, /*acquire*/ false, /*release*/ true,
8693 /*weak*/ true, noreg);
8694 __ csetw($res$$Register, Assembler::EQ);
8695 %}
8696 ins_pipe(pipe_slow);
8697 %}
8698
8699 // This pattern is generated automatically from cas.m4.
8700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8701 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8702 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8703 ins_cost(2 * VOLATILE_REF_COST);
8704 effect(KILL cr);
8705 format %{
8706 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8707 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8708 %}
8709 ins_encode %{
8710 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8711 Assembler::word, /*acquire*/ false, /*release*/ true,
8712 /*weak*/ true, noreg);
8713 __ csetw($res$$Register, Assembler::EQ);
8714 %}
8715 ins_pipe(pipe_slow);
8716 %}
8717
8718 // This pattern is generated automatically from cas.m4.
8719 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8720 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8721 predicate(n->as_LoadStore()->barrier_data() == 0);
8722 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8723 ins_cost(2 * VOLATILE_REF_COST);
8724 effect(KILL cr);
8725 format %{
8726 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8727 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8728 %}
8729 ins_encode %{
8730 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8731 Assembler::xword, /*acquire*/ false, /*release*/ true,
8732 /*weak*/ true, noreg);
8733 __ csetw($res$$Register, Assembler::EQ);
8734 %}
8735 ins_pipe(pipe_slow);
8736 %}
8737
8738 // This pattern is generated automatically from cas.m4.
8739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8740 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8741 predicate(needs_acquiring_load_exclusive(n));
8742 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8743 ins_cost(VOLATILE_REF_COST);
8744 effect(KILL cr);
8745 format %{
8746 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8747 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8748 %}
8749 ins_encode %{
8750 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8751 Assembler::byte, /*acquire*/ true, /*release*/ true,
8752 /*weak*/ true, noreg);
8753 __ csetw($res$$Register, Assembler::EQ);
8754 %}
8755 ins_pipe(pipe_slow);
8756 %}
8757
8758 // This pattern is generated automatically from cas.m4.
8759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8760 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8761 predicate(needs_acquiring_load_exclusive(n));
8762 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8763 ins_cost(VOLATILE_REF_COST);
8764 effect(KILL cr);
8765 format %{
8766 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8767 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8768 %}
8769 ins_encode %{
8770 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8771 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8772 /*weak*/ true, noreg);
8773 __ csetw($res$$Register, Assembler::EQ);
8774 %}
8775 ins_pipe(pipe_slow);
8776 %}
8777
8778 // This pattern is generated automatically from cas.m4.
8779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8780 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8781 predicate(needs_acquiring_load_exclusive(n));
8782 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8783 ins_cost(VOLATILE_REF_COST);
8784 effect(KILL cr);
8785 format %{
8786 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8787 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8788 %}
8789 ins_encode %{
8790 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8791 Assembler::word, /*acquire*/ true, /*release*/ true,
8792 /*weak*/ true, noreg);
8793 __ csetw($res$$Register, Assembler::EQ);
8794 %}
8795 ins_pipe(pipe_slow);
8796 %}
8797
8798 // This pattern is generated automatically from cas.m4.
8799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8800 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8801 predicate(needs_acquiring_load_exclusive(n));
8802 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8803 ins_cost(VOLATILE_REF_COST);
8804 effect(KILL cr);
8805 format %{
8806 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8807 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8808 %}
8809 ins_encode %{
8810 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8811 Assembler::xword, /*acquire*/ true, /*release*/ true,
8812 /*weak*/ true, noreg);
8813 __ csetw($res$$Register, Assembler::EQ);
8814 %}
8815 ins_pipe(pipe_slow);
8816 %}
8817
8818 // This pattern is generated automatically from cas.m4.
8819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8820 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8821 predicate(needs_acquiring_load_exclusive(n));
8822 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8823 ins_cost(VOLATILE_REF_COST);
8824 effect(KILL cr);
8825 format %{
8826 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8827 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8828 %}
8829 ins_encode %{
8830 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8831 Assembler::word, /*acquire*/ true, /*release*/ true,
8832 /*weak*/ true, noreg);
8833 __ csetw($res$$Register, Assembler::EQ);
8834 %}
8835 ins_pipe(pipe_slow);
8836 %}
8837
8838 // This pattern is generated automatically from cas.m4.
8839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8840 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8841 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8842 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8843 ins_cost(VOLATILE_REF_COST);
8844 effect(KILL cr);
8845 format %{
8846 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8847 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8848 %}
8849 ins_encode %{
8850 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8851 Assembler::xword, /*acquire*/ true, /*release*/ true,
8852 /*weak*/ true, noreg);
8853 __ csetw($res$$Register, Assembler::EQ);
8854 %}
8855 ins_pipe(pipe_slow);
8856 %}
8857
8858 // END This section of the file is automatically generated. Do not edit --------------
8859 // ---------------------------------------------------------------------
8860
8861 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8862 match(Set prev (GetAndSetI mem newv));
8863 ins_cost(2 * VOLATILE_REF_COST);
8864 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8865 ins_encode %{
8866 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8867 %}
8868 ins_pipe(pipe_serial);
8869 %}
8870
8871 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8872 match(Set prev (GetAndSetL mem newv));
8873 ins_cost(2 * VOLATILE_REF_COST);
8874 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8875 ins_encode %{
8876 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8877 %}
8878 ins_pipe(pipe_serial);
8879 %}
8880
8881 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8882 match(Set prev (GetAndSetN mem newv));
8883 ins_cost(2 * VOLATILE_REF_COST);
8884 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8885 ins_encode %{
8886 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8887 %}
8888 ins_pipe(pipe_serial);
8889 %}
8890
8891 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8892 predicate(n->as_LoadStore()->barrier_data() == 0);
8893 match(Set prev (GetAndSetP mem newv));
8894 ins_cost(2 * VOLATILE_REF_COST);
8895 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8896 ins_encode %{
8897 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8898 %}
8899 ins_pipe(pipe_serial);
8900 %}
8901
8902 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8903 predicate(needs_acquiring_load_exclusive(n));
8904 match(Set prev (GetAndSetI mem newv));
8905 ins_cost(VOLATILE_REF_COST);
8906 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8907 ins_encode %{
8908 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8909 %}
8910 ins_pipe(pipe_serial);
8911 %}
8912
8913 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8914 predicate(needs_acquiring_load_exclusive(n));
8915 match(Set prev (GetAndSetL mem newv));
8916 ins_cost(VOLATILE_REF_COST);
8917 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8918 ins_encode %{
8919 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8920 %}
8921 ins_pipe(pipe_serial);
8922 %}
8923
8924 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
8925 predicate(needs_acquiring_load_exclusive(n));
8926 match(Set prev (GetAndSetN mem newv));
8927 ins_cost(VOLATILE_REF_COST);
8928 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8929 ins_encode %{
8930 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8931 %}
8932 ins_pipe(pipe_serial);
8933 %}
8934
8935 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
8936 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8937 match(Set prev (GetAndSetP mem newv));
8938 ins_cost(VOLATILE_REF_COST);
8939 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8940 ins_encode %{
8941 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8942 %}
8943 ins_pipe(pipe_serial);
8944 %}
8945
8946
8947 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
8948 match(Set newval (GetAndAddL mem incr));
8949 ins_cost(2 * VOLATILE_REF_COST + 1);
8950 format %{ "get_and_addL $newval, [$mem], $incr" %}
8951 ins_encode %{
8952 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
8953 %}
8954 ins_pipe(pipe_serial);
8955 %}
8956
8957 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
8958 predicate(n->as_LoadStore()->result_not_used());
8959 match(Set dummy (GetAndAddL mem incr));
8960 ins_cost(2 * VOLATILE_REF_COST);
8961 format %{ "get_and_addL [$mem], $incr" %}
8962 ins_encode %{
8963 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
8964 %}
8965 ins_pipe(pipe_serial);
8966 %}
8967
8968 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
8969 match(Set newval (GetAndAddL mem incr));
8970 ins_cost(2 * VOLATILE_REF_COST + 1);
8971 format %{ "get_and_addL $newval, [$mem], $incr" %}
8972 ins_encode %{
8973 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
8974 %}
8975 ins_pipe(pipe_serial);
8976 %}
8977
8978 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
8979 predicate(n->as_LoadStore()->result_not_used());
8980 match(Set dummy (GetAndAddL mem incr));
8981 ins_cost(2 * VOLATILE_REF_COST);
8982 format %{ "get_and_addL [$mem], $incr" %}
8983 ins_encode %{
8984 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
8985 %}
8986 ins_pipe(pipe_serial);
8987 %}
8988
8989 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
8990 match(Set newval (GetAndAddI mem incr));
8991 ins_cost(2 * VOLATILE_REF_COST + 1);
8992 format %{ "get_and_addI $newval, [$mem], $incr" %}
8993 ins_encode %{
8994 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
8995 %}
8996 ins_pipe(pipe_serial);
8997 %}
8998
8999 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9000 predicate(n->as_LoadStore()->result_not_used());
9001 match(Set dummy (GetAndAddI mem incr));
9002 ins_cost(2 * VOLATILE_REF_COST);
9003 format %{ "get_and_addI [$mem], $incr" %}
9004 ins_encode %{
9005 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9006 %}
9007 ins_pipe(pipe_serial);
9008 %}
9009
9010 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9011 match(Set newval (GetAndAddI mem incr));
9012 ins_cost(2 * VOLATILE_REF_COST + 1);
9013 format %{ "get_and_addI $newval, [$mem], $incr" %}
9014 ins_encode %{
9015 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9016 %}
9017 ins_pipe(pipe_serial);
9018 %}
9019
9020 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9021 predicate(n->as_LoadStore()->result_not_used());
9022 match(Set dummy (GetAndAddI mem incr));
9023 ins_cost(2 * VOLATILE_REF_COST);
9024 format %{ "get_and_addI [$mem], $incr" %}
9025 ins_encode %{
9026 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9027 %}
9028 ins_pipe(pipe_serial);
9029 %}
9030
9031 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9032 predicate(needs_acquiring_load_exclusive(n));
9033 match(Set newval (GetAndAddL mem incr));
9034 ins_cost(VOLATILE_REF_COST + 1);
9035 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9036 ins_encode %{
9037 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9038 %}
9039 ins_pipe(pipe_serial);
9040 %}
9041
9042 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9043 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9044 match(Set dummy (GetAndAddL mem incr));
9045 ins_cost(VOLATILE_REF_COST);
9046 format %{ "get_and_addL_acq [$mem], $incr" %}
9047 ins_encode %{
9048 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9049 %}
9050 ins_pipe(pipe_serial);
9051 %}
9052
9053 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9054 predicate(needs_acquiring_load_exclusive(n));
9055 match(Set newval (GetAndAddL mem incr));
9056 ins_cost(VOLATILE_REF_COST + 1);
9057 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9058 ins_encode %{
9059 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9060 %}
9061 ins_pipe(pipe_serial);
9062 %}
9063
9064 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9065 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9066 match(Set dummy (GetAndAddL mem incr));
9067 ins_cost(VOLATILE_REF_COST);
9068 format %{ "get_and_addL_acq [$mem], $incr" %}
9069 ins_encode %{
9070 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9071 %}
9072 ins_pipe(pipe_serial);
9073 %}
9074
9075 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9076 predicate(needs_acquiring_load_exclusive(n));
9077 match(Set newval (GetAndAddI mem incr));
9078 ins_cost(VOLATILE_REF_COST + 1);
9079 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9080 ins_encode %{
9081 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9082 %}
9083 ins_pipe(pipe_serial);
9084 %}
9085
9086 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9087 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9088 match(Set dummy (GetAndAddI mem incr));
9089 ins_cost(VOLATILE_REF_COST);
9090 format %{ "get_and_addI_acq [$mem], $incr" %}
9091 ins_encode %{
9092 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9093 %}
9094 ins_pipe(pipe_serial);
9095 %}
9096
9097 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9098 predicate(needs_acquiring_load_exclusive(n));
9099 match(Set newval (GetAndAddI mem incr));
9100 ins_cost(VOLATILE_REF_COST + 1);
9101 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9102 ins_encode %{
9103 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9104 %}
9105 ins_pipe(pipe_serial);
9106 %}
9107
9108 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9109 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9110 match(Set dummy (GetAndAddI mem incr));
9111 ins_cost(VOLATILE_REF_COST);
9112 format %{ "get_and_addI_acq [$mem], $incr" %}
9113 ins_encode %{
9114 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9115 %}
9116 ins_pipe(pipe_serial);
9117 %}
9118
9119 // Manifest a CmpU result in an integer register.
9120 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9121 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9122 %{
9123 match(Set dst (CmpU3 src1 src2));
9124 effect(KILL flags);
9125
9126 ins_cost(INSN_COST * 3);
9127 format %{
9128 "cmpw $src1, $src2\n\t"
9129 "csetw $dst, ne\n\t"
9130 "cnegw $dst, lo\t# CmpU3(reg)"
9131 %}
9132 ins_encode %{
9133 __ cmpw($src1$$Register, $src2$$Register);
9134 __ csetw($dst$$Register, Assembler::NE);
9135 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9136 %}
9137
9138 ins_pipe(pipe_class_default);
9139 %}
9140
9141 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9142 %{
9143 match(Set dst (CmpU3 src1 src2));
9144 effect(KILL flags);
9145
9146 ins_cost(INSN_COST * 3);
9147 format %{
9148 "subsw zr, $src1, $src2\n\t"
9149 "csetw $dst, ne\n\t"
9150 "cnegw $dst, lo\t# CmpU3(imm)"
9151 %}
9152 ins_encode %{
9153 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9154 __ csetw($dst$$Register, Assembler::NE);
9155 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9156 %}
9157
9158 ins_pipe(pipe_class_default);
9159 %}
9160
9161 // Manifest a CmpUL result in an integer register.
9162 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9163 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9164 %{
9165 match(Set dst (CmpUL3 src1 src2));
9166 effect(KILL flags);
9167
9168 ins_cost(INSN_COST * 3);
9169 format %{
9170 "cmp $src1, $src2\n\t"
9171 "csetw $dst, ne\n\t"
9172 "cnegw $dst, lo\t# CmpUL3(reg)"
9173 %}
9174 ins_encode %{
9175 __ cmp($src1$$Register, $src2$$Register);
9176 __ csetw($dst$$Register, Assembler::NE);
9177 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9178 %}
9179
9180 ins_pipe(pipe_class_default);
9181 %}
9182
9183 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9184 %{
9185 match(Set dst (CmpUL3 src1 src2));
9186 effect(KILL flags);
9187
9188 ins_cost(INSN_COST * 3);
9189 format %{
9190 "subs zr, $src1, $src2\n\t"
9191 "csetw $dst, ne\n\t"
9192 "cnegw $dst, lo\t# CmpUL3(imm)"
9193 %}
9194 ins_encode %{
9195 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9196 __ csetw($dst$$Register, Assembler::NE);
9197 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9198 %}
9199
9200 ins_pipe(pipe_class_default);
9201 %}
9202
9203 // Manifest a CmpL result in an integer register.
9204 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9205 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9206 %{
9207 match(Set dst (CmpL3 src1 src2));
9208 effect(KILL flags);
9209
9210 ins_cost(INSN_COST * 3);
9211 format %{
9212 "cmp $src1, $src2\n\t"
9213 "csetw $dst, ne\n\t"
9214 "cnegw $dst, lt\t# CmpL3(reg)"
9215 %}
9216 ins_encode %{
9217 __ cmp($src1$$Register, $src2$$Register);
9218 __ csetw($dst$$Register, Assembler::NE);
9219 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9220 %}
9221
9222 ins_pipe(pipe_class_default);
9223 %}
9224
9225 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9226 %{
9227 match(Set dst (CmpL3 src1 src2));
9228 effect(KILL flags);
9229
9230 ins_cost(INSN_COST * 3);
9231 format %{
9232 "subs zr, $src1, $src2\n\t"
9233 "csetw $dst, ne\n\t"
9234 "cnegw $dst, lt\t# CmpL3(imm)"
9235 %}
9236 ins_encode %{
9237 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9238 __ csetw($dst$$Register, Assembler::NE);
9239 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9240 %}
9241
9242 ins_pipe(pipe_class_default);
9243 %}
9244
9245 // ============================================================================
9246 // Conditional Move Instructions
9247
9248 // n.b. we have identical rules for both a signed compare op (cmpOp)
9249 // and an unsigned compare op (cmpOpU). it would be nice if we could
9250 // define an op class which merged both inputs and use it to type the
9251 // argument to a single rule. unfortunatelyt his fails because the
9252 // opclass does not live up to the COND_INTER interface of its
9253 // component operands. When the generic code tries to negate the
9254 // operand it ends up running the generci Machoper::negate method
9255 // which throws a ShouldNotHappen. So, we have to provide two flavours
9256 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9257
9258 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9259 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9260
9261 ins_cost(INSN_COST * 2);
9262 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9263
9264 ins_encode %{
9265 __ cselw(as_Register($dst$$reg),
9266 as_Register($src2$$reg),
9267 as_Register($src1$$reg),
9268 (Assembler::Condition)$cmp$$cmpcode);
9269 %}
9270
9271 ins_pipe(icond_reg_reg);
9272 %}
9273
9274 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9275 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9276
9277 ins_cost(INSN_COST * 2);
9278 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9279
9280 ins_encode %{
9281 __ cselw(as_Register($dst$$reg),
9282 as_Register($src2$$reg),
9283 as_Register($src1$$reg),
9284 (Assembler::Condition)$cmp$$cmpcode);
9285 %}
9286
9287 ins_pipe(icond_reg_reg);
9288 %}
9289
9290 // special cases where one arg is zero
9291
9292 // n.b. this is selected in preference to the rule above because it
9293 // avoids loading constant 0 into a source register
9294
9295 // TODO
9296 // we ought only to be able to cull one of these variants as the ideal
9297 // transforms ought always to order the zero consistently (to left/right?)
9298
9299 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9300 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9301
9302 ins_cost(INSN_COST * 2);
9303 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9304
9305 ins_encode %{
9306 __ cselw(as_Register($dst$$reg),
9307 as_Register($src$$reg),
9308 zr,
9309 (Assembler::Condition)$cmp$$cmpcode);
9310 %}
9311
9312 ins_pipe(icond_reg);
9313 %}
9314
9315 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9316 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9317
9318 ins_cost(INSN_COST * 2);
9319 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9320
9321 ins_encode %{
9322 __ cselw(as_Register($dst$$reg),
9323 as_Register($src$$reg),
9324 zr,
9325 (Assembler::Condition)$cmp$$cmpcode);
9326 %}
9327
9328 ins_pipe(icond_reg);
9329 %}
9330
9331 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9332 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9333
9334 ins_cost(INSN_COST * 2);
9335 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9336
9337 ins_encode %{
9338 __ cselw(as_Register($dst$$reg),
9339 zr,
9340 as_Register($src$$reg),
9341 (Assembler::Condition)$cmp$$cmpcode);
9342 %}
9343
9344 ins_pipe(icond_reg);
9345 %}
9346
9347 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9348 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9349
9350 ins_cost(INSN_COST * 2);
9351 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9352
9353 ins_encode %{
9354 __ cselw(as_Register($dst$$reg),
9355 zr,
9356 as_Register($src$$reg),
9357 (Assembler::Condition)$cmp$$cmpcode);
9358 %}
9359
9360 ins_pipe(icond_reg);
9361 %}
9362
9363 // special case for creating a boolean 0 or 1
9364
9365 // n.b. this is selected in preference to the rule above because it
9366 // avoids loading constants 0 and 1 into a source register
9367
9368 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9369 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9370
9371 ins_cost(INSN_COST * 2);
9372 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9373
9374 ins_encode %{
9375 // equivalently
9376 // cset(as_Register($dst$$reg),
9377 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9378 __ csincw(as_Register($dst$$reg),
9379 zr,
9380 zr,
9381 (Assembler::Condition)$cmp$$cmpcode);
9382 %}
9383
9384 ins_pipe(icond_none);
9385 %}
9386
9387 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9388 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9389
9390 ins_cost(INSN_COST * 2);
9391 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9392
9393 ins_encode %{
9394 // equivalently
9395 // cset(as_Register($dst$$reg),
9396 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9397 __ csincw(as_Register($dst$$reg),
9398 zr,
9399 zr,
9400 (Assembler::Condition)$cmp$$cmpcode);
9401 %}
9402
9403 ins_pipe(icond_none);
9404 %}
9405
9406 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9407 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9408
9409 ins_cost(INSN_COST * 2);
9410 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9411
9412 ins_encode %{
9413 __ csel(as_Register($dst$$reg),
9414 as_Register($src2$$reg),
9415 as_Register($src1$$reg),
9416 (Assembler::Condition)$cmp$$cmpcode);
9417 %}
9418
9419 ins_pipe(icond_reg_reg);
9420 %}
9421
9422 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9423 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9424
9425 ins_cost(INSN_COST * 2);
9426 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9427
9428 ins_encode %{
9429 __ csel(as_Register($dst$$reg),
9430 as_Register($src2$$reg),
9431 as_Register($src1$$reg),
9432 (Assembler::Condition)$cmp$$cmpcode);
9433 %}
9434
9435 ins_pipe(icond_reg_reg);
9436 %}
9437
9438 // special cases where one arg is zero
9439
9440 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9441 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9442
9443 ins_cost(INSN_COST * 2);
9444 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9445
9446 ins_encode %{
9447 __ csel(as_Register($dst$$reg),
9448 zr,
9449 as_Register($src$$reg),
9450 (Assembler::Condition)$cmp$$cmpcode);
9451 %}
9452
9453 ins_pipe(icond_reg);
9454 %}
9455
9456 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9457 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9458
9459 ins_cost(INSN_COST * 2);
9460 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9461
9462 ins_encode %{
9463 __ csel(as_Register($dst$$reg),
9464 zr,
9465 as_Register($src$$reg),
9466 (Assembler::Condition)$cmp$$cmpcode);
9467 %}
9468
9469 ins_pipe(icond_reg);
9470 %}
9471
9472 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9473 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9474
9475 ins_cost(INSN_COST * 2);
9476 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9477
9478 ins_encode %{
9479 __ csel(as_Register($dst$$reg),
9480 as_Register($src$$reg),
9481 zr,
9482 (Assembler::Condition)$cmp$$cmpcode);
9483 %}
9484
9485 ins_pipe(icond_reg);
9486 %}
9487
9488 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9489 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9490
9491 ins_cost(INSN_COST * 2);
9492 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9493
9494 ins_encode %{
9495 __ csel(as_Register($dst$$reg),
9496 as_Register($src$$reg),
9497 zr,
9498 (Assembler::Condition)$cmp$$cmpcode);
9499 %}
9500
9501 ins_pipe(icond_reg);
9502 %}
9503
9504 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9505 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9506
9507 ins_cost(INSN_COST * 2);
9508 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9509
9510 ins_encode %{
9511 __ csel(as_Register($dst$$reg),
9512 as_Register($src2$$reg),
9513 as_Register($src1$$reg),
9514 (Assembler::Condition)$cmp$$cmpcode);
9515 %}
9516
9517 ins_pipe(icond_reg_reg);
9518 %}
9519
9520 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9521 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9522
9523 ins_cost(INSN_COST * 2);
9524 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9525
9526 ins_encode %{
9527 __ csel(as_Register($dst$$reg),
9528 as_Register($src2$$reg),
9529 as_Register($src1$$reg),
9530 (Assembler::Condition)$cmp$$cmpcode);
9531 %}
9532
9533 ins_pipe(icond_reg_reg);
9534 %}
9535
9536 // special cases where one arg is zero
9537
9538 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9539 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9540
9541 ins_cost(INSN_COST * 2);
9542 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9543
9544 ins_encode %{
9545 __ csel(as_Register($dst$$reg),
9546 zr,
9547 as_Register($src$$reg),
9548 (Assembler::Condition)$cmp$$cmpcode);
9549 %}
9550
9551 ins_pipe(icond_reg);
9552 %}
9553
9554 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9555 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9556
9557 ins_cost(INSN_COST * 2);
9558 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9559
9560 ins_encode %{
9561 __ csel(as_Register($dst$$reg),
9562 zr,
9563 as_Register($src$$reg),
9564 (Assembler::Condition)$cmp$$cmpcode);
9565 %}
9566
9567 ins_pipe(icond_reg);
9568 %}
9569
9570 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9571 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9572
9573 ins_cost(INSN_COST * 2);
9574 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9575
9576 ins_encode %{
9577 __ csel(as_Register($dst$$reg),
9578 as_Register($src$$reg),
9579 zr,
9580 (Assembler::Condition)$cmp$$cmpcode);
9581 %}
9582
9583 ins_pipe(icond_reg);
9584 %}
9585
9586 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9587 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9588
9589 ins_cost(INSN_COST * 2);
9590 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9591
9592 ins_encode %{
9593 __ csel(as_Register($dst$$reg),
9594 as_Register($src$$reg),
9595 zr,
9596 (Assembler::Condition)$cmp$$cmpcode);
9597 %}
9598
9599 ins_pipe(icond_reg);
9600 %}
9601
9602 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9603 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9604
9605 ins_cost(INSN_COST * 2);
9606 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9607
9608 ins_encode %{
9609 __ cselw(as_Register($dst$$reg),
9610 as_Register($src2$$reg),
9611 as_Register($src1$$reg),
9612 (Assembler::Condition)$cmp$$cmpcode);
9613 %}
9614
9615 ins_pipe(icond_reg_reg);
9616 %}
9617
9618 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9619 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9620
9621 ins_cost(INSN_COST * 2);
9622 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9623
9624 ins_encode %{
9625 __ cselw(as_Register($dst$$reg),
9626 as_Register($src2$$reg),
9627 as_Register($src1$$reg),
9628 (Assembler::Condition)$cmp$$cmpcode);
9629 %}
9630
9631 ins_pipe(icond_reg_reg);
9632 %}
9633
9634 // special cases where one arg is zero
9635
9636 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9637 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9638
9639 ins_cost(INSN_COST * 2);
9640 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9641
9642 ins_encode %{
9643 __ cselw(as_Register($dst$$reg),
9644 zr,
9645 as_Register($src$$reg),
9646 (Assembler::Condition)$cmp$$cmpcode);
9647 %}
9648
9649 ins_pipe(icond_reg);
9650 %}
9651
9652 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9653 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9654
9655 ins_cost(INSN_COST * 2);
9656 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9657
9658 ins_encode %{
9659 __ cselw(as_Register($dst$$reg),
9660 zr,
9661 as_Register($src$$reg),
9662 (Assembler::Condition)$cmp$$cmpcode);
9663 %}
9664
9665 ins_pipe(icond_reg);
9666 %}
9667
9668 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9669 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9670
9671 ins_cost(INSN_COST * 2);
9672 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9673
9674 ins_encode %{
9675 __ cselw(as_Register($dst$$reg),
9676 as_Register($src$$reg),
9677 zr,
9678 (Assembler::Condition)$cmp$$cmpcode);
9679 %}
9680
9681 ins_pipe(icond_reg);
9682 %}
9683
9684 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9685 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9686
9687 ins_cost(INSN_COST * 2);
9688 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9689
9690 ins_encode %{
9691 __ cselw(as_Register($dst$$reg),
9692 as_Register($src$$reg),
9693 zr,
9694 (Assembler::Condition)$cmp$$cmpcode);
9695 %}
9696
9697 ins_pipe(icond_reg);
9698 %}
9699
9700 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9701 %{
9702 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9703
9704 ins_cost(INSN_COST * 3);
9705
9706 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9707 ins_encode %{
9708 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9709 __ fcsels(as_FloatRegister($dst$$reg),
9710 as_FloatRegister($src2$$reg),
9711 as_FloatRegister($src1$$reg),
9712 cond);
9713 %}
9714
9715 ins_pipe(fp_cond_reg_reg_s);
9716 %}
9717
9718 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9719 %{
9720 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9721
9722 ins_cost(INSN_COST * 3);
9723
9724 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9725 ins_encode %{
9726 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9727 __ fcsels(as_FloatRegister($dst$$reg),
9728 as_FloatRegister($src2$$reg),
9729 as_FloatRegister($src1$$reg),
9730 cond);
9731 %}
9732
9733 ins_pipe(fp_cond_reg_reg_s);
9734 %}
9735
9736 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9737 %{
9738 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9739
9740 ins_cost(INSN_COST * 3);
9741
9742 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9743 ins_encode %{
9744 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9745 __ fcseld(as_FloatRegister($dst$$reg),
9746 as_FloatRegister($src2$$reg),
9747 as_FloatRegister($src1$$reg),
9748 cond);
9749 %}
9750
9751 ins_pipe(fp_cond_reg_reg_d);
9752 %}
9753
9754 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9755 %{
9756 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9757
9758 ins_cost(INSN_COST * 3);
9759
9760 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9761 ins_encode %{
9762 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9763 __ fcseld(as_FloatRegister($dst$$reg),
9764 as_FloatRegister($src2$$reg),
9765 as_FloatRegister($src1$$reg),
9766 cond);
9767 %}
9768
9769 ins_pipe(fp_cond_reg_reg_d);
9770 %}
9771
9772 // ============================================================================
9773 // Arithmetic Instructions
9774 //
9775
9776 // Integer Addition
9777
9778 // TODO
9779 // these currently employ operations which do not set CR and hence are
9780 // not flagged as killing CR but we would like to isolate the cases
9781 // where we want to set flags from those where we don't. need to work
9782 // out how to do that.
9783
9784 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9785 match(Set dst (AddI src1 src2));
9786
9787 ins_cost(INSN_COST);
9788 format %{ "addw $dst, $src1, $src2" %}
9789
9790 ins_encode %{
9791 __ addw(as_Register($dst$$reg),
9792 as_Register($src1$$reg),
9793 as_Register($src2$$reg));
9794 %}
9795
9796 ins_pipe(ialu_reg_reg);
9797 %}
9798
9799 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9800 match(Set dst (AddI src1 src2));
9801
9802 ins_cost(INSN_COST);
9803 format %{ "addw $dst, $src1, $src2" %}
9804
9805 // use opcode to indicate that this is an add not a sub
9806 opcode(0x0);
9807
9808 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9809
9810 ins_pipe(ialu_reg_imm);
9811 %}
9812
9813 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9814 match(Set dst (AddI (ConvL2I src1) src2));
9815
9816 ins_cost(INSN_COST);
9817 format %{ "addw $dst, $src1, $src2" %}
9818
9819 // use opcode to indicate that this is an add not a sub
9820 opcode(0x0);
9821
9822 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9823
9824 ins_pipe(ialu_reg_imm);
9825 %}
9826
9827 // Pointer Addition
9828 instruct addP_reg_reg(iRegPNoSp dst, iRegP src1, iRegL src2) %{
9829 match(Set dst (AddP src1 src2));
9830
9831 ins_cost(INSN_COST);
9832 format %{ "add $dst, $src1, $src2\t# ptr" %}
9833
9834 ins_encode %{
9835 __ add(as_Register($dst$$reg),
9836 as_Register($src1$$reg),
9837 as_Register($src2$$reg));
9838 %}
9839
9840 ins_pipe(ialu_reg_reg);
9841 %}
9842
9843 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegP src1, iRegIorL2I src2) %{
9844 match(Set dst (AddP src1 (ConvI2L src2)));
9845
9846 ins_cost(1.9 * INSN_COST);
9847 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9848
9849 ins_encode %{
9850 __ add(as_Register($dst$$reg),
9851 as_Register($src1$$reg),
9852 as_Register($src2$$reg), ext::sxtw);
9853 %}
9854
9855 ins_pipe(ialu_reg_reg);
9856 %}
9857
9858 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegP src1, iRegL src2, immIScale scale) %{
9859 match(Set dst (AddP src1 (LShiftL src2 scale)));
9860
9861 ins_cost(1.9 * INSN_COST);
9862 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9863
9864 ins_encode %{
9865 __ lea(as_Register($dst$$reg),
9866 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9867 Address::lsl($scale$$constant)));
9868 %}
9869
9870 ins_pipe(ialu_reg_reg_shift);
9871 %}
9872
9873 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegP src1, iRegIorL2I src2, immIScale scale) %{
9874 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9875
9876 ins_cost(1.9 * INSN_COST);
9877 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9878
9879 ins_encode %{
9880 __ lea(as_Register($dst$$reg),
9881 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9882 Address::sxtw($scale$$constant)));
9883 %}
9884
9885 ins_pipe(ialu_reg_reg_shift);
9886 %}
9887
9888 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9889 match(Set dst (LShiftL (ConvI2L src) scale));
9890
9891 ins_cost(INSN_COST);
9892 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9893
9894 ins_encode %{
9895 __ sbfiz(as_Register($dst$$reg),
9896 as_Register($src$$reg),
9897 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9898 %}
9899
9900 ins_pipe(ialu_reg_shift);
9901 %}
9902
9903 // Pointer Immediate Addition
9904 // n.b. this needs to be more expensive than using an indirect memory
9905 // operand
9906 instruct addP_reg_imm(iRegPNoSp dst, iRegP src1, immLAddSub src2) %{
9907 match(Set dst (AddP src1 src2));
9908
9909 ins_cost(INSN_COST);
9910 format %{ "add $dst, $src1, $src2\t# ptr" %}
9911
9912 // use opcode to indicate that this is an add not a sub
9913 opcode(0x0);
9914
9915 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9916
9917 ins_pipe(ialu_reg_imm);
9918 %}
9919
9920 // Long Addition
9921 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9922
9923 match(Set dst (AddL src1 src2));
9924
9925 ins_cost(INSN_COST);
9926 format %{ "add $dst, $src1, $src2" %}
9927
9928 ins_encode %{
9929 __ add(as_Register($dst$$reg),
9930 as_Register($src1$$reg),
9931 as_Register($src2$$reg));
9932 %}
9933
9934 ins_pipe(ialu_reg_reg);
9935 %}
9936
9937 // No constant pool entries requiredLong Immediate Addition.
9938 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9939 match(Set dst (AddL src1 src2));
9940
9941 ins_cost(INSN_COST);
9942 format %{ "add $dst, $src1, $src2" %}
9943
9944 // use opcode to indicate that this is an add not a sub
9945 opcode(0x0);
9946
9947 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9948
9949 ins_pipe(ialu_reg_imm);
9950 %}
9951
9952 // Integer Subtraction
9953 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9954 match(Set dst (SubI src1 src2));
9955
9956 ins_cost(INSN_COST);
9957 format %{ "subw $dst, $src1, $src2" %}
9958
9959 ins_encode %{
9960 __ subw(as_Register($dst$$reg),
9961 as_Register($src1$$reg),
9962 as_Register($src2$$reg));
9963 %}
9964
9965 ins_pipe(ialu_reg_reg);
9966 %}
9967
9968 // Immediate Subtraction
9969 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9970 match(Set dst (SubI src1 src2));
9971
9972 ins_cost(INSN_COST);
9973 format %{ "subw $dst, $src1, $src2" %}
9974
9975 // use opcode to indicate that this is a sub not an add
9976 opcode(0x1);
9977
9978 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9979
9980 ins_pipe(ialu_reg_imm);
9981 %}
9982
9983 // Long Subtraction
9984 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9985
9986 match(Set dst (SubL src1 src2));
9987
9988 ins_cost(INSN_COST);
9989 format %{ "sub $dst, $src1, $src2" %}
9990
9991 ins_encode %{
9992 __ sub(as_Register($dst$$reg),
9993 as_Register($src1$$reg),
9994 as_Register($src2$$reg));
9995 %}
9996
9997 ins_pipe(ialu_reg_reg);
9998 %}
9999
10000 // No constant pool entries requiredLong Immediate Subtraction.
10001 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10002 match(Set dst (SubL src1 src2));
10003
10004 ins_cost(INSN_COST);
10005 format %{ "sub$dst, $src1, $src2" %}
10006
10007 // use opcode to indicate that this is a sub not an add
10008 opcode(0x1);
10009
10010 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10011
10012 ins_pipe(ialu_reg_imm);
10013 %}
10014
10015 // Integer Negation (special case for sub)
10016
10017 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10018 match(Set dst (SubI zero src));
10019
10020 ins_cost(INSN_COST);
10021 format %{ "negw $dst, $src\t# int" %}
10022
10023 ins_encode %{
10024 __ negw(as_Register($dst$$reg),
10025 as_Register($src$$reg));
10026 %}
10027
10028 ins_pipe(ialu_reg);
10029 %}
10030
10031 // Long Negation
10032
10033 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10034 match(Set dst (SubL zero src));
10035
10036 ins_cost(INSN_COST);
10037 format %{ "neg $dst, $src\t# long" %}
10038
10039 ins_encode %{
10040 __ neg(as_Register($dst$$reg),
10041 as_Register($src$$reg));
10042 %}
10043
10044 ins_pipe(ialu_reg);
10045 %}
10046
10047 // Integer Multiply
10048
10049 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10050 match(Set dst (MulI src1 src2));
10051
10052 ins_cost(INSN_COST * 3);
10053 format %{ "mulw $dst, $src1, $src2" %}
10054
10055 ins_encode %{
10056 __ mulw(as_Register($dst$$reg),
10057 as_Register($src1$$reg),
10058 as_Register($src2$$reg));
10059 %}
10060
10061 ins_pipe(imul_reg_reg);
10062 %}
10063
10064 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10065 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10066
10067 ins_cost(INSN_COST * 3);
10068 format %{ "smull $dst, $src1, $src2" %}
10069
10070 ins_encode %{
10071 __ smull(as_Register($dst$$reg),
10072 as_Register($src1$$reg),
10073 as_Register($src2$$reg));
10074 %}
10075
10076 ins_pipe(imul_reg_reg);
10077 %}
10078
10079 // Long Multiply
10080
10081 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10082 match(Set dst (MulL src1 src2));
10083
10084 ins_cost(INSN_COST * 5);
10085 format %{ "mul $dst, $src1, $src2" %}
10086
10087 ins_encode %{
10088 __ mul(as_Register($dst$$reg),
10089 as_Register($src1$$reg),
10090 as_Register($src2$$reg));
10091 %}
10092
10093 ins_pipe(lmul_reg_reg);
10094 %}
10095
10096 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10097 %{
10098 match(Set dst (MulHiL src1 src2));
10099
10100 ins_cost(INSN_COST * 7);
10101 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10102
10103 ins_encode %{
10104 __ smulh(as_Register($dst$$reg),
10105 as_Register($src1$$reg),
10106 as_Register($src2$$reg));
10107 %}
10108
10109 ins_pipe(lmul_reg_reg);
10110 %}
10111
10112 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10113 %{
10114 match(Set dst (UMulHiL src1 src2));
10115
10116 ins_cost(INSN_COST * 7);
10117 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10118
10119 ins_encode %{
10120 __ umulh(as_Register($dst$$reg),
10121 as_Register($src1$$reg),
10122 as_Register($src2$$reg));
10123 %}
10124
10125 ins_pipe(lmul_reg_reg);
10126 %}
10127
10128 // Combined Integer Multiply & Add/Sub
10129
10130 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10131 match(Set dst (AddI src3 (MulI src1 src2)));
10132
10133 ins_cost(INSN_COST * 3);
10134 format %{ "madd $dst, $src1, $src2, $src3" %}
10135
10136 ins_encode %{
10137 __ maddw(as_Register($dst$$reg),
10138 as_Register($src1$$reg),
10139 as_Register($src2$$reg),
10140 as_Register($src3$$reg));
10141 %}
10142
10143 ins_pipe(imac_reg_reg);
10144 %}
10145
10146 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10147 match(Set dst (SubI src3 (MulI src1 src2)));
10148
10149 ins_cost(INSN_COST * 3);
10150 format %{ "msub $dst, $src1, $src2, $src3" %}
10151
10152 ins_encode %{
10153 __ msubw(as_Register($dst$$reg),
10154 as_Register($src1$$reg),
10155 as_Register($src2$$reg),
10156 as_Register($src3$$reg));
10157 %}
10158
10159 ins_pipe(imac_reg_reg);
10160 %}
10161
10162 // Combined Integer Multiply & Neg
10163
10164 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10165 match(Set dst (MulI (SubI zero src1) src2));
10166
10167 ins_cost(INSN_COST * 3);
10168 format %{ "mneg $dst, $src1, $src2" %}
10169
10170 ins_encode %{
10171 __ mnegw(as_Register($dst$$reg),
10172 as_Register($src1$$reg),
10173 as_Register($src2$$reg));
10174 %}
10175
10176 ins_pipe(imac_reg_reg);
10177 %}
10178
10179 // Combined Long Multiply & Add/Sub
10180
10181 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10182 match(Set dst (AddL src3 (MulL src1 src2)));
10183
10184 ins_cost(INSN_COST * 5);
10185 format %{ "madd $dst, $src1, $src2, $src3" %}
10186
10187 ins_encode %{
10188 __ madd(as_Register($dst$$reg),
10189 as_Register($src1$$reg),
10190 as_Register($src2$$reg),
10191 as_Register($src3$$reg));
10192 %}
10193
10194 ins_pipe(lmac_reg_reg);
10195 %}
10196
10197 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10198 match(Set dst (SubL src3 (MulL src1 src2)));
10199
10200 ins_cost(INSN_COST * 5);
10201 format %{ "msub $dst, $src1, $src2, $src3" %}
10202
10203 ins_encode %{
10204 __ msub(as_Register($dst$$reg),
10205 as_Register($src1$$reg),
10206 as_Register($src2$$reg),
10207 as_Register($src3$$reg));
10208 %}
10209
10210 ins_pipe(lmac_reg_reg);
10211 %}
10212
10213 // Combined Long Multiply & Neg
10214
10215 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10216 match(Set dst (MulL (SubL zero src1) src2));
10217
10218 ins_cost(INSN_COST * 5);
10219 format %{ "mneg $dst, $src1, $src2" %}
10220
10221 ins_encode %{
10222 __ mneg(as_Register($dst$$reg),
10223 as_Register($src1$$reg),
10224 as_Register($src2$$reg));
10225 %}
10226
10227 ins_pipe(lmac_reg_reg);
10228 %}
10229
10230 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10231
10232 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10233 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10234
10235 ins_cost(INSN_COST * 3);
10236 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10237
10238 ins_encode %{
10239 __ smaddl(as_Register($dst$$reg),
10240 as_Register($src1$$reg),
10241 as_Register($src2$$reg),
10242 as_Register($src3$$reg));
10243 %}
10244
10245 ins_pipe(imac_reg_reg);
10246 %}
10247
10248 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10249 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10250
10251 ins_cost(INSN_COST * 3);
10252 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10253
10254 ins_encode %{
10255 __ smsubl(as_Register($dst$$reg),
10256 as_Register($src1$$reg),
10257 as_Register($src2$$reg),
10258 as_Register($src3$$reg));
10259 %}
10260
10261 ins_pipe(imac_reg_reg);
10262 %}
10263
10264 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10265 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10266
10267 ins_cost(INSN_COST * 3);
10268 format %{ "smnegl $dst, $src1, $src2" %}
10269
10270 ins_encode %{
10271 __ smnegl(as_Register($dst$$reg),
10272 as_Register($src1$$reg),
10273 as_Register($src2$$reg));
10274 %}
10275
10276 ins_pipe(imac_reg_reg);
10277 %}
10278
10279 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10280
10281 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10282 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10283
10284 ins_cost(INSN_COST * 5);
10285 format %{ "mulw rscratch1, $src1, $src2\n\t"
10286 "maddw $dst, $src3, $src4, rscratch1" %}
10287
10288 ins_encode %{
10289 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10290 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10291
10292 ins_pipe(imac_reg_reg);
10293 %}
10294
10295 // Integer Divide
10296
10297 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10298 match(Set dst (DivI src1 src2));
10299
10300 ins_cost(INSN_COST * 19);
10301 format %{ "sdivw $dst, $src1, $src2" %}
10302
10303 ins_encode(aarch64_enc_divw(dst, src1, src2));
10304 ins_pipe(idiv_reg_reg);
10305 %}
10306
10307 // Long Divide
10308
10309 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10310 match(Set dst (DivL src1 src2));
10311
10312 ins_cost(INSN_COST * 35);
10313 format %{ "sdiv $dst, $src1, $src2" %}
10314
10315 ins_encode(aarch64_enc_div(dst, src1, src2));
10316 ins_pipe(ldiv_reg_reg);
10317 %}
10318
10319 // Integer Remainder
10320
10321 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10322 match(Set dst (ModI src1 src2));
10323
10324 ins_cost(INSN_COST * 22);
10325 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10326 "msubw $dst, rscratch1, $src2, $src1" %}
10327
10328 ins_encode(aarch64_enc_modw(dst, src1, src2));
10329 ins_pipe(idiv_reg_reg);
10330 %}
10331
10332 // Long Remainder
10333
10334 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10335 match(Set dst (ModL src1 src2));
10336
10337 ins_cost(INSN_COST * 38);
10338 format %{ "sdiv rscratch1, $src1, $src2\n"
10339 "msub $dst, rscratch1, $src2, $src1" %}
10340
10341 ins_encode(aarch64_enc_mod(dst, src1, src2));
10342 ins_pipe(ldiv_reg_reg);
10343 %}
10344
10345 // Unsigned Integer Divide
10346
10347 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10348 match(Set dst (UDivI src1 src2));
10349
10350 ins_cost(INSN_COST * 19);
10351 format %{ "udivw $dst, $src1, $src2" %}
10352
10353 ins_encode %{
10354 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10355 %}
10356
10357 ins_pipe(idiv_reg_reg);
10358 %}
10359
10360 // Unsigned Long Divide
10361
10362 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10363 match(Set dst (UDivL src1 src2));
10364
10365 ins_cost(INSN_COST * 35);
10366 format %{ "udiv $dst, $src1, $src2" %}
10367
10368 ins_encode %{
10369 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10370 %}
10371
10372 ins_pipe(ldiv_reg_reg);
10373 %}
10374
10375 // Unsigned Integer Remainder
10376
10377 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10378 match(Set dst (UModI src1 src2));
10379
10380 ins_cost(INSN_COST * 22);
10381 format %{ "udivw rscratch1, $src1, $src2\n\t"
10382 "msubw $dst, rscratch1, $src2, $src1" %}
10383
10384 ins_encode %{
10385 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10386 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10387 %}
10388
10389 ins_pipe(idiv_reg_reg);
10390 %}
10391
10392 // Unsigned Long Remainder
10393
10394 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10395 match(Set dst (UModL src1 src2));
10396
10397 ins_cost(INSN_COST * 38);
10398 format %{ "udiv rscratch1, $src1, $src2\n"
10399 "msub $dst, rscratch1, $src2, $src1" %}
10400
10401 ins_encode %{
10402 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10403 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10404 %}
10405
10406 ins_pipe(ldiv_reg_reg);
10407 %}
10408
10409 // Integer Shifts
10410
10411 // Shift Left Register
10412 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10413 match(Set dst (LShiftI src1 src2));
10414
10415 ins_cost(INSN_COST * 2);
10416 format %{ "lslvw $dst, $src1, $src2" %}
10417
10418 ins_encode %{
10419 __ lslvw(as_Register($dst$$reg),
10420 as_Register($src1$$reg),
10421 as_Register($src2$$reg));
10422 %}
10423
10424 ins_pipe(ialu_reg_reg_vshift);
10425 %}
10426
10427 // Shift Left Immediate
10428 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10429 match(Set dst (LShiftI src1 src2));
10430
10431 ins_cost(INSN_COST);
10432 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10433
10434 ins_encode %{
10435 __ lslw(as_Register($dst$$reg),
10436 as_Register($src1$$reg),
10437 $src2$$constant & 0x1f);
10438 %}
10439
10440 ins_pipe(ialu_reg_shift);
10441 %}
10442
10443 // Shift Right Logical Register
10444 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10445 match(Set dst (URShiftI src1 src2));
10446
10447 ins_cost(INSN_COST * 2);
10448 format %{ "lsrvw $dst, $src1, $src2" %}
10449
10450 ins_encode %{
10451 __ lsrvw(as_Register($dst$$reg),
10452 as_Register($src1$$reg),
10453 as_Register($src2$$reg));
10454 %}
10455
10456 ins_pipe(ialu_reg_reg_vshift);
10457 %}
10458
10459 // Shift Right Logical Immediate
10460 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10461 match(Set dst (URShiftI src1 src2));
10462
10463 ins_cost(INSN_COST);
10464 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10465
10466 ins_encode %{
10467 __ lsrw(as_Register($dst$$reg),
10468 as_Register($src1$$reg),
10469 $src2$$constant & 0x1f);
10470 %}
10471
10472 ins_pipe(ialu_reg_shift);
10473 %}
10474
10475 // Shift Right Arithmetic Register
10476 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10477 match(Set dst (RShiftI src1 src2));
10478
10479 ins_cost(INSN_COST * 2);
10480 format %{ "asrvw $dst, $src1, $src2" %}
10481
10482 ins_encode %{
10483 __ asrvw(as_Register($dst$$reg),
10484 as_Register($src1$$reg),
10485 as_Register($src2$$reg));
10486 %}
10487
10488 ins_pipe(ialu_reg_reg_vshift);
10489 %}
10490
10491 // Shift Right Arithmetic Immediate
10492 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10493 match(Set dst (RShiftI src1 src2));
10494
10495 ins_cost(INSN_COST);
10496 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10497
10498 ins_encode %{
10499 __ asrw(as_Register($dst$$reg),
10500 as_Register($src1$$reg),
10501 $src2$$constant & 0x1f);
10502 %}
10503
10504 ins_pipe(ialu_reg_shift);
10505 %}
10506
10507 // Combined Int Mask and Right Shift (using UBFM)
10508 // TODO
10509
10510 // Long Shifts
10511
10512 // Shift Left Register
10513 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10514 match(Set dst (LShiftL src1 src2));
10515
10516 ins_cost(INSN_COST * 2);
10517 format %{ "lslv $dst, $src1, $src2" %}
10518
10519 ins_encode %{
10520 __ lslv(as_Register($dst$$reg),
10521 as_Register($src1$$reg),
10522 as_Register($src2$$reg));
10523 %}
10524
10525 ins_pipe(ialu_reg_reg_vshift);
10526 %}
10527
10528 // Shift Left Immediate
10529 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10530 match(Set dst (LShiftL src1 src2));
10531
10532 ins_cost(INSN_COST);
10533 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10534
10535 ins_encode %{
10536 __ lsl(as_Register($dst$$reg),
10537 as_Register($src1$$reg),
10538 $src2$$constant & 0x3f);
10539 %}
10540
10541 ins_pipe(ialu_reg_shift);
10542 %}
10543
10544 // Shift Right Logical Register
10545 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10546 match(Set dst (URShiftL src1 src2));
10547
10548 ins_cost(INSN_COST * 2);
10549 format %{ "lsrv $dst, $src1, $src2" %}
10550
10551 ins_encode %{
10552 __ lsrv(as_Register($dst$$reg),
10553 as_Register($src1$$reg),
10554 as_Register($src2$$reg));
10555 %}
10556
10557 ins_pipe(ialu_reg_reg_vshift);
10558 %}
10559
10560 // Shift Right Logical Immediate
10561 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10562 match(Set dst (URShiftL src1 src2));
10563
10564 ins_cost(INSN_COST);
10565 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10566
10567 ins_encode %{
10568 __ lsr(as_Register($dst$$reg),
10569 as_Register($src1$$reg),
10570 $src2$$constant & 0x3f);
10571 %}
10572
10573 ins_pipe(ialu_reg_shift);
10574 %}
10575
10576 // A special-case pattern for card table stores.
10577 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10578 match(Set dst (URShiftL (CastP2X src1) src2));
10579
10580 ins_cost(INSN_COST);
10581 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10582
10583 ins_encode %{
10584 __ lsr(as_Register($dst$$reg),
10585 as_Register($src1$$reg),
10586 $src2$$constant & 0x3f);
10587 %}
10588
10589 ins_pipe(ialu_reg_shift);
10590 %}
10591
10592 // Shift Right Arithmetic Register
10593 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10594 match(Set dst (RShiftL src1 src2));
10595
10596 ins_cost(INSN_COST * 2);
10597 format %{ "asrv $dst, $src1, $src2" %}
10598
10599 ins_encode %{
10600 __ asrv(as_Register($dst$$reg),
10601 as_Register($src1$$reg),
10602 as_Register($src2$$reg));
10603 %}
10604
10605 ins_pipe(ialu_reg_reg_vshift);
10606 %}
10607
10608 // Shift Right Arithmetic Immediate
10609 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10610 match(Set dst (RShiftL src1 src2));
10611
10612 ins_cost(INSN_COST);
10613 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10614
10615 ins_encode %{
10616 __ asr(as_Register($dst$$reg),
10617 as_Register($src1$$reg),
10618 $src2$$constant & 0x3f);
10619 %}
10620
10621 ins_pipe(ialu_reg_shift);
10622 %}
10623
10624 // BEGIN This section of the file is automatically generated. Do not edit --------------
10625 // This section is generated from aarch64_ad.m4
10626
10627 // This pattern is automatically generated from aarch64_ad.m4
10628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10629 instruct regL_not_reg(iRegLNoSp dst,
10630 iRegL src1, immL_M1 m1,
10631 rFlagsReg cr) %{
10632 match(Set dst (XorL src1 m1));
10633 ins_cost(INSN_COST);
10634 format %{ "eon $dst, $src1, zr" %}
10635
10636 ins_encode %{
10637 __ eon(as_Register($dst$$reg),
10638 as_Register($src1$$reg),
10639 zr,
10640 Assembler::LSL, 0);
10641 %}
10642
10643 ins_pipe(ialu_reg);
10644 %}
10645
10646 // This pattern is automatically generated from aarch64_ad.m4
10647 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10648 instruct regI_not_reg(iRegINoSp dst,
10649 iRegIorL2I src1, immI_M1 m1,
10650 rFlagsReg cr) %{
10651 match(Set dst (XorI src1 m1));
10652 ins_cost(INSN_COST);
10653 format %{ "eonw $dst, $src1, zr" %}
10654
10655 ins_encode %{
10656 __ eonw(as_Register($dst$$reg),
10657 as_Register($src1$$reg),
10658 zr,
10659 Assembler::LSL, 0);
10660 %}
10661
10662 ins_pipe(ialu_reg);
10663 %}
10664
10665 // This pattern is automatically generated from aarch64_ad.m4
10666 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10667 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10668 immI0 zero, iRegIorL2I src1, immI src2) %{
10669 match(Set dst (SubI zero (URShiftI src1 src2)));
10670
10671 ins_cost(1.9 * INSN_COST);
10672 format %{ "negw $dst, $src1, LSR $src2" %}
10673
10674 ins_encode %{
10675 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10676 Assembler::LSR, $src2$$constant & 0x1f);
10677 %}
10678
10679 ins_pipe(ialu_reg_shift);
10680 %}
10681
10682 // This pattern is automatically generated from aarch64_ad.m4
10683 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10684 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10685 immI0 zero, iRegIorL2I src1, immI src2) %{
10686 match(Set dst (SubI zero (RShiftI src1 src2)));
10687
10688 ins_cost(1.9 * INSN_COST);
10689 format %{ "negw $dst, $src1, ASR $src2" %}
10690
10691 ins_encode %{
10692 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10693 Assembler::ASR, $src2$$constant & 0x1f);
10694 %}
10695
10696 ins_pipe(ialu_reg_shift);
10697 %}
10698
10699 // This pattern is automatically generated from aarch64_ad.m4
10700 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10701 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10702 immI0 zero, iRegIorL2I src1, immI src2) %{
10703 match(Set dst (SubI zero (LShiftI src1 src2)));
10704
10705 ins_cost(1.9 * INSN_COST);
10706 format %{ "negw $dst, $src1, LSL $src2" %}
10707
10708 ins_encode %{
10709 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10710 Assembler::LSL, $src2$$constant & 0x1f);
10711 %}
10712
10713 ins_pipe(ialu_reg_shift);
10714 %}
10715
10716 // This pattern is automatically generated from aarch64_ad.m4
10717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10718 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10719 immL0 zero, iRegL src1, immI src2) %{
10720 match(Set dst (SubL zero (URShiftL src1 src2)));
10721
10722 ins_cost(1.9 * INSN_COST);
10723 format %{ "neg $dst, $src1, LSR $src2" %}
10724
10725 ins_encode %{
10726 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10727 Assembler::LSR, $src2$$constant & 0x3f);
10728 %}
10729
10730 ins_pipe(ialu_reg_shift);
10731 %}
10732
10733 // This pattern is automatically generated from aarch64_ad.m4
10734 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10735 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10736 immL0 zero, iRegL src1, immI src2) %{
10737 match(Set dst (SubL zero (RShiftL src1 src2)));
10738
10739 ins_cost(1.9 * INSN_COST);
10740 format %{ "neg $dst, $src1, ASR $src2" %}
10741
10742 ins_encode %{
10743 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10744 Assembler::ASR, $src2$$constant & 0x3f);
10745 %}
10746
10747 ins_pipe(ialu_reg_shift);
10748 %}
10749
10750 // This pattern is automatically generated from aarch64_ad.m4
10751 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10752 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10753 immL0 zero, iRegL src1, immI src2) %{
10754 match(Set dst (SubL zero (LShiftL src1 src2)));
10755
10756 ins_cost(1.9 * INSN_COST);
10757 format %{ "neg $dst, $src1, LSL $src2" %}
10758
10759 ins_encode %{
10760 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10761 Assembler::LSL, $src2$$constant & 0x3f);
10762 %}
10763
10764 ins_pipe(ialu_reg_shift);
10765 %}
10766
10767 // This pattern is automatically generated from aarch64_ad.m4
10768 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10769 instruct AndI_reg_not_reg(iRegINoSp dst,
10770 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10771 match(Set dst (AndI src1 (XorI src2 m1)));
10772 ins_cost(INSN_COST);
10773 format %{ "bicw $dst, $src1, $src2" %}
10774
10775 ins_encode %{
10776 __ bicw(as_Register($dst$$reg),
10777 as_Register($src1$$reg),
10778 as_Register($src2$$reg),
10779 Assembler::LSL, 0);
10780 %}
10781
10782 ins_pipe(ialu_reg_reg);
10783 %}
10784
10785 // This pattern is automatically generated from aarch64_ad.m4
10786 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10787 instruct AndL_reg_not_reg(iRegLNoSp dst,
10788 iRegL src1, iRegL src2, immL_M1 m1) %{
10789 match(Set dst (AndL src1 (XorL src2 m1)));
10790 ins_cost(INSN_COST);
10791 format %{ "bic $dst, $src1, $src2" %}
10792
10793 ins_encode %{
10794 __ bic(as_Register($dst$$reg),
10795 as_Register($src1$$reg),
10796 as_Register($src2$$reg),
10797 Assembler::LSL, 0);
10798 %}
10799
10800 ins_pipe(ialu_reg_reg);
10801 %}
10802
10803 // This pattern is automatically generated from aarch64_ad.m4
10804 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10805 instruct OrI_reg_not_reg(iRegINoSp dst,
10806 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10807 match(Set dst (OrI src1 (XorI src2 m1)));
10808 ins_cost(INSN_COST);
10809 format %{ "ornw $dst, $src1, $src2" %}
10810
10811 ins_encode %{
10812 __ ornw(as_Register($dst$$reg),
10813 as_Register($src1$$reg),
10814 as_Register($src2$$reg),
10815 Assembler::LSL, 0);
10816 %}
10817
10818 ins_pipe(ialu_reg_reg);
10819 %}
10820
10821 // This pattern is automatically generated from aarch64_ad.m4
10822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10823 instruct OrL_reg_not_reg(iRegLNoSp dst,
10824 iRegL src1, iRegL src2, immL_M1 m1) %{
10825 match(Set dst (OrL src1 (XorL src2 m1)));
10826 ins_cost(INSN_COST);
10827 format %{ "orn $dst, $src1, $src2" %}
10828
10829 ins_encode %{
10830 __ orn(as_Register($dst$$reg),
10831 as_Register($src1$$reg),
10832 as_Register($src2$$reg),
10833 Assembler::LSL, 0);
10834 %}
10835
10836 ins_pipe(ialu_reg_reg);
10837 %}
10838
10839 // This pattern is automatically generated from aarch64_ad.m4
10840 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10841 instruct XorI_reg_not_reg(iRegINoSp dst,
10842 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10843 match(Set dst (XorI m1 (XorI src2 src1)));
10844 ins_cost(INSN_COST);
10845 format %{ "eonw $dst, $src1, $src2" %}
10846
10847 ins_encode %{
10848 __ eonw(as_Register($dst$$reg),
10849 as_Register($src1$$reg),
10850 as_Register($src2$$reg),
10851 Assembler::LSL, 0);
10852 %}
10853
10854 ins_pipe(ialu_reg_reg);
10855 %}
10856
10857 // This pattern is automatically generated from aarch64_ad.m4
10858 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10859 instruct XorL_reg_not_reg(iRegLNoSp dst,
10860 iRegL src1, iRegL src2, immL_M1 m1) %{
10861 match(Set dst (XorL m1 (XorL src2 src1)));
10862 ins_cost(INSN_COST);
10863 format %{ "eon $dst, $src1, $src2" %}
10864
10865 ins_encode %{
10866 __ eon(as_Register($dst$$reg),
10867 as_Register($src1$$reg),
10868 as_Register($src2$$reg),
10869 Assembler::LSL, 0);
10870 %}
10871
10872 ins_pipe(ialu_reg_reg);
10873 %}
10874
10875 // This pattern is automatically generated from aarch64_ad.m4
10876 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10877 // val & (-1 ^ (val >>> shift)) ==> bicw
10878 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10879 iRegIorL2I src1, iRegIorL2I src2,
10880 immI src3, immI_M1 src4) %{
10881 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10882 ins_cost(1.9 * INSN_COST);
10883 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10884
10885 ins_encode %{
10886 __ bicw(as_Register($dst$$reg),
10887 as_Register($src1$$reg),
10888 as_Register($src2$$reg),
10889 Assembler::LSR,
10890 $src3$$constant & 0x1f);
10891 %}
10892
10893 ins_pipe(ialu_reg_reg_shift);
10894 %}
10895
10896 // This pattern is automatically generated from aarch64_ad.m4
10897 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10898 // val & (-1 ^ (val >>> shift)) ==> bic
10899 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10900 iRegL src1, iRegL src2,
10901 immI src3, immL_M1 src4) %{
10902 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10903 ins_cost(1.9 * INSN_COST);
10904 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10905
10906 ins_encode %{
10907 __ bic(as_Register($dst$$reg),
10908 as_Register($src1$$reg),
10909 as_Register($src2$$reg),
10910 Assembler::LSR,
10911 $src3$$constant & 0x3f);
10912 %}
10913
10914 ins_pipe(ialu_reg_reg_shift);
10915 %}
10916
10917 // This pattern is automatically generated from aarch64_ad.m4
10918 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10919 // val & (-1 ^ (val >> shift)) ==> bicw
10920 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10921 iRegIorL2I src1, iRegIorL2I src2,
10922 immI src3, immI_M1 src4) %{
10923 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10924 ins_cost(1.9 * INSN_COST);
10925 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10926
10927 ins_encode %{
10928 __ bicw(as_Register($dst$$reg),
10929 as_Register($src1$$reg),
10930 as_Register($src2$$reg),
10931 Assembler::ASR,
10932 $src3$$constant & 0x1f);
10933 %}
10934
10935 ins_pipe(ialu_reg_reg_shift);
10936 %}
10937
10938 // This pattern is automatically generated from aarch64_ad.m4
10939 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10940 // val & (-1 ^ (val >> shift)) ==> bic
10941 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10942 iRegL src1, iRegL src2,
10943 immI src3, immL_M1 src4) %{
10944 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10945 ins_cost(1.9 * INSN_COST);
10946 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10947
10948 ins_encode %{
10949 __ bic(as_Register($dst$$reg),
10950 as_Register($src1$$reg),
10951 as_Register($src2$$reg),
10952 Assembler::ASR,
10953 $src3$$constant & 0x3f);
10954 %}
10955
10956 ins_pipe(ialu_reg_reg_shift);
10957 %}
10958
10959 // This pattern is automatically generated from aarch64_ad.m4
10960 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10961 // val & (-1 ^ (val ror shift)) ==> bicw
10962 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10963 iRegIorL2I src1, iRegIorL2I src2,
10964 immI src3, immI_M1 src4) %{
10965 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10966 ins_cost(1.9 * INSN_COST);
10967 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10968
10969 ins_encode %{
10970 __ bicw(as_Register($dst$$reg),
10971 as_Register($src1$$reg),
10972 as_Register($src2$$reg),
10973 Assembler::ROR,
10974 $src3$$constant & 0x1f);
10975 %}
10976
10977 ins_pipe(ialu_reg_reg_shift);
10978 %}
10979
10980 // This pattern is automatically generated from aarch64_ad.m4
10981 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10982 // val & (-1 ^ (val ror shift)) ==> bic
10983 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10984 iRegL src1, iRegL src2,
10985 immI src3, immL_M1 src4) %{
10986 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10987 ins_cost(1.9 * INSN_COST);
10988 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10989
10990 ins_encode %{
10991 __ bic(as_Register($dst$$reg),
10992 as_Register($src1$$reg),
10993 as_Register($src2$$reg),
10994 Assembler::ROR,
10995 $src3$$constant & 0x3f);
10996 %}
10997
10998 ins_pipe(ialu_reg_reg_shift);
10999 %}
11000
11001 // This pattern is automatically generated from aarch64_ad.m4
11002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11003 // val & (-1 ^ (val << shift)) ==> bicw
11004 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11005 iRegIorL2I src1, iRegIorL2I src2,
11006 immI src3, immI_M1 src4) %{
11007 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11008 ins_cost(1.9 * INSN_COST);
11009 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11010
11011 ins_encode %{
11012 __ bicw(as_Register($dst$$reg),
11013 as_Register($src1$$reg),
11014 as_Register($src2$$reg),
11015 Assembler::LSL,
11016 $src3$$constant & 0x1f);
11017 %}
11018
11019 ins_pipe(ialu_reg_reg_shift);
11020 %}
11021
11022 // This pattern is automatically generated from aarch64_ad.m4
11023 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11024 // val & (-1 ^ (val << shift)) ==> bic
11025 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11026 iRegL src1, iRegL src2,
11027 immI src3, immL_M1 src4) %{
11028 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11029 ins_cost(1.9 * INSN_COST);
11030 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11031
11032 ins_encode %{
11033 __ bic(as_Register($dst$$reg),
11034 as_Register($src1$$reg),
11035 as_Register($src2$$reg),
11036 Assembler::LSL,
11037 $src3$$constant & 0x3f);
11038 %}
11039
11040 ins_pipe(ialu_reg_reg_shift);
11041 %}
11042
11043 // This pattern is automatically generated from aarch64_ad.m4
11044 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11045 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11046 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11047 iRegIorL2I src1, iRegIorL2I src2,
11048 immI src3, immI_M1 src4) %{
11049 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11050 ins_cost(1.9 * INSN_COST);
11051 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11052
11053 ins_encode %{
11054 __ eonw(as_Register($dst$$reg),
11055 as_Register($src1$$reg),
11056 as_Register($src2$$reg),
11057 Assembler::LSR,
11058 $src3$$constant & 0x1f);
11059 %}
11060
11061 ins_pipe(ialu_reg_reg_shift);
11062 %}
11063
11064 // This pattern is automatically generated from aarch64_ad.m4
11065 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11066 // val ^ (-1 ^ (val >>> shift)) ==> eon
11067 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11068 iRegL src1, iRegL src2,
11069 immI src3, immL_M1 src4) %{
11070 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11071 ins_cost(1.9 * INSN_COST);
11072 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11073
11074 ins_encode %{
11075 __ eon(as_Register($dst$$reg),
11076 as_Register($src1$$reg),
11077 as_Register($src2$$reg),
11078 Assembler::LSR,
11079 $src3$$constant & 0x3f);
11080 %}
11081
11082 ins_pipe(ialu_reg_reg_shift);
11083 %}
11084
11085 // This pattern is automatically generated from aarch64_ad.m4
11086 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11087 // val ^ (-1 ^ (val >> shift)) ==> eonw
11088 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11089 iRegIorL2I src1, iRegIorL2I src2,
11090 immI src3, immI_M1 src4) %{
11091 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11092 ins_cost(1.9 * INSN_COST);
11093 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11094
11095 ins_encode %{
11096 __ eonw(as_Register($dst$$reg),
11097 as_Register($src1$$reg),
11098 as_Register($src2$$reg),
11099 Assembler::ASR,
11100 $src3$$constant & 0x1f);
11101 %}
11102
11103 ins_pipe(ialu_reg_reg_shift);
11104 %}
11105
11106 // This pattern is automatically generated from aarch64_ad.m4
11107 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11108 // val ^ (-1 ^ (val >> shift)) ==> eon
11109 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11110 iRegL src1, iRegL src2,
11111 immI src3, immL_M1 src4) %{
11112 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11113 ins_cost(1.9 * INSN_COST);
11114 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11115
11116 ins_encode %{
11117 __ eon(as_Register($dst$$reg),
11118 as_Register($src1$$reg),
11119 as_Register($src2$$reg),
11120 Assembler::ASR,
11121 $src3$$constant & 0x3f);
11122 %}
11123
11124 ins_pipe(ialu_reg_reg_shift);
11125 %}
11126
11127 // This pattern is automatically generated from aarch64_ad.m4
11128 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11129 // val ^ (-1 ^ (val ror shift)) ==> eonw
11130 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11131 iRegIorL2I src1, iRegIorL2I src2,
11132 immI src3, immI_M1 src4) %{
11133 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11134 ins_cost(1.9 * INSN_COST);
11135 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11136
11137 ins_encode %{
11138 __ eonw(as_Register($dst$$reg),
11139 as_Register($src1$$reg),
11140 as_Register($src2$$reg),
11141 Assembler::ROR,
11142 $src3$$constant & 0x1f);
11143 %}
11144
11145 ins_pipe(ialu_reg_reg_shift);
11146 %}
11147
11148 // This pattern is automatically generated from aarch64_ad.m4
11149 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11150 // val ^ (-1 ^ (val ror shift)) ==> eon
11151 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11152 iRegL src1, iRegL src2,
11153 immI src3, immL_M1 src4) %{
11154 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11155 ins_cost(1.9 * INSN_COST);
11156 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11157
11158 ins_encode %{
11159 __ eon(as_Register($dst$$reg),
11160 as_Register($src1$$reg),
11161 as_Register($src2$$reg),
11162 Assembler::ROR,
11163 $src3$$constant & 0x3f);
11164 %}
11165
11166 ins_pipe(ialu_reg_reg_shift);
11167 %}
11168
11169 // This pattern is automatically generated from aarch64_ad.m4
11170 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11171 // val ^ (-1 ^ (val << shift)) ==> eonw
11172 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11173 iRegIorL2I src1, iRegIorL2I src2,
11174 immI src3, immI_M1 src4) %{
11175 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11176 ins_cost(1.9 * INSN_COST);
11177 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11178
11179 ins_encode %{
11180 __ eonw(as_Register($dst$$reg),
11181 as_Register($src1$$reg),
11182 as_Register($src2$$reg),
11183 Assembler::LSL,
11184 $src3$$constant & 0x1f);
11185 %}
11186
11187 ins_pipe(ialu_reg_reg_shift);
11188 %}
11189
11190 // This pattern is automatically generated from aarch64_ad.m4
11191 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11192 // val ^ (-1 ^ (val << shift)) ==> eon
11193 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11194 iRegL src1, iRegL src2,
11195 immI src3, immL_M1 src4) %{
11196 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11197 ins_cost(1.9 * INSN_COST);
11198 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11199
11200 ins_encode %{
11201 __ eon(as_Register($dst$$reg),
11202 as_Register($src1$$reg),
11203 as_Register($src2$$reg),
11204 Assembler::LSL,
11205 $src3$$constant & 0x3f);
11206 %}
11207
11208 ins_pipe(ialu_reg_reg_shift);
11209 %}
11210
11211 // This pattern is automatically generated from aarch64_ad.m4
11212 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11213 // val | (-1 ^ (val >>> shift)) ==> ornw
11214 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11215 iRegIorL2I src1, iRegIorL2I src2,
11216 immI src3, immI_M1 src4) %{
11217 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11218 ins_cost(1.9 * INSN_COST);
11219 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11220
11221 ins_encode %{
11222 __ ornw(as_Register($dst$$reg),
11223 as_Register($src1$$reg),
11224 as_Register($src2$$reg),
11225 Assembler::LSR,
11226 $src3$$constant & 0x1f);
11227 %}
11228
11229 ins_pipe(ialu_reg_reg_shift);
11230 %}
11231
11232 // This pattern is automatically generated from aarch64_ad.m4
11233 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11234 // val | (-1 ^ (val >>> shift)) ==> orn
11235 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11236 iRegL src1, iRegL src2,
11237 immI src3, immL_M1 src4) %{
11238 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11239 ins_cost(1.9 * INSN_COST);
11240 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11241
11242 ins_encode %{
11243 __ orn(as_Register($dst$$reg),
11244 as_Register($src1$$reg),
11245 as_Register($src2$$reg),
11246 Assembler::LSR,
11247 $src3$$constant & 0x3f);
11248 %}
11249
11250 ins_pipe(ialu_reg_reg_shift);
11251 %}
11252
11253 // This pattern is automatically generated from aarch64_ad.m4
11254 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11255 // val | (-1 ^ (val >> shift)) ==> ornw
11256 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11257 iRegIorL2I src1, iRegIorL2I src2,
11258 immI src3, immI_M1 src4) %{
11259 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11260 ins_cost(1.9 * INSN_COST);
11261 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11262
11263 ins_encode %{
11264 __ ornw(as_Register($dst$$reg),
11265 as_Register($src1$$reg),
11266 as_Register($src2$$reg),
11267 Assembler::ASR,
11268 $src3$$constant & 0x1f);
11269 %}
11270
11271 ins_pipe(ialu_reg_reg_shift);
11272 %}
11273
11274 // This pattern is automatically generated from aarch64_ad.m4
11275 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11276 // val | (-1 ^ (val >> shift)) ==> orn
11277 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11278 iRegL src1, iRegL src2,
11279 immI src3, immL_M1 src4) %{
11280 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11281 ins_cost(1.9 * INSN_COST);
11282 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11283
11284 ins_encode %{
11285 __ orn(as_Register($dst$$reg),
11286 as_Register($src1$$reg),
11287 as_Register($src2$$reg),
11288 Assembler::ASR,
11289 $src3$$constant & 0x3f);
11290 %}
11291
11292 ins_pipe(ialu_reg_reg_shift);
11293 %}
11294
11295 // This pattern is automatically generated from aarch64_ad.m4
11296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11297 // val | (-1 ^ (val ror shift)) ==> ornw
11298 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11299 iRegIorL2I src1, iRegIorL2I src2,
11300 immI src3, immI_M1 src4) %{
11301 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11302 ins_cost(1.9 * INSN_COST);
11303 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11304
11305 ins_encode %{
11306 __ ornw(as_Register($dst$$reg),
11307 as_Register($src1$$reg),
11308 as_Register($src2$$reg),
11309 Assembler::ROR,
11310 $src3$$constant & 0x1f);
11311 %}
11312
11313 ins_pipe(ialu_reg_reg_shift);
11314 %}
11315
11316 // This pattern is automatically generated from aarch64_ad.m4
11317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11318 // val | (-1 ^ (val ror shift)) ==> orn
11319 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11320 iRegL src1, iRegL src2,
11321 immI src3, immL_M1 src4) %{
11322 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11323 ins_cost(1.9 * INSN_COST);
11324 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11325
11326 ins_encode %{
11327 __ orn(as_Register($dst$$reg),
11328 as_Register($src1$$reg),
11329 as_Register($src2$$reg),
11330 Assembler::ROR,
11331 $src3$$constant & 0x3f);
11332 %}
11333
11334 ins_pipe(ialu_reg_reg_shift);
11335 %}
11336
11337 // This pattern is automatically generated from aarch64_ad.m4
11338 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11339 // val | (-1 ^ (val << shift)) ==> ornw
11340 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11341 iRegIorL2I src1, iRegIorL2I src2,
11342 immI src3, immI_M1 src4) %{
11343 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11344 ins_cost(1.9 * INSN_COST);
11345 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11346
11347 ins_encode %{
11348 __ ornw(as_Register($dst$$reg),
11349 as_Register($src1$$reg),
11350 as_Register($src2$$reg),
11351 Assembler::LSL,
11352 $src3$$constant & 0x1f);
11353 %}
11354
11355 ins_pipe(ialu_reg_reg_shift);
11356 %}
11357
11358 // This pattern is automatically generated from aarch64_ad.m4
11359 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11360 // val | (-1 ^ (val << shift)) ==> orn
11361 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11362 iRegL src1, iRegL src2,
11363 immI src3, immL_M1 src4) %{
11364 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11365 ins_cost(1.9 * INSN_COST);
11366 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11367
11368 ins_encode %{
11369 __ orn(as_Register($dst$$reg),
11370 as_Register($src1$$reg),
11371 as_Register($src2$$reg),
11372 Assembler::LSL,
11373 $src3$$constant & 0x3f);
11374 %}
11375
11376 ins_pipe(ialu_reg_reg_shift);
11377 %}
11378
11379 // This pattern is automatically generated from aarch64_ad.m4
11380 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11381 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11382 iRegIorL2I src1, iRegIorL2I src2,
11383 immI src3) %{
11384 match(Set dst (AndI src1 (URShiftI src2 src3)));
11385
11386 ins_cost(1.9 * INSN_COST);
11387 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11388
11389 ins_encode %{
11390 __ andw(as_Register($dst$$reg),
11391 as_Register($src1$$reg),
11392 as_Register($src2$$reg),
11393 Assembler::LSR,
11394 $src3$$constant & 0x1f);
11395 %}
11396
11397 ins_pipe(ialu_reg_reg_shift);
11398 %}
11399
11400 // This pattern is automatically generated from aarch64_ad.m4
11401 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11402 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11403 iRegL src1, iRegL src2,
11404 immI src3) %{
11405 match(Set dst (AndL src1 (URShiftL src2 src3)));
11406
11407 ins_cost(1.9 * INSN_COST);
11408 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11409
11410 ins_encode %{
11411 __ andr(as_Register($dst$$reg),
11412 as_Register($src1$$reg),
11413 as_Register($src2$$reg),
11414 Assembler::LSR,
11415 $src3$$constant & 0x3f);
11416 %}
11417
11418 ins_pipe(ialu_reg_reg_shift);
11419 %}
11420
11421 // This pattern is automatically generated from aarch64_ad.m4
11422 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11423 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11424 iRegIorL2I src1, iRegIorL2I src2,
11425 immI src3) %{
11426 match(Set dst (AndI src1 (RShiftI src2 src3)));
11427
11428 ins_cost(1.9 * INSN_COST);
11429 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11430
11431 ins_encode %{
11432 __ andw(as_Register($dst$$reg),
11433 as_Register($src1$$reg),
11434 as_Register($src2$$reg),
11435 Assembler::ASR,
11436 $src3$$constant & 0x1f);
11437 %}
11438
11439 ins_pipe(ialu_reg_reg_shift);
11440 %}
11441
11442 // This pattern is automatically generated from aarch64_ad.m4
11443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11444 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11445 iRegL src1, iRegL src2,
11446 immI src3) %{
11447 match(Set dst (AndL src1 (RShiftL src2 src3)));
11448
11449 ins_cost(1.9 * INSN_COST);
11450 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11451
11452 ins_encode %{
11453 __ andr(as_Register($dst$$reg),
11454 as_Register($src1$$reg),
11455 as_Register($src2$$reg),
11456 Assembler::ASR,
11457 $src3$$constant & 0x3f);
11458 %}
11459
11460 ins_pipe(ialu_reg_reg_shift);
11461 %}
11462
11463 // This pattern is automatically generated from aarch64_ad.m4
11464 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11465 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11466 iRegIorL2I src1, iRegIorL2I src2,
11467 immI src3) %{
11468 match(Set dst (AndI src1 (LShiftI src2 src3)));
11469
11470 ins_cost(1.9 * INSN_COST);
11471 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11472
11473 ins_encode %{
11474 __ andw(as_Register($dst$$reg),
11475 as_Register($src1$$reg),
11476 as_Register($src2$$reg),
11477 Assembler::LSL,
11478 $src3$$constant & 0x1f);
11479 %}
11480
11481 ins_pipe(ialu_reg_reg_shift);
11482 %}
11483
11484 // This pattern is automatically generated from aarch64_ad.m4
11485 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11486 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11487 iRegL src1, iRegL src2,
11488 immI src3) %{
11489 match(Set dst (AndL src1 (LShiftL src2 src3)));
11490
11491 ins_cost(1.9 * INSN_COST);
11492 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11493
11494 ins_encode %{
11495 __ andr(as_Register($dst$$reg),
11496 as_Register($src1$$reg),
11497 as_Register($src2$$reg),
11498 Assembler::LSL,
11499 $src3$$constant & 0x3f);
11500 %}
11501
11502 ins_pipe(ialu_reg_reg_shift);
11503 %}
11504
11505 // This pattern is automatically generated from aarch64_ad.m4
11506 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11507 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11508 iRegIorL2I src1, iRegIorL2I src2,
11509 immI src3) %{
11510 match(Set dst (AndI src1 (RotateRight src2 src3)));
11511
11512 ins_cost(1.9 * INSN_COST);
11513 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11514
11515 ins_encode %{
11516 __ andw(as_Register($dst$$reg),
11517 as_Register($src1$$reg),
11518 as_Register($src2$$reg),
11519 Assembler::ROR,
11520 $src3$$constant & 0x1f);
11521 %}
11522
11523 ins_pipe(ialu_reg_reg_shift);
11524 %}
11525
11526 // This pattern is automatically generated from aarch64_ad.m4
11527 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11528 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11529 iRegL src1, iRegL src2,
11530 immI src3) %{
11531 match(Set dst (AndL src1 (RotateRight src2 src3)));
11532
11533 ins_cost(1.9 * INSN_COST);
11534 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11535
11536 ins_encode %{
11537 __ andr(as_Register($dst$$reg),
11538 as_Register($src1$$reg),
11539 as_Register($src2$$reg),
11540 Assembler::ROR,
11541 $src3$$constant & 0x3f);
11542 %}
11543
11544 ins_pipe(ialu_reg_reg_shift);
11545 %}
11546
11547 // This pattern is automatically generated from aarch64_ad.m4
11548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11549 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11550 iRegIorL2I src1, iRegIorL2I src2,
11551 immI src3) %{
11552 match(Set dst (XorI src1 (URShiftI src2 src3)));
11553
11554 ins_cost(1.9 * INSN_COST);
11555 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11556
11557 ins_encode %{
11558 __ eorw(as_Register($dst$$reg),
11559 as_Register($src1$$reg),
11560 as_Register($src2$$reg),
11561 Assembler::LSR,
11562 $src3$$constant & 0x1f);
11563 %}
11564
11565 ins_pipe(ialu_reg_reg_shift);
11566 %}
11567
11568 // This pattern is automatically generated from aarch64_ad.m4
11569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11570 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11571 iRegL src1, iRegL src2,
11572 immI src3) %{
11573 match(Set dst (XorL src1 (URShiftL src2 src3)));
11574
11575 ins_cost(1.9 * INSN_COST);
11576 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11577
11578 ins_encode %{
11579 __ eor(as_Register($dst$$reg),
11580 as_Register($src1$$reg),
11581 as_Register($src2$$reg),
11582 Assembler::LSR,
11583 $src3$$constant & 0x3f);
11584 %}
11585
11586 ins_pipe(ialu_reg_reg_shift);
11587 %}
11588
11589 // This pattern is automatically generated from aarch64_ad.m4
11590 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11591 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11592 iRegIorL2I src1, iRegIorL2I src2,
11593 immI src3) %{
11594 match(Set dst (XorI src1 (RShiftI src2 src3)));
11595
11596 ins_cost(1.9 * INSN_COST);
11597 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11598
11599 ins_encode %{
11600 __ eorw(as_Register($dst$$reg),
11601 as_Register($src1$$reg),
11602 as_Register($src2$$reg),
11603 Assembler::ASR,
11604 $src3$$constant & 0x1f);
11605 %}
11606
11607 ins_pipe(ialu_reg_reg_shift);
11608 %}
11609
11610 // This pattern is automatically generated from aarch64_ad.m4
11611 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11612 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11613 iRegL src1, iRegL src2,
11614 immI src3) %{
11615 match(Set dst (XorL src1 (RShiftL src2 src3)));
11616
11617 ins_cost(1.9 * INSN_COST);
11618 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11619
11620 ins_encode %{
11621 __ eor(as_Register($dst$$reg),
11622 as_Register($src1$$reg),
11623 as_Register($src2$$reg),
11624 Assembler::ASR,
11625 $src3$$constant & 0x3f);
11626 %}
11627
11628 ins_pipe(ialu_reg_reg_shift);
11629 %}
11630
11631 // This pattern is automatically generated from aarch64_ad.m4
11632 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11633 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11634 iRegIorL2I src1, iRegIorL2I src2,
11635 immI src3) %{
11636 match(Set dst (XorI src1 (LShiftI src2 src3)));
11637
11638 ins_cost(1.9 * INSN_COST);
11639 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11640
11641 ins_encode %{
11642 __ eorw(as_Register($dst$$reg),
11643 as_Register($src1$$reg),
11644 as_Register($src2$$reg),
11645 Assembler::LSL,
11646 $src3$$constant & 0x1f);
11647 %}
11648
11649 ins_pipe(ialu_reg_reg_shift);
11650 %}
11651
11652 // This pattern is automatically generated from aarch64_ad.m4
11653 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11654 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11655 iRegL src1, iRegL src2,
11656 immI src3) %{
11657 match(Set dst (XorL src1 (LShiftL src2 src3)));
11658
11659 ins_cost(1.9 * INSN_COST);
11660 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11661
11662 ins_encode %{
11663 __ eor(as_Register($dst$$reg),
11664 as_Register($src1$$reg),
11665 as_Register($src2$$reg),
11666 Assembler::LSL,
11667 $src3$$constant & 0x3f);
11668 %}
11669
11670 ins_pipe(ialu_reg_reg_shift);
11671 %}
11672
11673 // This pattern is automatically generated from aarch64_ad.m4
11674 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11675 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11676 iRegIorL2I src1, iRegIorL2I src2,
11677 immI src3) %{
11678 match(Set dst (XorI src1 (RotateRight src2 src3)));
11679
11680 ins_cost(1.9 * INSN_COST);
11681 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11682
11683 ins_encode %{
11684 __ eorw(as_Register($dst$$reg),
11685 as_Register($src1$$reg),
11686 as_Register($src2$$reg),
11687 Assembler::ROR,
11688 $src3$$constant & 0x1f);
11689 %}
11690
11691 ins_pipe(ialu_reg_reg_shift);
11692 %}
11693
11694 // This pattern is automatically generated from aarch64_ad.m4
11695 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11696 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11697 iRegL src1, iRegL src2,
11698 immI src3) %{
11699 match(Set dst (XorL src1 (RotateRight src2 src3)));
11700
11701 ins_cost(1.9 * INSN_COST);
11702 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11703
11704 ins_encode %{
11705 __ eor(as_Register($dst$$reg),
11706 as_Register($src1$$reg),
11707 as_Register($src2$$reg),
11708 Assembler::ROR,
11709 $src3$$constant & 0x3f);
11710 %}
11711
11712 ins_pipe(ialu_reg_reg_shift);
11713 %}
11714
11715 // This pattern is automatically generated from aarch64_ad.m4
11716 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11717 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11718 iRegIorL2I src1, iRegIorL2I src2,
11719 immI src3) %{
11720 match(Set dst (OrI src1 (URShiftI src2 src3)));
11721
11722 ins_cost(1.9 * INSN_COST);
11723 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11724
11725 ins_encode %{
11726 __ orrw(as_Register($dst$$reg),
11727 as_Register($src1$$reg),
11728 as_Register($src2$$reg),
11729 Assembler::LSR,
11730 $src3$$constant & 0x1f);
11731 %}
11732
11733 ins_pipe(ialu_reg_reg_shift);
11734 %}
11735
11736 // This pattern is automatically generated from aarch64_ad.m4
11737 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11738 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11739 iRegL src1, iRegL src2,
11740 immI src3) %{
11741 match(Set dst (OrL src1 (URShiftL src2 src3)));
11742
11743 ins_cost(1.9 * INSN_COST);
11744 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11745
11746 ins_encode %{
11747 __ orr(as_Register($dst$$reg),
11748 as_Register($src1$$reg),
11749 as_Register($src2$$reg),
11750 Assembler::LSR,
11751 $src3$$constant & 0x3f);
11752 %}
11753
11754 ins_pipe(ialu_reg_reg_shift);
11755 %}
11756
11757 // This pattern is automatically generated from aarch64_ad.m4
11758 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11759 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11760 iRegIorL2I src1, iRegIorL2I src2,
11761 immI src3) %{
11762 match(Set dst (OrI src1 (RShiftI src2 src3)));
11763
11764 ins_cost(1.9 * INSN_COST);
11765 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11766
11767 ins_encode %{
11768 __ orrw(as_Register($dst$$reg),
11769 as_Register($src1$$reg),
11770 as_Register($src2$$reg),
11771 Assembler::ASR,
11772 $src3$$constant & 0x1f);
11773 %}
11774
11775 ins_pipe(ialu_reg_reg_shift);
11776 %}
11777
11778 // This pattern is automatically generated from aarch64_ad.m4
11779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11780 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11781 iRegL src1, iRegL src2,
11782 immI src3) %{
11783 match(Set dst (OrL src1 (RShiftL src2 src3)));
11784
11785 ins_cost(1.9 * INSN_COST);
11786 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11787
11788 ins_encode %{
11789 __ orr(as_Register($dst$$reg),
11790 as_Register($src1$$reg),
11791 as_Register($src2$$reg),
11792 Assembler::ASR,
11793 $src3$$constant & 0x3f);
11794 %}
11795
11796 ins_pipe(ialu_reg_reg_shift);
11797 %}
11798
11799 // This pattern is automatically generated from aarch64_ad.m4
11800 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11801 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11802 iRegIorL2I src1, iRegIorL2I src2,
11803 immI src3) %{
11804 match(Set dst (OrI src1 (LShiftI src2 src3)));
11805
11806 ins_cost(1.9 * INSN_COST);
11807 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11808
11809 ins_encode %{
11810 __ orrw(as_Register($dst$$reg),
11811 as_Register($src1$$reg),
11812 as_Register($src2$$reg),
11813 Assembler::LSL,
11814 $src3$$constant & 0x1f);
11815 %}
11816
11817 ins_pipe(ialu_reg_reg_shift);
11818 %}
11819
11820 // This pattern is automatically generated from aarch64_ad.m4
11821 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11822 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11823 iRegL src1, iRegL src2,
11824 immI src3) %{
11825 match(Set dst (OrL src1 (LShiftL src2 src3)));
11826
11827 ins_cost(1.9 * INSN_COST);
11828 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11829
11830 ins_encode %{
11831 __ orr(as_Register($dst$$reg),
11832 as_Register($src1$$reg),
11833 as_Register($src2$$reg),
11834 Assembler::LSL,
11835 $src3$$constant & 0x3f);
11836 %}
11837
11838 ins_pipe(ialu_reg_reg_shift);
11839 %}
11840
11841 // This pattern is automatically generated from aarch64_ad.m4
11842 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11843 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11844 iRegIorL2I src1, iRegIorL2I src2,
11845 immI src3) %{
11846 match(Set dst (OrI src1 (RotateRight src2 src3)));
11847
11848 ins_cost(1.9 * INSN_COST);
11849 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11850
11851 ins_encode %{
11852 __ orrw(as_Register($dst$$reg),
11853 as_Register($src1$$reg),
11854 as_Register($src2$$reg),
11855 Assembler::ROR,
11856 $src3$$constant & 0x1f);
11857 %}
11858
11859 ins_pipe(ialu_reg_reg_shift);
11860 %}
11861
11862 // This pattern is automatically generated from aarch64_ad.m4
11863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11864 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11865 iRegL src1, iRegL src2,
11866 immI src3) %{
11867 match(Set dst (OrL src1 (RotateRight src2 src3)));
11868
11869 ins_cost(1.9 * INSN_COST);
11870 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11871
11872 ins_encode %{
11873 __ orr(as_Register($dst$$reg),
11874 as_Register($src1$$reg),
11875 as_Register($src2$$reg),
11876 Assembler::ROR,
11877 $src3$$constant & 0x3f);
11878 %}
11879
11880 ins_pipe(ialu_reg_reg_shift);
11881 %}
11882
11883 // This pattern is automatically generated from aarch64_ad.m4
11884 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11885 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11886 iRegIorL2I src1, iRegIorL2I src2,
11887 immI src3) %{
11888 match(Set dst (AddI src1 (URShiftI src2 src3)));
11889
11890 ins_cost(1.9 * INSN_COST);
11891 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11892
11893 ins_encode %{
11894 __ addw(as_Register($dst$$reg),
11895 as_Register($src1$$reg),
11896 as_Register($src2$$reg),
11897 Assembler::LSR,
11898 $src3$$constant & 0x1f);
11899 %}
11900
11901 ins_pipe(ialu_reg_reg_shift);
11902 %}
11903
11904 // This pattern is automatically generated from aarch64_ad.m4
11905 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11906 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11907 iRegL src1, iRegL src2,
11908 immI src3) %{
11909 match(Set dst (AddL src1 (URShiftL src2 src3)));
11910
11911 ins_cost(1.9 * INSN_COST);
11912 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11913
11914 ins_encode %{
11915 __ add(as_Register($dst$$reg),
11916 as_Register($src1$$reg),
11917 as_Register($src2$$reg),
11918 Assembler::LSR,
11919 $src3$$constant & 0x3f);
11920 %}
11921
11922 ins_pipe(ialu_reg_reg_shift);
11923 %}
11924
11925 // This pattern is automatically generated from aarch64_ad.m4
11926 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11927 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11928 iRegIorL2I src1, iRegIorL2I src2,
11929 immI src3) %{
11930 match(Set dst (AddI src1 (RShiftI src2 src3)));
11931
11932 ins_cost(1.9 * INSN_COST);
11933 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11934
11935 ins_encode %{
11936 __ addw(as_Register($dst$$reg),
11937 as_Register($src1$$reg),
11938 as_Register($src2$$reg),
11939 Assembler::ASR,
11940 $src3$$constant & 0x1f);
11941 %}
11942
11943 ins_pipe(ialu_reg_reg_shift);
11944 %}
11945
11946 // This pattern is automatically generated from aarch64_ad.m4
11947 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11948 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11949 iRegL src1, iRegL src2,
11950 immI src3) %{
11951 match(Set dst (AddL src1 (RShiftL src2 src3)));
11952
11953 ins_cost(1.9 * INSN_COST);
11954 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11955
11956 ins_encode %{
11957 __ add(as_Register($dst$$reg),
11958 as_Register($src1$$reg),
11959 as_Register($src2$$reg),
11960 Assembler::ASR,
11961 $src3$$constant & 0x3f);
11962 %}
11963
11964 ins_pipe(ialu_reg_reg_shift);
11965 %}
11966
11967 // This pattern is automatically generated from aarch64_ad.m4
11968 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11969 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11970 iRegIorL2I src1, iRegIorL2I src2,
11971 immI src3) %{
11972 match(Set dst (AddI src1 (LShiftI src2 src3)));
11973
11974 ins_cost(1.9 * INSN_COST);
11975 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11976
11977 ins_encode %{
11978 __ addw(as_Register($dst$$reg),
11979 as_Register($src1$$reg),
11980 as_Register($src2$$reg),
11981 Assembler::LSL,
11982 $src3$$constant & 0x1f);
11983 %}
11984
11985 ins_pipe(ialu_reg_reg_shift);
11986 %}
11987
11988 // This pattern is automatically generated from aarch64_ad.m4
11989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11990 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11991 iRegL src1, iRegL src2,
11992 immI src3) %{
11993 match(Set dst (AddL src1 (LShiftL src2 src3)));
11994
11995 ins_cost(1.9 * INSN_COST);
11996 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11997
11998 ins_encode %{
11999 __ add(as_Register($dst$$reg),
12000 as_Register($src1$$reg),
12001 as_Register($src2$$reg),
12002 Assembler::LSL,
12003 $src3$$constant & 0x3f);
12004 %}
12005
12006 ins_pipe(ialu_reg_reg_shift);
12007 %}
12008
12009 // This pattern is automatically generated from aarch64_ad.m4
12010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12011 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12012 iRegIorL2I src1, iRegIorL2I src2,
12013 immI src3) %{
12014 match(Set dst (SubI src1 (URShiftI src2 src3)));
12015
12016 ins_cost(1.9 * INSN_COST);
12017 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12018
12019 ins_encode %{
12020 __ subw(as_Register($dst$$reg),
12021 as_Register($src1$$reg),
12022 as_Register($src2$$reg),
12023 Assembler::LSR,
12024 $src3$$constant & 0x1f);
12025 %}
12026
12027 ins_pipe(ialu_reg_reg_shift);
12028 %}
12029
12030 // This pattern is automatically generated from aarch64_ad.m4
12031 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12032 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12033 iRegL src1, iRegL src2,
12034 immI src3) %{
12035 match(Set dst (SubL src1 (URShiftL src2 src3)));
12036
12037 ins_cost(1.9 * INSN_COST);
12038 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12039
12040 ins_encode %{
12041 __ sub(as_Register($dst$$reg),
12042 as_Register($src1$$reg),
12043 as_Register($src2$$reg),
12044 Assembler::LSR,
12045 $src3$$constant & 0x3f);
12046 %}
12047
12048 ins_pipe(ialu_reg_reg_shift);
12049 %}
12050
12051 // This pattern is automatically generated from aarch64_ad.m4
12052 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12053 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12054 iRegIorL2I src1, iRegIorL2I src2,
12055 immI src3) %{
12056 match(Set dst (SubI src1 (RShiftI src2 src3)));
12057
12058 ins_cost(1.9 * INSN_COST);
12059 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12060
12061 ins_encode %{
12062 __ subw(as_Register($dst$$reg),
12063 as_Register($src1$$reg),
12064 as_Register($src2$$reg),
12065 Assembler::ASR,
12066 $src3$$constant & 0x1f);
12067 %}
12068
12069 ins_pipe(ialu_reg_reg_shift);
12070 %}
12071
12072 // This pattern is automatically generated from aarch64_ad.m4
12073 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12074 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12075 iRegL src1, iRegL src2,
12076 immI src3) %{
12077 match(Set dst (SubL src1 (RShiftL src2 src3)));
12078
12079 ins_cost(1.9 * INSN_COST);
12080 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12081
12082 ins_encode %{
12083 __ sub(as_Register($dst$$reg),
12084 as_Register($src1$$reg),
12085 as_Register($src2$$reg),
12086 Assembler::ASR,
12087 $src3$$constant & 0x3f);
12088 %}
12089
12090 ins_pipe(ialu_reg_reg_shift);
12091 %}
12092
12093 // This pattern is automatically generated from aarch64_ad.m4
12094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12095 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12096 iRegIorL2I src1, iRegIorL2I src2,
12097 immI src3) %{
12098 match(Set dst (SubI src1 (LShiftI src2 src3)));
12099
12100 ins_cost(1.9 * INSN_COST);
12101 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12102
12103 ins_encode %{
12104 __ subw(as_Register($dst$$reg),
12105 as_Register($src1$$reg),
12106 as_Register($src2$$reg),
12107 Assembler::LSL,
12108 $src3$$constant & 0x1f);
12109 %}
12110
12111 ins_pipe(ialu_reg_reg_shift);
12112 %}
12113
12114 // This pattern is automatically generated from aarch64_ad.m4
12115 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12116 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12117 iRegL src1, iRegL src2,
12118 immI src3) %{
12119 match(Set dst (SubL src1 (LShiftL src2 src3)));
12120
12121 ins_cost(1.9 * INSN_COST);
12122 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12123
12124 ins_encode %{
12125 __ sub(as_Register($dst$$reg),
12126 as_Register($src1$$reg),
12127 as_Register($src2$$reg),
12128 Assembler::LSL,
12129 $src3$$constant & 0x3f);
12130 %}
12131
12132 ins_pipe(ialu_reg_reg_shift);
12133 %}
12134
12135 // This pattern is automatically generated from aarch64_ad.m4
12136 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12137
12138 // Shift Left followed by Shift Right.
12139 // This idiom is used by the compiler for the i2b bytecode etc.
12140 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12141 %{
12142 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12143 ins_cost(INSN_COST * 2);
12144 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12145 ins_encode %{
12146 int lshift = $lshift_count$$constant & 63;
12147 int rshift = $rshift_count$$constant & 63;
12148 int s = 63 - lshift;
12149 int r = (rshift - lshift) & 63;
12150 __ sbfm(as_Register($dst$$reg),
12151 as_Register($src$$reg),
12152 r, s);
12153 %}
12154
12155 ins_pipe(ialu_reg_shift);
12156 %}
12157
12158 // This pattern is automatically generated from aarch64_ad.m4
12159 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12160
12161 // Shift Left followed by Shift Right.
12162 // This idiom is used by the compiler for the i2b bytecode etc.
12163 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12164 %{
12165 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12166 ins_cost(INSN_COST * 2);
12167 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12168 ins_encode %{
12169 int lshift = $lshift_count$$constant & 31;
12170 int rshift = $rshift_count$$constant & 31;
12171 int s = 31 - lshift;
12172 int r = (rshift - lshift) & 31;
12173 __ sbfmw(as_Register($dst$$reg),
12174 as_Register($src$$reg),
12175 r, s);
12176 %}
12177
12178 ins_pipe(ialu_reg_shift);
12179 %}
12180
12181 // This pattern is automatically generated from aarch64_ad.m4
12182 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12183
12184 // Shift Left followed by Shift Right.
12185 // This idiom is used by the compiler for the i2b bytecode etc.
12186 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12187 %{
12188 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12189 ins_cost(INSN_COST * 2);
12190 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12191 ins_encode %{
12192 int lshift = $lshift_count$$constant & 63;
12193 int rshift = $rshift_count$$constant & 63;
12194 int s = 63 - lshift;
12195 int r = (rshift - lshift) & 63;
12196 __ ubfm(as_Register($dst$$reg),
12197 as_Register($src$$reg),
12198 r, s);
12199 %}
12200
12201 ins_pipe(ialu_reg_shift);
12202 %}
12203
12204 // This pattern is automatically generated from aarch64_ad.m4
12205 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12206
12207 // Shift Left followed by Shift Right.
12208 // This idiom is used by the compiler for the i2b bytecode etc.
12209 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12210 %{
12211 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12212 ins_cost(INSN_COST * 2);
12213 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12214 ins_encode %{
12215 int lshift = $lshift_count$$constant & 31;
12216 int rshift = $rshift_count$$constant & 31;
12217 int s = 31 - lshift;
12218 int r = (rshift - lshift) & 31;
12219 __ ubfmw(as_Register($dst$$reg),
12220 as_Register($src$$reg),
12221 r, s);
12222 %}
12223
12224 ins_pipe(ialu_reg_shift);
12225 %}
12226
12227 // Bitfield extract with shift & mask
12228
12229 // This pattern is automatically generated from aarch64_ad.m4
12230 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12231 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12232 %{
12233 match(Set dst (AndI (URShiftI src rshift) mask));
12234 // Make sure we are not going to exceed what ubfxw can do.
12235 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12236
12237 ins_cost(INSN_COST);
12238 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12239 ins_encode %{
12240 int rshift = $rshift$$constant & 31;
12241 intptr_t mask = $mask$$constant;
12242 int width = exact_log2(mask+1);
12243 __ ubfxw(as_Register($dst$$reg),
12244 as_Register($src$$reg), rshift, width);
12245 %}
12246 ins_pipe(ialu_reg_shift);
12247 %}
12248
12249 // This pattern is automatically generated from aarch64_ad.m4
12250 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12251 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12252 %{
12253 match(Set dst (AndL (URShiftL src rshift) mask));
12254 // Make sure we are not going to exceed what ubfx can do.
12255 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12256
12257 ins_cost(INSN_COST);
12258 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12259 ins_encode %{
12260 int rshift = $rshift$$constant & 63;
12261 intptr_t mask = $mask$$constant;
12262 int width = exact_log2_long(mask+1);
12263 __ ubfx(as_Register($dst$$reg),
12264 as_Register($src$$reg), rshift, width);
12265 %}
12266 ins_pipe(ialu_reg_shift);
12267 %}
12268
12269
12270 // This pattern is automatically generated from aarch64_ad.m4
12271 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12272
12273 // We can use ubfx when extending an And with a mask when we know mask
12274 // is positive. We know that because immI_bitmask guarantees it.
12275 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12276 %{
12277 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12278 // Make sure we are not going to exceed what ubfxw can do.
12279 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12280
12281 ins_cost(INSN_COST * 2);
12282 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12283 ins_encode %{
12284 int rshift = $rshift$$constant & 31;
12285 intptr_t mask = $mask$$constant;
12286 int width = exact_log2(mask+1);
12287 __ ubfx(as_Register($dst$$reg),
12288 as_Register($src$$reg), rshift, width);
12289 %}
12290 ins_pipe(ialu_reg_shift);
12291 %}
12292
12293
12294 // This pattern is automatically generated from aarch64_ad.m4
12295 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12296
12297 // We can use ubfiz when masking by a positive number and then left shifting the result.
12298 // We know that the mask is positive because immI_bitmask guarantees it.
12299 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12300 %{
12301 match(Set dst (LShiftI (AndI src mask) lshift));
12302 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12303
12304 ins_cost(INSN_COST);
12305 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12306 ins_encode %{
12307 int lshift = $lshift$$constant & 31;
12308 intptr_t mask = $mask$$constant;
12309 int width = exact_log2(mask+1);
12310 __ ubfizw(as_Register($dst$$reg),
12311 as_Register($src$$reg), lshift, width);
12312 %}
12313 ins_pipe(ialu_reg_shift);
12314 %}
12315
12316 // This pattern is automatically generated from aarch64_ad.m4
12317 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12318
12319 // We can use ubfiz when masking by a positive number and then left shifting the result.
12320 // We know that the mask is positive because immL_bitmask guarantees it.
12321 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12322 %{
12323 match(Set dst (LShiftL (AndL src mask) lshift));
12324 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12325
12326 ins_cost(INSN_COST);
12327 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12328 ins_encode %{
12329 int lshift = $lshift$$constant & 63;
12330 intptr_t mask = $mask$$constant;
12331 int width = exact_log2_long(mask+1);
12332 __ ubfiz(as_Register($dst$$reg),
12333 as_Register($src$$reg), lshift, width);
12334 %}
12335 ins_pipe(ialu_reg_shift);
12336 %}
12337
12338 // This pattern is automatically generated from aarch64_ad.m4
12339 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12340
12341 // We can use ubfiz when masking by a positive number and then left shifting the result.
12342 // We know that the mask is positive because immI_bitmask guarantees it.
12343 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12344 %{
12345 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12346 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12347
12348 ins_cost(INSN_COST);
12349 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12350 ins_encode %{
12351 int lshift = $lshift$$constant & 31;
12352 intptr_t mask = $mask$$constant;
12353 int width = exact_log2(mask+1);
12354 __ ubfizw(as_Register($dst$$reg),
12355 as_Register($src$$reg), lshift, width);
12356 %}
12357 ins_pipe(ialu_reg_shift);
12358 %}
12359
12360 // This pattern is automatically generated from aarch64_ad.m4
12361 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12362
12363 // We can use ubfiz when masking by a positive number and then left shifting the result.
12364 // We know that the mask is positive because immL_bitmask guarantees it.
12365 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12366 %{
12367 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12368 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12369
12370 ins_cost(INSN_COST);
12371 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12372 ins_encode %{
12373 int lshift = $lshift$$constant & 63;
12374 intptr_t mask = $mask$$constant;
12375 int width = exact_log2_long(mask+1);
12376 __ ubfiz(as_Register($dst$$reg),
12377 as_Register($src$$reg), lshift, width);
12378 %}
12379 ins_pipe(ialu_reg_shift);
12380 %}
12381
12382
12383 // This pattern is automatically generated from aarch64_ad.m4
12384 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12385
12386 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12387 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12388 %{
12389 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12390 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12391
12392 ins_cost(INSN_COST);
12393 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12394 ins_encode %{
12395 int lshift = $lshift$$constant & 63;
12396 intptr_t mask = $mask$$constant;
12397 int width = exact_log2(mask+1);
12398 __ ubfiz(as_Register($dst$$reg),
12399 as_Register($src$$reg), lshift, width);
12400 %}
12401 ins_pipe(ialu_reg_shift);
12402 %}
12403
12404 // This pattern is automatically generated from aarch64_ad.m4
12405 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12406
12407 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12408 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12409 %{
12410 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12411 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12412
12413 ins_cost(INSN_COST);
12414 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12415 ins_encode %{
12416 int lshift = $lshift$$constant & 31;
12417 intptr_t mask = $mask$$constant;
12418 int width = exact_log2(mask+1);
12419 __ ubfiz(as_Register($dst$$reg),
12420 as_Register($src$$reg), lshift, width);
12421 %}
12422 ins_pipe(ialu_reg_shift);
12423 %}
12424
12425 // This pattern is automatically generated from aarch64_ad.m4
12426 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12427
12428 // Can skip int2long conversions after AND with small bitmask
12429 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12430 %{
12431 match(Set dst (ConvI2L (AndI src msk)));
12432 ins_cost(INSN_COST);
12433 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12434 ins_encode %{
12435 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12436 %}
12437 ins_pipe(ialu_reg_shift);
12438 %}
12439
12440
12441 // Rotations
12442
12443 // This pattern is automatically generated from aarch64_ad.m4
12444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12445 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12446 %{
12447 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12448 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12449
12450 ins_cost(INSN_COST);
12451 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12452
12453 ins_encode %{
12454 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12455 $rshift$$constant & 63);
12456 %}
12457 ins_pipe(ialu_reg_reg_extr);
12458 %}
12459
12460
12461 // This pattern is automatically generated from aarch64_ad.m4
12462 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12463 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12464 %{
12465 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12466 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12467
12468 ins_cost(INSN_COST);
12469 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12470
12471 ins_encode %{
12472 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12473 $rshift$$constant & 31);
12474 %}
12475 ins_pipe(ialu_reg_reg_extr);
12476 %}
12477
12478
12479 // This pattern is automatically generated from aarch64_ad.m4
12480 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12481 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12482 %{
12483 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12484 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12485
12486 ins_cost(INSN_COST);
12487 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12488
12489 ins_encode %{
12490 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12491 $rshift$$constant & 63);
12492 %}
12493 ins_pipe(ialu_reg_reg_extr);
12494 %}
12495
12496
12497 // This pattern is automatically generated from aarch64_ad.m4
12498 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12499 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12500 %{
12501 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12502 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12503
12504 ins_cost(INSN_COST);
12505 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12506
12507 ins_encode %{
12508 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12509 $rshift$$constant & 31);
12510 %}
12511 ins_pipe(ialu_reg_reg_extr);
12512 %}
12513
12514 // This pattern is automatically generated from aarch64_ad.m4
12515 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12516 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12517 %{
12518 match(Set dst (RotateRight src shift));
12519
12520 ins_cost(INSN_COST);
12521 format %{ "ror $dst, $src, $shift" %}
12522
12523 ins_encode %{
12524 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12525 $shift$$constant & 0x1f);
12526 %}
12527 ins_pipe(ialu_reg_reg_vshift);
12528 %}
12529
12530 // This pattern is automatically generated from aarch64_ad.m4
12531 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12532 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12533 %{
12534 match(Set dst (RotateRight src shift));
12535
12536 ins_cost(INSN_COST);
12537 format %{ "ror $dst, $src, $shift" %}
12538
12539 ins_encode %{
12540 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12541 $shift$$constant & 0x3f);
12542 %}
12543 ins_pipe(ialu_reg_reg_vshift);
12544 %}
12545
12546 // This pattern is automatically generated from aarch64_ad.m4
12547 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12548 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12549 %{
12550 match(Set dst (RotateRight src shift));
12551
12552 ins_cost(INSN_COST);
12553 format %{ "ror $dst, $src, $shift" %}
12554
12555 ins_encode %{
12556 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12557 %}
12558 ins_pipe(ialu_reg_reg_vshift);
12559 %}
12560
12561 // This pattern is automatically generated from aarch64_ad.m4
12562 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12563 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12564 %{
12565 match(Set dst (RotateRight src shift));
12566
12567 ins_cost(INSN_COST);
12568 format %{ "ror $dst, $src, $shift" %}
12569
12570 ins_encode %{
12571 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12572 %}
12573 ins_pipe(ialu_reg_reg_vshift);
12574 %}
12575
12576 // This pattern is automatically generated from aarch64_ad.m4
12577 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12578 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12579 %{
12580 match(Set dst (RotateLeft src shift));
12581
12582 ins_cost(INSN_COST);
12583 format %{ "rol $dst, $src, $shift" %}
12584
12585 ins_encode %{
12586 __ subw(rscratch1, zr, as_Register($shift$$reg));
12587 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12588 %}
12589 ins_pipe(ialu_reg_reg_vshift);
12590 %}
12591
12592 // This pattern is automatically generated from aarch64_ad.m4
12593 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12594 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12595 %{
12596 match(Set dst (RotateLeft src shift));
12597
12598 ins_cost(INSN_COST);
12599 format %{ "rol $dst, $src, $shift" %}
12600
12601 ins_encode %{
12602 __ subw(rscratch1, zr, as_Register($shift$$reg));
12603 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12604 %}
12605 ins_pipe(ialu_reg_reg_vshift);
12606 %}
12607
12608
12609 // Add/subtract (extended)
12610
12611 // This pattern is automatically generated from aarch64_ad.m4
12612 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12613 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12614 %{
12615 match(Set dst (AddL src1 (ConvI2L src2)));
12616 ins_cost(INSN_COST);
12617 format %{ "add $dst, $src1, $src2, sxtw" %}
12618
12619 ins_encode %{
12620 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12621 as_Register($src2$$reg), ext::sxtw);
12622 %}
12623 ins_pipe(ialu_reg_reg);
12624 %}
12625
12626 // This pattern is automatically generated from aarch64_ad.m4
12627 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12628 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12629 %{
12630 match(Set dst (SubL src1 (ConvI2L src2)));
12631 ins_cost(INSN_COST);
12632 format %{ "sub $dst, $src1, $src2, sxtw" %}
12633
12634 ins_encode %{
12635 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12636 as_Register($src2$$reg), ext::sxtw);
12637 %}
12638 ins_pipe(ialu_reg_reg);
12639 %}
12640
12641 // This pattern is automatically generated from aarch64_ad.m4
12642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12643 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12644 %{
12645 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12646 ins_cost(INSN_COST);
12647 format %{ "add $dst, $src1, $src2, sxth" %}
12648
12649 ins_encode %{
12650 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12651 as_Register($src2$$reg), ext::sxth);
12652 %}
12653 ins_pipe(ialu_reg_reg);
12654 %}
12655
12656 // This pattern is automatically generated from aarch64_ad.m4
12657 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12658 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12659 %{
12660 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12661 ins_cost(INSN_COST);
12662 format %{ "add $dst, $src1, $src2, sxtb" %}
12663
12664 ins_encode %{
12665 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12666 as_Register($src2$$reg), ext::sxtb);
12667 %}
12668 ins_pipe(ialu_reg_reg);
12669 %}
12670
12671 // This pattern is automatically generated from aarch64_ad.m4
12672 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12673 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12674 %{
12675 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12676 ins_cost(INSN_COST);
12677 format %{ "add $dst, $src1, $src2, uxtb" %}
12678
12679 ins_encode %{
12680 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12681 as_Register($src2$$reg), ext::uxtb);
12682 %}
12683 ins_pipe(ialu_reg_reg);
12684 %}
12685
12686 // This pattern is automatically generated from aarch64_ad.m4
12687 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12688 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12689 %{
12690 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12691 ins_cost(INSN_COST);
12692 format %{ "add $dst, $src1, $src2, sxth" %}
12693
12694 ins_encode %{
12695 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12696 as_Register($src2$$reg), ext::sxth);
12697 %}
12698 ins_pipe(ialu_reg_reg);
12699 %}
12700
12701 // This pattern is automatically generated from aarch64_ad.m4
12702 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12703 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12704 %{
12705 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12706 ins_cost(INSN_COST);
12707 format %{ "add $dst, $src1, $src2, sxtw" %}
12708
12709 ins_encode %{
12710 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12711 as_Register($src2$$reg), ext::sxtw);
12712 %}
12713 ins_pipe(ialu_reg_reg);
12714 %}
12715
12716 // This pattern is automatically generated from aarch64_ad.m4
12717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12718 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12719 %{
12720 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12721 ins_cost(INSN_COST);
12722 format %{ "add $dst, $src1, $src2, sxtb" %}
12723
12724 ins_encode %{
12725 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12726 as_Register($src2$$reg), ext::sxtb);
12727 %}
12728 ins_pipe(ialu_reg_reg);
12729 %}
12730
12731 // This pattern is automatically generated from aarch64_ad.m4
12732 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12733 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12734 %{
12735 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12736 ins_cost(INSN_COST);
12737 format %{ "add $dst, $src1, $src2, uxtb" %}
12738
12739 ins_encode %{
12740 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12741 as_Register($src2$$reg), ext::uxtb);
12742 %}
12743 ins_pipe(ialu_reg_reg);
12744 %}
12745
12746 // This pattern is automatically generated from aarch64_ad.m4
12747 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12748 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12749 %{
12750 match(Set dst (AddI src1 (AndI src2 mask)));
12751 ins_cost(INSN_COST);
12752 format %{ "addw $dst, $src1, $src2, uxtb" %}
12753
12754 ins_encode %{
12755 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12756 as_Register($src2$$reg), ext::uxtb);
12757 %}
12758 ins_pipe(ialu_reg_reg);
12759 %}
12760
12761 // This pattern is automatically generated from aarch64_ad.m4
12762 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12763 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12764 %{
12765 match(Set dst (AddI src1 (AndI src2 mask)));
12766 ins_cost(INSN_COST);
12767 format %{ "addw $dst, $src1, $src2, uxth" %}
12768
12769 ins_encode %{
12770 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12771 as_Register($src2$$reg), ext::uxth);
12772 %}
12773 ins_pipe(ialu_reg_reg);
12774 %}
12775
12776 // This pattern is automatically generated from aarch64_ad.m4
12777 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12778 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12779 %{
12780 match(Set dst (AddL src1 (AndL src2 mask)));
12781 ins_cost(INSN_COST);
12782 format %{ "add $dst, $src1, $src2, uxtb" %}
12783
12784 ins_encode %{
12785 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12786 as_Register($src2$$reg), ext::uxtb);
12787 %}
12788 ins_pipe(ialu_reg_reg);
12789 %}
12790
12791 // This pattern is automatically generated from aarch64_ad.m4
12792 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12793 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12794 %{
12795 match(Set dst (AddL src1 (AndL src2 mask)));
12796 ins_cost(INSN_COST);
12797 format %{ "add $dst, $src1, $src2, uxth" %}
12798
12799 ins_encode %{
12800 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12801 as_Register($src2$$reg), ext::uxth);
12802 %}
12803 ins_pipe(ialu_reg_reg);
12804 %}
12805
12806 // This pattern is automatically generated from aarch64_ad.m4
12807 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12808 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12809 %{
12810 match(Set dst (AddL src1 (AndL src2 mask)));
12811 ins_cost(INSN_COST);
12812 format %{ "add $dst, $src1, $src2, uxtw" %}
12813
12814 ins_encode %{
12815 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12816 as_Register($src2$$reg), ext::uxtw);
12817 %}
12818 ins_pipe(ialu_reg_reg);
12819 %}
12820
12821 // This pattern is automatically generated from aarch64_ad.m4
12822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12823 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12824 %{
12825 match(Set dst (SubI src1 (AndI src2 mask)));
12826 ins_cost(INSN_COST);
12827 format %{ "subw $dst, $src1, $src2, uxtb" %}
12828
12829 ins_encode %{
12830 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12831 as_Register($src2$$reg), ext::uxtb);
12832 %}
12833 ins_pipe(ialu_reg_reg);
12834 %}
12835
12836 // This pattern is automatically generated from aarch64_ad.m4
12837 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12838 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12839 %{
12840 match(Set dst (SubI src1 (AndI src2 mask)));
12841 ins_cost(INSN_COST);
12842 format %{ "subw $dst, $src1, $src2, uxth" %}
12843
12844 ins_encode %{
12845 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12846 as_Register($src2$$reg), ext::uxth);
12847 %}
12848 ins_pipe(ialu_reg_reg);
12849 %}
12850
12851 // This pattern is automatically generated from aarch64_ad.m4
12852 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12853 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12854 %{
12855 match(Set dst (SubL src1 (AndL src2 mask)));
12856 ins_cost(INSN_COST);
12857 format %{ "sub $dst, $src1, $src2, uxtb" %}
12858
12859 ins_encode %{
12860 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12861 as_Register($src2$$reg), ext::uxtb);
12862 %}
12863 ins_pipe(ialu_reg_reg);
12864 %}
12865
12866 // This pattern is automatically generated from aarch64_ad.m4
12867 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12868 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12869 %{
12870 match(Set dst (SubL src1 (AndL src2 mask)));
12871 ins_cost(INSN_COST);
12872 format %{ "sub $dst, $src1, $src2, uxth" %}
12873
12874 ins_encode %{
12875 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12876 as_Register($src2$$reg), ext::uxth);
12877 %}
12878 ins_pipe(ialu_reg_reg);
12879 %}
12880
12881 // This pattern is automatically generated from aarch64_ad.m4
12882 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12883 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12884 %{
12885 match(Set dst (SubL src1 (AndL src2 mask)));
12886 ins_cost(INSN_COST);
12887 format %{ "sub $dst, $src1, $src2, uxtw" %}
12888
12889 ins_encode %{
12890 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12891 as_Register($src2$$reg), ext::uxtw);
12892 %}
12893 ins_pipe(ialu_reg_reg);
12894 %}
12895
12896
12897 // This pattern is automatically generated from aarch64_ad.m4
12898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12899 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12900 %{
12901 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12902 ins_cost(1.9 * INSN_COST);
12903 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12904
12905 ins_encode %{
12906 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12907 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12908 %}
12909 ins_pipe(ialu_reg_reg_shift);
12910 %}
12911
12912 // This pattern is automatically generated from aarch64_ad.m4
12913 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12914 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12915 %{
12916 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12917 ins_cost(1.9 * INSN_COST);
12918 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12919
12920 ins_encode %{
12921 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12922 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12923 %}
12924 ins_pipe(ialu_reg_reg_shift);
12925 %}
12926
12927 // This pattern is automatically generated from aarch64_ad.m4
12928 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12929 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12930 %{
12931 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12932 ins_cost(1.9 * INSN_COST);
12933 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12934
12935 ins_encode %{
12936 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12937 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12938 %}
12939 ins_pipe(ialu_reg_reg_shift);
12940 %}
12941
12942 // This pattern is automatically generated from aarch64_ad.m4
12943 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12944 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12945 %{
12946 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12947 ins_cost(1.9 * INSN_COST);
12948 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12949
12950 ins_encode %{
12951 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12952 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12953 %}
12954 ins_pipe(ialu_reg_reg_shift);
12955 %}
12956
12957 // This pattern is automatically generated from aarch64_ad.m4
12958 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12959 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12960 %{
12961 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12962 ins_cost(1.9 * INSN_COST);
12963 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12964
12965 ins_encode %{
12966 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12967 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12968 %}
12969 ins_pipe(ialu_reg_reg_shift);
12970 %}
12971
12972 // This pattern is automatically generated from aarch64_ad.m4
12973 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12974 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12975 %{
12976 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12977 ins_cost(1.9 * INSN_COST);
12978 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12979
12980 ins_encode %{
12981 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12982 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12983 %}
12984 ins_pipe(ialu_reg_reg_shift);
12985 %}
12986
12987 // This pattern is automatically generated from aarch64_ad.m4
12988 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12989 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12990 %{
12991 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12992 ins_cost(1.9 * INSN_COST);
12993 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12994
12995 ins_encode %{
12996 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12997 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12998 %}
12999 ins_pipe(ialu_reg_reg_shift);
13000 %}
13001
13002 // This pattern is automatically generated from aarch64_ad.m4
13003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13004 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13005 %{
13006 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13007 ins_cost(1.9 * INSN_COST);
13008 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13009
13010 ins_encode %{
13011 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13012 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13013 %}
13014 ins_pipe(ialu_reg_reg_shift);
13015 %}
13016
13017 // This pattern is automatically generated from aarch64_ad.m4
13018 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13019 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13020 %{
13021 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13022 ins_cost(1.9 * INSN_COST);
13023 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13024
13025 ins_encode %{
13026 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13027 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13028 %}
13029 ins_pipe(ialu_reg_reg_shift);
13030 %}
13031
13032 // This pattern is automatically generated from aarch64_ad.m4
13033 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13034 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13035 %{
13036 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13037 ins_cost(1.9 * INSN_COST);
13038 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13039
13040 ins_encode %{
13041 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13042 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13043 %}
13044 ins_pipe(ialu_reg_reg_shift);
13045 %}
13046
13047 // This pattern is automatically generated from aarch64_ad.m4
13048 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13049 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13050 %{
13051 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13052 ins_cost(1.9 * INSN_COST);
13053 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13054
13055 ins_encode %{
13056 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13057 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13058 %}
13059 ins_pipe(ialu_reg_reg_shift);
13060 %}
13061
13062 // This pattern is automatically generated from aarch64_ad.m4
13063 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13064 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13065 %{
13066 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13067 ins_cost(1.9 * INSN_COST);
13068 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13069
13070 ins_encode %{
13071 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13072 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13073 %}
13074 ins_pipe(ialu_reg_reg_shift);
13075 %}
13076
13077 // This pattern is automatically generated from aarch64_ad.m4
13078 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13079 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13080 %{
13081 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13082 ins_cost(1.9 * INSN_COST);
13083 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13084
13085 ins_encode %{
13086 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13087 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13088 %}
13089 ins_pipe(ialu_reg_reg_shift);
13090 %}
13091
13092 // This pattern is automatically generated from aarch64_ad.m4
13093 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13094 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13095 %{
13096 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13097 ins_cost(1.9 * INSN_COST);
13098 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13099
13100 ins_encode %{
13101 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13102 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13103 %}
13104 ins_pipe(ialu_reg_reg_shift);
13105 %}
13106
13107 // This pattern is automatically generated from aarch64_ad.m4
13108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13109 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13110 %{
13111 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13112 ins_cost(1.9 * INSN_COST);
13113 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13114
13115 ins_encode %{
13116 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13117 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13118 %}
13119 ins_pipe(ialu_reg_reg_shift);
13120 %}
13121
13122 // This pattern is automatically generated from aarch64_ad.m4
13123 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13124 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13125 %{
13126 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13127 ins_cost(1.9 * INSN_COST);
13128 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13129
13130 ins_encode %{
13131 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13132 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13133 %}
13134 ins_pipe(ialu_reg_reg_shift);
13135 %}
13136
13137 // This pattern is automatically generated from aarch64_ad.m4
13138 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13139 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13140 %{
13141 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13142 ins_cost(1.9 * INSN_COST);
13143 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13144
13145 ins_encode %{
13146 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13147 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13148 %}
13149 ins_pipe(ialu_reg_reg_shift);
13150 %}
13151
13152 // This pattern is automatically generated from aarch64_ad.m4
13153 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13154 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13155 %{
13156 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13157 ins_cost(1.9 * INSN_COST);
13158 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13159
13160 ins_encode %{
13161 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13162 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13163 %}
13164 ins_pipe(ialu_reg_reg_shift);
13165 %}
13166
13167 // This pattern is automatically generated from aarch64_ad.m4
13168 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13169 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13170 %{
13171 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13172 ins_cost(1.9 * INSN_COST);
13173 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13174
13175 ins_encode %{
13176 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13177 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13178 %}
13179 ins_pipe(ialu_reg_reg_shift);
13180 %}
13181
13182 // This pattern is automatically generated from aarch64_ad.m4
13183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13184 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13185 %{
13186 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13187 ins_cost(1.9 * INSN_COST);
13188 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13189
13190 ins_encode %{
13191 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13192 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13193 %}
13194 ins_pipe(ialu_reg_reg_shift);
13195 %}
13196
13197 // This pattern is automatically generated from aarch64_ad.m4
13198 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13199 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13200 %{
13201 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13202 ins_cost(1.9 * INSN_COST);
13203 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13204
13205 ins_encode %{
13206 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13207 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13208 %}
13209 ins_pipe(ialu_reg_reg_shift);
13210 %}
13211
13212 // This pattern is automatically generated from aarch64_ad.m4
13213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13214 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13215 %{
13216 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13217 ins_cost(1.9 * INSN_COST);
13218 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13219
13220 ins_encode %{
13221 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13222 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13223 %}
13224 ins_pipe(ialu_reg_reg_shift);
13225 %}
13226
13227 // This pattern is automatically generated from aarch64_ad.m4
13228 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13229 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13230 %{
13231 effect(DEF dst, USE src1, USE src2, USE cr);
13232 ins_cost(INSN_COST * 2);
13233 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13234
13235 ins_encode %{
13236 __ cselw($dst$$Register,
13237 $src1$$Register,
13238 $src2$$Register,
13239 Assembler::LT);
13240 %}
13241 ins_pipe(icond_reg_reg);
13242 %}
13243
13244 // This pattern is automatically generated from aarch64_ad.m4
13245 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13246 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13247 %{
13248 effect(DEF dst, USE src1, USE src2, USE cr);
13249 ins_cost(INSN_COST * 2);
13250 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13251
13252 ins_encode %{
13253 __ cselw($dst$$Register,
13254 $src1$$Register,
13255 $src2$$Register,
13256 Assembler::GT);
13257 %}
13258 ins_pipe(icond_reg_reg);
13259 %}
13260
13261 // This pattern is automatically generated from aarch64_ad.m4
13262 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13263 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13264 %{
13265 effect(DEF dst, USE src1, USE cr);
13266 ins_cost(INSN_COST * 2);
13267 format %{ "cselw $dst, $src1, zr lt\t" %}
13268
13269 ins_encode %{
13270 __ cselw($dst$$Register,
13271 $src1$$Register,
13272 zr,
13273 Assembler::LT);
13274 %}
13275 ins_pipe(icond_reg);
13276 %}
13277
13278 // This pattern is automatically generated from aarch64_ad.m4
13279 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13280 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13281 %{
13282 effect(DEF dst, USE src1, USE cr);
13283 ins_cost(INSN_COST * 2);
13284 format %{ "cselw $dst, $src1, zr gt\t" %}
13285
13286 ins_encode %{
13287 __ cselw($dst$$Register,
13288 $src1$$Register,
13289 zr,
13290 Assembler::GT);
13291 %}
13292 ins_pipe(icond_reg);
13293 %}
13294
13295 // This pattern is automatically generated from aarch64_ad.m4
13296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13297 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13298 %{
13299 effect(DEF dst, USE src1, USE cr);
13300 ins_cost(INSN_COST * 2);
13301 format %{ "csincw $dst, $src1, zr le\t" %}
13302
13303 ins_encode %{
13304 __ csincw($dst$$Register,
13305 $src1$$Register,
13306 zr,
13307 Assembler::LE);
13308 %}
13309 ins_pipe(icond_reg);
13310 %}
13311
13312 // This pattern is automatically generated from aarch64_ad.m4
13313 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13314 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13315 %{
13316 effect(DEF dst, USE src1, USE cr);
13317 ins_cost(INSN_COST * 2);
13318 format %{ "csincw $dst, $src1, zr gt\t" %}
13319
13320 ins_encode %{
13321 __ csincw($dst$$Register,
13322 $src1$$Register,
13323 zr,
13324 Assembler::GT);
13325 %}
13326 ins_pipe(icond_reg);
13327 %}
13328
13329 // This pattern is automatically generated from aarch64_ad.m4
13330 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13331 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13332 %{
13333 effect(DEF dst, USE src1, USE cr);
13334 ins_cost(INSN_COST * 2);
13335 format %{ "csinvw $dst, $src1, zr lt\t" %}
13336
13337 ins_encode %{
13338 __ csinvw($dst$$Register,
13339 $src1$$Register,
13340 zr,
13341 Assembler::LT);
13342 %}
13343 ins_pipe(icond_reg);
13344 %}
13345
13346 // This pattern is automatically generated from aarch64_ad.m4
13347 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13348 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13349 %{
13350 effect(DEF dst, USE src1, USE cr);
13351 ins_cost(INSN_COST * 2);
13352 format %{ "csinvw $dst, $src1, zr ge\t" %}
13353
13354 ins_encode %{
13355 __ csinvw($dst$$Register,
13356 $src1$$Register,
13357 zr,
13358 Assembler::GE);
13359 %}
13360 ins_pipe(icond_reg);
13361 %}
13362
13363 // This pattern is automatically generated from aarch64_ad.m4
13364 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13365 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13366 %{
13367 match(Set dst (MinI src imm));
13368 ins_cost(INSN_COST * 3);
13369 expand %{
13370 rFlagsReg cr;
13371 compI_reg_imm0(cr, src);
13372 cmovI_reg_imm0_lt(dst, src, cr);
13373 %}
13374 %}
13375
13376 // This pattern is automatically generated from aarch64_ad.m4
13377 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13378 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13379 %{
13380 match(Set dst (MinI imm src));
13381 ins_cost(INSN_COST * 3);
13382 expand %{
13383 rFlagsReg cr;
13384 compI_reg_imm0(cr, src);
13385 cmovI_reg_imm0_lt(dst, src, cr);
13386 %}
13387 %}
13388
13389 // This pattern is automatically generated from aarch64_ad.m4
13390 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13391 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13392 %{
13393 match(Set dst (MinI src imm));
13394 ins_cost(INSN_COST * 3);
13395 expand %{
13396 rFlagsReg cr;
13397 compI_reg_imm0(cr, src);
13398 cmovI_reg_imm1_le(dst, src, cr);
13399 %}
13400 %}
13401
13402 // This pattern is automatically generated from aarch64_ad.m4
13403 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13404 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13405 %{
13406 match(Set dst (MinI imm src));
13407 ins_cost(INSN_COST * 3);
13408 expand %{
13409 rFlagsReg cr;
13410 compI_reg_imm0(cr, src);
13411 cmovI_reg_imm1_le(dst, src, cr);
13412 %}
13413 %}
13414
13415 // This pattern is automatically generated from aarch64_ad.m4
13416 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13417 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13418 %{
13419 match(Set dst (MinI src imm));
13420 ins_cost(INSN_COST * 3);
13421 expand %{
13422 rFlagsReg cr;
13423 compI_reg_imm0(cr, src);
13424 cmovI_reg_immM1_lt(dst, src, cr);
13425 %}
13426 %}
13427
13428 // This pattern is automatically generated from aarch64_ad.m4
13429 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13430 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13431 %{
13432 match(Set dst (MinI imm src));
13433 ins_cost(INSN_COST * 3);
13434 expand %{
13435 rFlagsReg cr;
13436 compI_reg_imm0(cr, src);
13437 cmovI_reg_immM1_lt(dst, src, cr);
13438 %}
13439 %}
13440
13441 // This pattern is automatically generated from aarch64_ad.m4
13442 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13443 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13444 %{
13445 match(Set dst (MaxI src imm));
13446 ins_cost(INSN_COST * 3);
13447 expand %{
13448 rFlagsReg cr;
13449 compI_reg_imm0(cr, src);
13450 cmovI_reg_imm0_gt(dst, src, cr);
13451 %}
13452 %}
13453
13454 // This pattern is automatically generated from aarch64_ad.m4
13455 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13456 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13457 %{
13458 match(Set dst (MaxI imm src));
13459 ins_cost(INSN_COST * 3);
13460 expand %{
13461 rFlagsReg cr;
13462 compI_reg_imm0(cr, src);
13463 cmovI_reg_imm0_gt(dst, src, cr);
13464 %}
13465 %}
13466
13467 // This pattern is automatically generated from aarch64_ad.m4
13468 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13469 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13470 %{
13471 match(Set dst (MaxI src imm));
13472 ins_cost(INSN_COST * 3);
13473 expand %{
13474 rFlagsReg cr;
13475 compI_reg_imm0(cr, src);
13476 cmovI_reg_imm1_gt(dst, src, cr);
13477 %}
13478 %}
13479
13480 // This pattern is automatically generated from aarch64_ad.m4
13481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13482 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13483 %{
13484 match(Set dst (MaxI imm src));
13485 ins_cost(INSN_COST * 3);
13486 expand %{
13487 rFlagsReg cr;
13488 compI_reg_imm0(cr, src);
13489 cmovI_reg_imm1_gt(dst, src, cr);
13490 %}
13491 %}
13492
13493 // This pattern is automatically generated from aarch64_ad.m4
13494 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13495 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13496 %{
13497 match(Set dst (MaxI src imm));
13498 ins_cost(INSN_COST * 3);
13499 expand %{
13500 rFlagsReg cr;
13501 compI_reg_imm0(cr, src);
13502 cmovI_reg_immM1_ge(dst, src, cr);
13503 %}
13504 %}
13505
13506 // This pattern is automatically generated from aarch64_ad.m4
13507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13508 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13509 %{
13510 match(Set dst (MaxI imm src));
13511 ins_cost(INSN_COST * 3);
13512 expand %{
13513 rFlagsReg cr;
13514 compI_reg_imm0(cr, src);
13515 cmovI_reg_immM1_ge(dst, src, cr);
13516 %}
13517 %}
13518
13519 // This pattern is automatically generated from aarch64_ad.m4
13520 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13521 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13522 %{
13523 match(Set dst (ReverseI src));
13524 ins_cost(INSN_COST);
13525 format %{ "rbitw $dst, $src" %}
13526 ins_encode %{
13527 __ rbitw($dst$$Register, $src$$Register);
13528 %}
13529 ins_pipe(ialu_reg);
13530 %}
13531
13532 // This pattern is automatically generated from aarch64_ad.m4
13533 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13534 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13535 %{
13536 match(Set dst (ReverseL src));
13537 ins_cost(INSN_COST);
13538 format %{ "rbit $dst, $src" %}
13539 ins_encode %{
13540 __ rbit($dst$$Register, $src$$Register);
13541 %}
13542 ins_pipe(ialu_reg);
13543 %}
13544
13545
13546 // END This section of the file is automatically generated. Do not edit --------------
13547
13548
13549 // ============================================================================
13550 // Floating Point Arithmetic Instructions
13551
13552 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13553 match(Set dst (AddF src1 src2));
13554
13555 ins_cost(INSN_COST * 5);
13556 format %{ "fadds $dst, $src1, $src2" %}
13557
13558 ins_encode %{
13559 __ fadds(as_FloatRegister($dst$$reg),
13560 as_FloatRegister($src1$$reg),
13561 as_FloatRegister($src2$$reg));
13562 %}
13563
13564 ins_pipe(fp_dop_reg_reg_s);
13565 %}
13566
13567 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13568 match(Set dst (AddD src1 src2));
13569
13570 ins_cost(INSN_COST * 5);
13571 format %{ "faddd $dst, $src1, $src2" %}
13572
13573 ins_encode %{
13574 __ faddd(as_FloatRegister($dst$$reg),
13575 as_FloatRegister($src1$$reg),
13576 as_FloatRegister($src2$$reg));
13577 %}
13578
13579 ins_pipe(fp_dop_reg_reg_d);
13580 %}
13581
13582 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13583 match(Set dst (SubF src1 src2));
13584
13585 ins_cost(INSN_COST * 5);
13586 format %{ "fsubs $dst, $src1, $src2" %}
13587
13588 ins_encode %{
13589 __ fsubs(as_FloatRegister($dst$$reg),
13590 as_FloatRegister($src1$$reg),
13591 as_FloatRegister($src2$$reg));
13592 %}
13593
13594 ins_pipe(fp_dop_reg_reg_s);
13595 %}
13596
13597 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13598 match(Set dst (SubD src1 src2));
13599
13600 ins_cost(INSN_COST * 5);
13601 format %{ "fsubd $dst, $src1, $src2" %}
13602
13603 ins_encode %{
13604 __ fsubd(as_FloatRegister($dst$$reg),
13605 as_FloatRegister($src1$$reg),
13606 as_FloatRegister($src2$$reg));
13607 %}
13608
13609 ins_pipe(fp_dop_reg_reg_d);
13610 %}
13611
13612 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13613 match(Set dst (MulF src1 src2));
13614
13615 ins_cost(INSN_COST * 6);
13616 format %{ "fmuls $dst, $src1, $src2" %}
13617
13618 ins_encode %{
13619 __ fmuls(as_FloatRegister($dst$$reg),
13620 as_FloatRegister($src1$$reg),
13621 as_FloatRegister($src2$$reg));
13622 %}
13623
13624 ins_pipe(fp_dop_reg_reg_s);
13625 %}
13626
13627 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13628 match(Set dst (MulD src1 src2));
13629
13630 ins_cost(INSN_COST * 6);
13631 format %{ "fmuld $dst, $src1, $src2" %}
13632
13633 ins_encode %{
13634 __ fmuld(as_FloatRegister($dst$$reg),
13635 as_FloatRegister($src1$$reg),
13636 as_FloatRegister($src2$$reg));
13637 %}
13638
13639 ins_pipe(fp_dop_reg_reg_d);
13640 %}
13641
13642 // src1 * src2 + src3
13643 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13644 match(Set dst (FmaF src3 (Binary src1 src2)));
13645
13646 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13647
13648 ins_encode %{
13649 assert(UseFMA, "Needs FMA instructions support.");
13650 __ fmadds(as_FloatRegister($dst$$reg),
13651 as_FloatRegister($src1$$reg),
13652 as_FloatRegister($src2$$reg),
13653 as_FloatRegister($src3$$reg));
13654 %}
13655
13656 ins_pipe(pipe_class_default);
13657 %}
13658
13659 // src1 * src2 + src3
13660 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13661 match(Set dst (FmaD src3 (Binary src1 src2)));
13662
13663 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13664
13665 ins_encode %{
13666 assert(UseFMA, "Needs FMA instructions support.");
13667 __ fmaddd(as_FloatRegister($dst$$reg),
13668 as_FloatRegister($src1$$reg),
13669 as_FloatRegister($src2$$reg),
13670 as_FloatRegister($src3$$reg));
13671 %}
13672
13673 ins_pipe(pipe_class_default);
13674 %}
13675
13676 // src1 * (-src2) + src3
13677 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13678 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13679 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13680
13681 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13682
13683 ins_encode %{
13684 assert(UseFMA, "Needs FMA instructions support.");
13685 __ fmsubs(as_FloatRegister($dst$$reg),
13686 as_FloatRegister($src1$$reg),
13687 as_FloatRegister($src2$$reg),
13688 as_FloatRegister($src3$$reg));
13689 %}
13690
13691 ins_pipe(pipe_class_default);
13692 %}
13693
13694 // src1 * (-src2) + src3
13695 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13696 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13697 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13698
13699 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13700
13701 ins_encode %{
13702 assert(UseFMA, "Needs FMA instructions support.");
13703 __ fmsubd(as_FloatRegister($dst$$reg),
13704 as_FloatRegister($src1$$reg),
13705 as_FloatRegister($src2$$reg),
13706 as_FloatRegister($src3$$reg));
13707 %}
13708
13709 ins_pipe(pipe_class_default);
13710 %}
13711
13712 // src1 * (-src2) - src3
13713 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13714 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13715 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13716
13717 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13718
13719 ins_encode %{
13720 assert(UseFMA, "Needs FMA instructions support.");
13721 __ fnmadds(as_FloatRegister($dst$$reg),
13722 as_FloatRegister($src1$$reg),
13723 as_FloatRegister($src2$$reg),
13724 as_FloatRegister($src3$$reg));
13725 %}
13726
13727 ins_pipe(pipe_class_default);
13728 %}
13729
13730 // src1 * (-src2) - src3
13731 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13732 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13733 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13734
13735 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13736
13737 ins_encode %{
13738 assert(UseFMA, "Needs FMA instructions support.");
13739 __ fnmaddd(as_FloatRegister($dst$$reg),
13740 as_FloatRegister($src1$$reg),
13741 as_FloatRegister($src2$$reg),
13742 as_FloatRegister($src3$$reg));
13743 %}
13744
13745 ins_pipe(pipe_class_default);
13746 %}
13747
13748 // src1 * src2 - src3
13749 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13750 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13751
13752 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13753
13754 ins_encode %{
13755 assert(UseFMA, "Needs FMA instructions support.");
13756 __ fnmsubs(as_FloatRegister($dst$$reg),
13757 as_FloatRegister($src1$$reg),
13758 as_FloatRegister($src2$$reg),
13759 as_FloatRegister($src3$$reg));
13760 %}
13761
13762 ins_pipe(pipe_class_default);
13763 %}
13764
13765 // src1 * src2 - src3
13766 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13767 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13768
13769 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13770
13771 ins_encode %{
13772 assert(UseFMA, "Needs FMA instructions support.");
13773 // n.b. insn name should be fnmsubd
13774 __ fnmsub(as_FloatRegister($dst$$reg),
13775 as_FloatRegister($src1$$reg),
13776 as_FloatRegister($src2$$reg),
13777 as_FloatRegister($src3$$reg));
13778 %}
13779
13780 ins_pipe(pipe_class_default);
13781 %}
13782
13783
13784 // Math.max(FF)F
13785 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13786 match(Set dst (MaxF src1 src2));
13787
13788 format %{ "fmaxs $dst, $src1, $src2" %}
13789 ins_encode %{
13790 __ fmaxs(as_FloatRegister($dst$$reg),
13791 as_FloatRegister($src1$$reg),
13792 as_FloatRegister($src2$$reg));
13793 %}
13794
13795 ins_pipe(fp_dop_reg_reg_s);
13796 %}
13797
13798 // Math.min(FF)F
13799 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13800 match(Set dst (MinF src1 src2));
13801
13802 format %{ "fmins $dst, $src1, $src2" %}
13803 ins_encode %{
13804 __ fmins(as_FloatRegister($dst$$reg),
13805 as_FloatRegister($src1$$reg),
13806 as_FloatRegister($src2$$reg));
13807 %}
13808
13809 ins_pipe(fp_dop_reg_reg_s);
13810 %}
13811
13812 // Math.max(DD)D
13813 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13814 match(Set dst (MaxD src1 src2));
13815
13816 format %{ "fmaxd $dst, $src1, $src2" %}
13817 ins_encode %{
13818 __ fmaxd(as_FloatRegister($dst$$reg),
13819 as_FloatRegister($src1$$reg),
13820 as_FloatRegister($src2$$reg));
13821 %}
13822
13823 ins_pipe(fp_dop_reg_reg_d);
13824 %}
13825
13826 // Math.min(DD)D
13827 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13828 match(Set dst (MinD src1 src2));
13829
13830 format %{ "fmind $dst, $src1, $src2" %}
13831 ins_encode %{
13832 __ fmind(as_FloatRegister($dst$$reg),
13833 as_FloatRegister($src1$$reg),
13834 as_FloatRegister($src2$$reg));
13835 %}
13836
13837 ins_pipe(fp_dop_reg_reg_d);
13838 %}
13839
13840
13841 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13842 match(Set dst (DivF src1 src2));
13843
13844 ins_cost(INSN_COST * 18);
13845 format %{ "fdivs $dst, $src1, $src2" %}
13846
13847 ins_encode %{
13848 __ fdivs(as_FloatRegister($dst$$reg),
13849 as_FloatRegister($src1$$reg),
13850 as_FloatRegister($src2$$reg));
13851 %}
13852
13853 ins_pipe(fp_div_s);
13854 %}
13855
13856 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13857 match(Set dst (DivD src1 src2));
13858
13859 ins_cost(INSN_COST * 32);
13860 format %{ "fdivd $dst, $src1, $src2" %}
13861
13862 ins_encode %{
13863 __ fdivd(as_FloatRegister($dst$$reg),
13864 as_FloatRegister($src1$$reg),
13865 as_FloatRegister($src2$$reg));
13866 %}
13867
13868 ins_pipe(fp_div_d);
13869 %}
13870
13871 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13872 match(Set dst (NegF src));
13873
13874 ins_cost(INSN_COST * 3);
13875 format %{ "fneg $dst, $src" %}
13876
13877 ins_encode %{
13878 __ fnegs(as_FloatRegister($dst$$reg),
13879 as_FloatRegister($src$$reg));
13880 %}
13881
13882 ins_pipe(fp_uop_s);
13883 %}
13884
13885 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13886 match(Set dst (NegD src));
13887
13888 ins_cost(INSN_COST * 3);
13889 format %{ "fnegd $dst, $src" %}
13890
13891 ins_encode %{
13892 __ fnegd(as_FloatRegister($dst$$reg),
13893 as_FloatRegister($src$$reg));
13894 %}
13895
13896 ins_pipe(fp_uop_d);
13897 %}
13898
13899 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13900 %{
13901 match(Set dst (AbsI src));
13902
13903 effect(KILL cr);
13904 ins_cost(INSN_COST * 2);
13905 format %{ "cmpw $src, zr\n\t"
13906 "cnegw $dst, $src, Assembler::LT\t# int abs"
13907 %}
13908
13909 ins_encode %{
13910 __ cmpw(as_Register($src$$reg), zr);
13911 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13912 %}
13913 ins_pipe(pipe_class_default);
13914 %}
13915
13916 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13917 %{
13918 match(Set dst (AbsL src));
13919
13920 effect(KILL cr);
13921 ins_cost(INSN_COST * 2);
13922 format %{ "cmp $src, zr\n\t"
13923 "cneg $dst, $src, Assembler::LT\t# long abs"
13924 %}
13925
13926 ins_encode %{
13927 __ cmp(as_Register($src$$reg), zr);
13928 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13929 %}
13930 ins_pipe(pipe_class_default);
13931 %}
13932
13933 instruct absF_reg(vRegF dst, vRegF src) %{
13934 match(Set dst (AbsF src));
13935
13936 ins_cost(INSN_COST * 3);
13937 format %{ "fabss $dst, $src" %}
13938 ins_encode %{
13939 __ fabss(as_FloatRegister($dst$$reg),
13940 as_FloatRegister($src$$reg));
13941 %}
13942
13943 ins_pipe(fp_uop_s);
13944 %}
13945
13946 instruct absD_reg(vRegD dst, vRegD src) %{
13947 match(Set dst (AbsD src));
13948
13949 ins_cost(INSN_COST * 3);
13950 format %{ "fabsd $dst, $src" %}
13951 ins_encode %{
13952 __ fabsd(as_FloatRegister($dst$$reg),
13953 as_FloatRegister($src$$reg));
13954 %}
13955
13956 ins_pipe(fp_uop_d);
13957 %}
13958
13959 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13960 match(Set dst (AbsF (SubF src1 src2)));
13961
13962 ins_cost(INSN_COST * 3);
13963 format %{ "fabds $dst, $src1, $src2" %}
13964 ins_encode %{
13965 __ fabds(as_FloatRegister($dst$$reg),
13966 as_FloatRegister($src1$$reg),
13967 as_FloatRegister($src2$$reg));
13968 %}
13969
13970 ins_pipe(fp_uop_s);
13971 %}
13972
13973 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13974 match(Set dst (AbsD (SubD src1 src2)));
13975
13976 ins_cost(INSN_COST * 3);
13977 format %{ "fabdd $dst, $src1, $src2" %}
13978 ins_encode %{
13979 __ fabdd(as_FloatRegister($dst$$reg),
13980 as_FloatRegister($src1$$reg),
13981 as_FloatRegister($src2$$reg));
13982 %}
13983
13984 ins_pipe(fp_uop_d);
13985 %}
13986
13987 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13988 match(Set dst (SqrtD src));
13989
13990 ins_cost(INSN_COST * 50);
13991 format %{ "fsqrtd $dst, $src" %}
13992 ins_encode %{
13993 __ fsqrtd(as_FloatRegister($dst$$reg),
13994 as_FloatRegister($src$$reg));
13995 %}
13996
13997 ins_pipe(fp_div_s);
13998 %}
13999
14000 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14001 match(Set dst (SqrtF src));
14002
14003 ins_cost(INSN_COST * 50);
14004 format %{ "fsqrts $dst, $src" %}
14005 ins_encode %{
14006 __ fsqrts(as_FloatRegister($dst$$reg),
14007 as_FloatRegister($src$$reg));
14008 %}
14009
14010 ins_pipe(fp_div_d);
14011 %}
14012
14013 // Math.rint, floor, ceil
14014 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14015 match(Set dst (RoundDoubleMode src rmode));
14016 format %{ "frint $dst, $src, $rmode" %}
14017 ins_encode %{
14018 switch ($rmode$$constant) {
14019 case RoundDoubleModeNode::rmode_rint:
14020 __ frintnd(as_FloatRegister($dst$$reg),
14021 as_FloatRegister($src$$reg));
14022 break;
14023 case RoundDoubleModeNode::rmode_floor:
14024 __ frintmd(as_FloatRegister($dst$$reg),
14025 as_FloatRegister($src$$reg));
14026 break;
14027 case RoundDoubleModeNode::rmode_ceil:
14028 __ frintpd(as_FloatRegister($dst$$reg),
14029 as_FloatRegister($src$$reg));
14030 break;
14031 }
14032 %}
14033 ins_pipe(fp_uop_d);
14034 %}
14035
14036 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14037 match(Set dst (CopySignD src1 (Binary src2 zero)));
14038 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14039 format %{ "CopySignD $dst $src1 $src2" %}
14040 ins_encode %{
14041 FloatRegister dst = as_FloatRegister($dst$$reg),
14042 src1 = as_FloatRegister($src1$$reg),
14043 src2 = as_FloatRegister($src2$$reg),
14044 zero = as_FloatRegister($zero$$reg);
14045 __ fnegd(dst, zero);
14046 __ bsl(dst, __ T8B, src2, src1);
14047 %}
14048 ins_pipe(fp_uop_d);
14049 %}
14050
14051 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14052 match(Set dst (CopySignF src1 src2));
14053 effect(TEMP_DEF dst, USE src1, USE src2);
14054 format %{ "CopySignF $dst $src1 $src2" %}
14055 ins_encode %{
14056 FloatRegister dst = as_FloatRegister($dst$$reg),
14057 src1 = as_FloatRegister($src1$$reg),
14058 src2 = as_FloatRegister($src2$$reg);
14059 __ movi(dst, __ T2S, 0x80, 24);
14060 __ bsl(dst, __ T8B, src2, src1);
14061 %}
14062 ins_pipe(fp_uop_d);
14063 %}
14064
14065 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14066 match(Set dst (SignumD src (Binary zero one)));
14067 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14068 format %{ "signumD $dst, $src" %}
14069 ins_encode %{
14070 FloatRegister src = as_FloatRegister($src$$reg),
14071 dst = as_FloatRegister($dst$$reg),
14072 zero = as_FloatRegister($zero$$reg),
14073 one = as_FloatRegister($one$$reg);
14074 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14075 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14076 // Bit selection instruction gets bit from "one" for each enabled bit in
14077 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14078 // NaN the whole "src" will be copied because "dst" is zero. For all other
14079 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14080 // from "src", and all other bits are copied from 1.0.
14081 __ bsl(dst, __ T8B, one, src);
14082 %}
14083 ins_pipe(fp_uop_d);
14084 %}
14085
14086 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14087 match(Set dst (SignumF src (Binary zero one)));
14088 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14089 format %{ "signumF $dst, $src" %}
14090 ins_encode %{
14091 FloatRegister src = as_FloatRegister($src$$reg),
14092 dst = as_FloatRegister($dst$$reg),
14093 zero = as_FloatRegister($zero$$reg),
14094 one = as_FloatRegister($one$$reg);
14095 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14096 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14097 // Bit selection instruction gets bit from "one" for each enabled bit in
14098 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14099 // NaN the whole "src" will be copied because "dst" is zero. For all other
14100 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14101 // from "src", and all other bits are copied from 1.0.
14102 __ bsl(dst, __ T8B, one, src);
14103 %}
14104 ins_pipe(fp_uop_d);
14105 %}
14106
14107 instruct onspinwait() %{
14108 match(OnSpinWait);
14109 ins_cost(INSN_COST);
14110
14111 format %{ "onspinwait" %}
14112
14113 ins_encode %{
14114 __ spin_wait();
14115 %}
14116 ins_pipe(pipe_class_empty);
14117 %}
14118
14119 // ============================================================================
14120 // Logical Instructions
14121
14122 // Integer Logical Instructions
14123
14124 // And Instructions
14125
14126
14127 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14128 match(Set dst (AndI src1 src2));
14129
14130 format %{ "andw $dst, $src1, $src2\t# int" %}
14131
14132 ins_cost(INSN_COST);
14133 ins_encode %{
14134 __ andw(as_Register($dst$$reg),
14135 as_Register($src1$$reg),
14136 as_Register($src2$$reg));
14137 %}
14138
14139 ins_pipe(ialu_reg_reg);
14140 %}
14141
14142 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14143 match(Set dst (AndI src1 src2));
14144
14145 format %{ "andsw $dst, $src1, $src2\t# int" %}
14146
14147 ins_cost(INSN_COST);
14148 ins_encode %{
14149 __ andw(as_Register($dst$$reg),
14150 as_Register($src1$$reg),
14151 (uint64_t)($src2$$constant));
14152 %}
14153
14154 ins_pipe(ialu_reg_imm);
14155 %}
14156
14157 // Or Instructions
14158
14159 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14160 match(Set dst (OrI src1 src2));
14161
14162 format %{ "orrw $dst, $src1, $src2\t# int" %}
14163
14164 ins_cost(INSN_COST);
14165 ins_encode %{
14166 __ orrw(as_Register($dst$$reg),
14167 as_Register($src1$$reg),
14168 as_Register($src2$$reg));
14169 %}
14170
14171 ins_pipe(ialu_reg_reg);
14172 %}
14173
14174 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14175 match(Set dst (OrI src1 src2));
14176
14177 format %{ "orrw $dst, $src1, $src2\t# int" %}
14178
14179 ins_cost(INSN_COST);
14180 ins_encode %{
14181 __ orrw(as_Register($dst$$reg),
14182 as_Register($src1$$reg),
14183 (uint64_t)($src2$$constant));
14184 %}
14185
14186 ins_pipe(ialu_reg_imm);
14187 %}
14188
14189 // Xor Instructions
14190
14191 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14192 match(Set dst (XorI src1 src2));
14193
14194 format %{ "eorw $dst, $src1, $src2\t# int" %}
14195
14196 ins_cost(INSN_COST);
14197 ins_encode %{
14198 __ eorw(as_Register($dst$$reg),
14199 as_Register($src1$$reg),
14200 as_Register($src2$$reg));
14201 %}
14202
14203 ins_pipe(ialu_reg_reg);
14204 %}
14205
14206 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14207 match(Set dst (XorI src1 src2));
14208
14209 format %{ "eorw $dst, $src1, $src2\t# int" %}
14210
14211 ins_cost(INSN_COST);
14212 ins_encode %{
14213 __ eorw(as_Register($dst$$reg),
14214 as_Register($src1$$reg),
14215 (uint64_t)($src2$$constant));
14216 %}
14217
14218 ins_pipe(ialu_reg_imm);
14219 %}
14220
14221 // Long Logical Instructions
14222 // TODO
14223
14224 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14225 match(Set dst (AndL src1 src2));
14226
14227 format %{ "and $dst, $src1, $src2\t# int" %}
14228
14229 ins_cost(INSN_COST);
14230 ins_encode %{
14231 __ andr(as_Register($dst$$reg),
14232 as_Register($src1$$reg),
14233 as_Register($src2$$reg));
14234 %}
14235
14236 ins_pipe(ialu_reg_reg);
14237 %}
14238
14239 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14240 match(Set dst (AndL src1 src2));
14241
14242 format %{ "and $dst, $src1, $src2\t# int" %}
14243
14244 ins_cost(INSN_COST);
14245 ins_encode %{
14246 __ andr(as_Register($dst$$reg),
14247 as_Register($src1$$reg),
14248 (uint64_t)($src2$$constant));
14249 %}
14250
14251 ins_pipe(ialu_reg_imm);
14252 %}
14253
14254 // Or Instructions
14255
14256 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14257 match(Set dst (OrL src1 src2));
14258
14259 format %{ "orr $dst, $src1, $src2\t# int" %}
14260
14261 ins_cost(INSN_COST);
14262 ins_encode %{
14263 __ orr(as_Register($dst$$reg),
14264 as_Register($src1$$reg),
14265 as_Register($src2$$reg));
14266 %}
14267
14268 ins_pipe(ialu_reg_reg);
14269 %}
14270
14271 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14272 match(Set dst (OrL src1 src2));
14273
14274 format %{ "orr $dst, $src1, $src2\t# int" %}
14275
14276 ins_cost(INSN_COST);
14277 ins_encode %{
14278 __ orr(as_Register($dst$$reg),
14279 as_Register($src1$$reg),
14280 (uint64_t)($src2$$constant));
14281 %}
14282
14283 ins_pipe(ialu_reg_imm);
14284 %}
14285
14286 // Xor Instructions
14287
14288 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14289 match(Set dst (XorL src1 src2));
14290
14291 format %{ "eor $dst, $src1, $src2\t# int" %}
14292
14293 ins_cost(INSN_COST);
14294 ins_encode %{
14295 __ eor(as_Register($dst$$reg),
14296 as_Register($src1$$reg),
14297 as_Register($src2$$reg));
14298 %}
14299
14300 ins_pipe(ialu_reg_reg);
14301 %}
14302
14303 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14304 match(Set dst (XorL src1 src2));
14305
14306 ins_cost(INSN_COST);
14307 format %{ "eor $dst, $src1, $src2\t# int" %}
14308
14309 ins_encode %{
14310 __ eor(as_Register($dst$$reg),
14311 as_Register($src1$$reg),
14312 (uint64_t)($src2$$constant));
14313 %}
14314
14315 ins_pipe(ialu_reg_imm);
14316 %}
14317
14318 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14319 %{
14320 match(Set dst (ConvI2L src));
14321
14322 ins_cost(INSN_COST);
14323 format %{ "sxtw $dst, $src\t# i2l" %}
14324 ins_encode %{
14325 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14326 %}
14327 ins_pipe(ialu_reg_shift);
14328 %}
14329
14330 // this pattern occurs in bigmath arithmetic
14331 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14332 %{
14333 match(Set dst (AndL (ConvI2L src) mask));
14334
14335 ins_cost(INSN_COST);
14336 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14337 ins_encode %{
14338 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14339 %}
14340
14341 ins_pipe(ialu_reg_shift);
14342 %}
14343
14344 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14345 match(Set dst (ConvL2I src));
14346
14347 ins_cost(INSN_COST);
14348 format %{ "movw $dst, $src \t// l2i" %}
14349
14350 ins_encode %{
14351 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14352 %}
14353
14354 ins_pipe(ialu_reg);
14355 %}
14356
14357 instruct convD2F_reg(vRegF dst, vRegD src) %{
14358 match(Set dst (ConvD2F src));
14359
14360 ins_cost(INSN_COST * 5);
14361 format %{ "fcvtd $dst, $src \t// d2f" %}
14362
14363 ins_encode %{
14364 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14365 %}
14366
14367 ins_pipe(fp_d2f);
14368 %}
14369
14370 instruct convF2D_reg(vRegD dst, vRegF src) %{
14371 match(Set dst (ConvF2D src));
14372
14373 ins_cost(INSN_COST * 5);
14374 format %{ "fcvts $dst, $src \t// f2d" %}
14375
14376 ins_encode %{
14377 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14378 %}
14379
14380 ins_pipe(fp_f2d);
14381 %}
14382
14383 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14384 match(Set dst (ConvF2I src));
14385
14386 ins_cost(INSN_COST * 5);
14387 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14388
14389 ins_encode %{
14390 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14391 %}
14392
14393 ins_pipe(fp_f2i);
14394 %}
14395
14396 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14397 match(Set dst (ConvF2L src));
14398
14399 ins_cost(INSN_COST * 5);
14400 format %{ "fcvtzs $dst, $src \t// f2l" %}
14401
14402 ins_encode %{
14403 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14404 %}
14405
14406 ins_pipe(fp_f2l);
14407 %}
14408
14409 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14410 match(Set dst (ConvF2HF src));
14411 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14412 "smov $dst, $tmp\t# move result from $tmp to $dst"
14413 %}
14414 effect(TEMP tmp);
14415 ins_encode %{
14416 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14417 %}
14418 ins_pipe(pipe_slow);
14419 %}
14420
14421 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14422 match(Set dst (ConvHF2F src));
14423 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14424 "fcvt $dst, $tmp\t# convert half to single precision"
14425 %}
14426 effect(TEMP tmp);
14427 ins_encode %{
14428 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14429 %}
14430 ins_pipe(pipe_slow);
14431 %}
14432
14433 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14434 match(Set dst (ConvI2F src));
14435
14436 ins_cost(INSN_COST * 5);
14437 format %{ "scvtfws $dst, $src \t// i2f" %}
14438
14439 ins_encode %{
14440 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14441 %}
14442
14443 ins_pipe(fp_i2f);
14444 %}
14445
14446 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14447 match(Set dst (ConvL2F src));
14448
14449 ins_cost(INSN_COST * 5);
14450 format %{ "scvtfs $dst, $src \t// l2f" %}
14451
14452 ins_encode %{
14453 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14454 %}
14455
14456 ins_pipe(fp_l2f);
14457 %}
14458
14459 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14460 match(Set dst (ConvD2I src));
14461
14462 ins_cost(INSN_COST * 5);
14463 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14464
14465 ins_encode %{
14466 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14467 %}
14468
14469 ins_pipe(fp_d2i);
14470 %}
14471
14472 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14473 match(Set dst (ConvD2L src));
14474
14475 ins_cost(INSN_COST * 5);
14476 format %{ "fcvtzd $dst, $src \t// d2l" %}
14477
14478 ins_encode %{
14479 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14480 %}
14481
14482 ins_pipe(fp_d2l);
14483 %}
14484
14485 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14486 match(Set dst (ConvI2D src));
14487
14488 ins_cost(INSN_COST * 5);
14489 format %{ "scvtfwd $dst, $src \t// i2d" %}
14490
14491 ins_encode %{
14492 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14493 %}
14494
14495 ins_pipe(fp_i2d);
14496 %}
14497
14498 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14499 match(Set dst (ConvL2D src));
14500
14501 ins_cost(INSN_COST * 5);
14502 format %{ "scvtfd $dst, $src \t// l2d" %}
14503
14504 ins_encode %{
14505 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14506 %}
14507
14508 ins_pipe(fp_l2d);
14509 %}
14510
14511 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14512 %{
14513 match(Set dst (RoundD src));
14514 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14515 format %{ "java_round_double $dst,$src"%}
14516 ins_encode %{
14517 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14518 as_FloatRegister($ftmp$$reg));
14519 %}
14520 ins_pipe(pipe_slow);
14521 %}
14522
14523 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14524 %{
14525 match(Set dst (RoundF src));
14526 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14527 format %{ "java_round_float $dst,$src"%}
14528 ins_encode %{
14529 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14530 as_FloatRegister($ftmp$$reg));
14531 %}
14532 ins_pipe(pipe_slow);
14533 %}
14534
14535 // stack <-> reg and reg <-> reg shuffles with no conversion
14536
14537 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14538
14539 match(Set dst (MoveF2I src));
14540
14541 effect(DEF dst, USE src);
14542
14543 ins_cost(4 * INSN_COST);
14544
14545 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14546
14547 ins_encode %{
14548 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14549 %}
14550
14551 ins_pipe(iload_reg_reg);
14552
14553 %}
14554
14555 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14556
14557 match(Set dst (MoveI2F src));
14558
14559 effect(DEF dst, USE src);
14560
14561 ins_cost(4 * INSN_COST);
14562
14563 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14564
14565 ins_encode %{
14566 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14567 %}
14568
14569 ins_pipe(pipe_class_memory);
14570
14571 %}
14572
14573 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14574
14575 match(Set dst (MoveD2L src));
14576
14577 effect(DEF dst, USE src);
14578
14579 ins_cost(4 * INSN_COST);
14580
14581 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14582
14583 ins_encode %{
14584 __ ldr($dst$$Register, Address(sp, $src$$disp));
14585 %}
14586
14587 ins_pipe(iload_reg_reg);
14588
14589 %}
14590
14591 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14592
14593 match(Set dst (MoveL2D src));
14594
14595 effect(DEF dst, USE src);
14596
14597 ins_cost(4 * INSN_COST);
14598
14599 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14600
14601 ins_encode %{
14602 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14603 %}
14604
14605 ins_pipe(pipe_class_memory);
14606
14607 %}
14608
14609 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14610
14611 match(Set dst (MoveF2I src));
14612
14613 effect(DEF dst, USE src);
14614
14615 ins_cost(INSN_COST);
14616
14617 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14618
14619 ins_encode %{
14620 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14621 %}
14622
14623 ins_pipe(pipe_class_memory);
14624
14625 %}
14626
14627 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14628
14629 match(Set dst (MoveI2F src));
14630
14631 effect(DEF dst, USE src);
14632
14633 ins_cost(INSN_COST);
14634
14635 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14636
14637 ins_encode %{
14638 __ strw($src$$Register, Address(sp, $dst$$disp));
14639 %}
14640
14641 ins_pipe(istore_reg_reg);
14642
14643 %}
14644
14645 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14646
14647 match(Set dst (MoveD2L src));
14648
14649 effect(DEF dst, USE src);
14650
14651 ins_cost(INSN_COST);
14652
14653 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14654
14655 ins_encode %{
14656 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14657 %}
14658
14659 ins_pipe(pipe_class_memory);
14660
14661 %}
14662
14663 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14664
14665 match(Set dst (MoveL2D src));
14666
14667 effect(DEF dst, USE src);
14668
14669 ins_cost(INSN_COST);
14670
14671 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14672
14673 ins_encode %{
14674 __ str($src$$Register, Address(sp, $dst$$disp));
14675 %}
14676
14677 ins_pipe(istore_reg_reg);
14678
14679 %}
14680
14681 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14682
14683 match(Set dst (MoveF2I src));
14684
14685 effect(DEF dst, USE src);
14686
14687 ins_cost(INSN_COST);
14688
14689 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14690
14691 ins_encode %{
14692 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14693 %}
14694
14695 ins_pipe(fp_f2i);
14696
14697 %}
14698
14699 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14700
14701 match(Set dst (MoveI2F src));
14702
14703 effect(DEF dst, USE src);
14704
14705 ins_cost(INSN_COST);
14706
14707 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14708
14709 ins_encode %{
14710 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14711 %}
14712
14713 ins_pipe(fp_i2f);
14714
14715 %}
14716
14717 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14718
14719 match(Set dst (MoveD2L src));
14720
14721 effect(DEF dst, USE src);
14722
14723 ins_cost(INSN_COST);
14724
14725 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14726
14727 ins_encode %{
14728 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14729 %}
14730
14731 ins_pipe(fp_d2l);
14732
14733 %}
14734
14735 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14736
14737 match(Set dst (MoveL2D src));
14738
14739 effect(DEF dst, USE src);
14740
14741 ins_cost(INSN_COST);
14742
14743 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14744
14745 ins_encode %{
14746 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14747 %}
14748
14749 ins_pipe(fp_l2d);
14750
14751 %}
14752
14753 // ============================================================================
14754 // clearing of an array
14755
14756 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14757 %{
14758 match(Set dummy (ClearArray cnt base));
14759 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14760
14761 ins_cost(4 * INSN_COST);
14762 format %{ "ClearArray $cnt, $base" %}
14763
14764 ins_encode %{
14765 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14766 if (tpc == nullptr) {
14767 ciEnv::current()->record_failure("CodeCache is full");
14768 return;
14769 }
14770 %}
14771
14772 ins_pipe(pipe_class_memory);
14773 %}
14774
14775 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14776 %{
14777 predicate((uint64_t)n->in(2)->get_long()
14778 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14779 match(Set dummy (ClearArray cnt base));
14780 effect(TEMP temp, USE_KILL base, KILL cr);
14781
14782 ins_cost(4 * INSN_COST);
14783 format %{ "ClearArray $cnt, $base" %}
14784
14785 ins_encode %{
14786 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14787 if (tpc == nullptr) {
14788 ciEnv::current()->record_failure("CodeCache is full");
14789 return;
14790 }
14791 %}
14792
14793 ins_pipe(pipe_class_memory);
14794 %}
14795
14796 // ============================================================================
14797 // Overflow Math Instructions
14798
14799 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14800 %{
14801 match(Set cr (OverflowAddI op1 op2));
14802
14803 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14804 ins_cost(INSN_COST);
14805 ins_encode %{
14806 __ cmnw($op1$$Register, $op2$$Register);
14807 %}
14808
14809 ins_pipe(icmp_reg_reg);
14810 %}
14811
14812 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14813 %{
14814 match(Set cr (OverflowAddI op1 op2));
14815
14816 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14817 ins_cost(INSN_COST);
14818 ins_encode %{
14819 __ cmnw($op1$$Register, $op2$$constant);
14820 %}
14821
14822 ins_pipe(icmp_reg_imm);
14823 %}
14824
14825 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14826 %{
14827 match(Set cr (OverflowAddL op1 op2));
14828
14829 format %{ "cmn $op1, $op2\t# overflow check long" %}
14830 ins_cost(INSN_COST);
14831 ins_encode %{
14832 __ cmn($op1$$Register, $op2$$Register);
14833 %}
14834
14835 ins_pipe(icmp_reg_reg);
14836 %}
14837
14838 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14839 %{
14840 match(Set cr (OverflowAddL op1 op2));
14841
14842 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14843 ins_cost(INSN_COST);
14844 ins_encode %{
14845 __ adds(zr, $op1$$Register, $op2$$constant);
14846 %}
14847
14848 ins_pipe(icmp_reg_imm);
14849 %}
14850
14851 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14852 %{
14853 match(Set cr (OverflowSubI op1 op2));
14854
14855 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14856 ins_cost(INSN_COST);
14857 ins_encode %{
14858 __ cmpw($op1$$Register, $op2$$Register);
14859 %}
14860
14861 ins_pipe(icmp_reg_reg);
14862 %}
14863
14864 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14865 %{
14866 match(Set cr (OverflowSubI op1 op2));
14867
14868 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14869 ins_cost(INSN_COST);
14870 ins_encode %{
14871 __ cmpw($op1$$Register, $op2$$constant);
14872 %}
14873
14874 ins_pipe(icmp_reg_imm);
14875 %}
14876
14877 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14878 %{
14879 match(Set cr (OverflowSubL op1 op2));
14880
14881 format %{ "cmp $op1, $op2\t# overflow check long" %}
14882 ins_cost(INSN_COST);
14883 ins_encode %{
14884 __ cmp($op1$$Register, $op2$$Register);
14885 %}
14886
14887 ins_pipe(icmp_reg_reg);
14888 %}
14889
14890 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14891 %{
14892 match(Set cr (OverflowSubL op1 op2));
14893
14894 format %{ "cmp $op1, $op2\t# overflow check long" %}
14895 ins_cost(INSN_COST);
14896 ins_encode %{
14897 __ subs(zr, $op1$$Register, $op2$$constant);
14898 %}
14899
14900 ins_pipe(icmp_reg_imm);
14901 %}
14902
14903 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14904 %{
14905 match(Set cr (OverflowSubI zero op1));
14906
14907 format %{ "cmpw zr, $op1\t# overflow check int" %}
14908 ins_cost(INSN_COST);
14909 ins_encode %{
14910 __ cmpw(zr, $op1$$Register);
14911 %}
14912
14913 ins_pipe(icmp_reg_imm);
14914 %}
14915
14916 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14917 %{
14918 match(Set cr (OverflowSubL zero op1));
14919
14920 format %{ "cmp zr, $op1\t# overflow check long" %}
14921 ins_cost(INSN_COST);
14922 ins_encode %{
14923 __ cmp(zr, $op1$$Register);
14924 %}
14925
14926 ins_pipe(icmp_reg_imm);
14927 %}
14928
14929 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14930 %{
14931 match(Set cr (OverflowMulI op1 op2));
14932
14933 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14934 "cmp rscratch1, rscratch1, sxtw\n\t"
14935 "movw rscratch1, #0x80000000\n\t"
14936 "cselw rscratch1, rscratch1, zr, NE\n\t"
14937 "cmpw rscratch1, #1" %}
14938 ins_cost(5 * INSN_COST);
14939 ins_encode %{
14940 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14941 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14942 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14943 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14944 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14945 %}
14946
14947 ins_pipe(pipe_slow);
14948 %}
14949
14950 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14951 %{
14952 match(If cmp (OverflowMulI op1 op2));
14953 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14954 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14955 effect(USE labl, KILL cr);
14956
14957 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14958 "cmp rscratch1, rscratch1, sxtw\n\t"
14959 "b$cmp $labl" %}
14960 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14961 ins_encode %{
14962 Label* L = $labl$$label;
14963 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14964 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14965 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14966 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14967 %}
14968
14969 ins_pipe(pipe_serial);
14970 %}
14971
14972 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14973 %{
14974 match(Set cr (OverflowMulL op1 op2));
14975
14976 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14977 "smulh rscratch2, $op1, $op2\n\t"
14978 "cmp rscratch2, rscratch1, ASR #63\n\t"
14979 "movw rscratch1, #0x80000000\n\t"
14980 "cselw rscratch1, rscratch1, zr, NE\n\t"
14981 "cmpw rscratch1, #1" %}
14982 ins_cost(6 * INSN_COST);
14983 ins_encode %{
14984 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14985 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14986 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14987 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14988 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14989 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14990 %}
14991
14992 ins_pipe(pipe_slow);
14993 %}
14994
14995 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14996 %{
14997 match(If cmp (OverflowMulL op1 op2));
14998 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14999 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15000 effect(USE labl, KILL cr);
15001
15002 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15003 "smulh rscratch2, $op1, $op2\n\t"
15004 "cmp rscratch2, rscratch1, ASR #63\n\t"
15005 "b$cmp $labl" %}
15006 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15007 ins_encode %{
15008 Label* L = $labl$$label;
15009 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15010 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15011 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15012 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15013 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15014 %}
15015
15016 ins_pipe(pipe_serial);
15017 %}
15018
15019 // ============================================================================
15020 // Compare Instructions
15021
15022 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15023 %{
15024 match(Set cr (CmpI op1 op2));
15025
15026 effect(DEF cr, USE op1, USE op2);
15027
15028 ins_cost(INSN_COST);
15029 format %{ "cmpw $op1, $op2" %}
15030
15031 ins_encode(aarch64_enc_cmpw(op1, op2));
15032
15033 ins_pipe(icmp_reg_reg);
15034 %}
15035
15036 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15037 %{
15038 match(Set cr (CmpI op1 zero));
15039
15040 effect(DEF cr, USE op1);
15041
15042 ins_cost(INSN_COST);
15043 format %{ "cmpw $op1, 0" %}
15044
15045 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15046
15047 ins_pipe(icmp_reg_imm);
15048 %}
15049
15050 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15051 %{
15052 match(Set cr (CmpI op1 op2));
15053
15054 effect(DEF cr, USE op1);
15055
15056 ins_cost(INSN_COST);
15057 format %{ "cmpw $op1, $op2" %}
15058
15059 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15060
15061 ins_pipe(icmp_reg_imm);
15062 %}
15063
15064 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15065 %{
15066 match(Set cr (CmpI op1 op2));
15067
15068 effect(DEF cr, USE op1);
15069
15070 ins_cost(INSN_COST * 2);
15071 format %{ "cmpw $op1, $op2" %}
15072
15073 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15074
15075 ins_pipe(icmp_reg_imm);
15076 %}
15077
15078 // Unsigned compare Instructions; really, same as signed compare
15079 // except it should only be used to feed an If or a CMovI which takes a
15080 // cmpOpU.
15081
15082 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15083 %{
15084 match(Set cr (CmpU op1 op2));
15085
15086 effect(DEF cr, USE op1, USE op2);
15087
15088 ins_cost(INSN_COST);
15089 format %{ "cmpw $op1, $op2\t# unsigned" %}
15090
15091 ins_encode(aarch64_enc_cmpw(op1, op2));
15092
15093 ins_pipe(icmp_reg_reg);
15094 %}
15095
15096 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15097 %{
15098 match(Set cr (CmpU op1 zero));
15099
15100 effect(DEF cr, USE op1);
15101
15102 ins_cost(INSN_COST);
15103 format %{ "cmpw $op1, #0\t# unsigned" %}
15104
15105 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15106
15107 ins_pipe(icmp_reg_imm);
15108 %}
15109
15110 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15111 %{
15112 match(Set cr (CmpU op1 op2));
15113
15114 effect(DEF cr, USE op1);
15115
15116 ins_cost(INSN_COST);
15117 format %{ "cmpw $op1, $op2\t# unsigned" %}
15118
15119 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15120
15121 ins_pipe(icmp_reg_imm);
15122 %}
15123
15124 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15125 %{
15126 match(Set cr (CmpU op1 op2));
15127
15128 effect(DEF cr, USE op1);
15129
15130 ins_cost(INSN_COST * 2);
15131 format %{ "cmpw $op1, $op2\t# unsigned" %}
15132
15133 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15134
15135 ins_pipe(icmp_reg_imm);
15136 %}
15137
15138 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15139 %{
15140 match(Set cr (CmpL op1 op2));
15141
15142 effect(DEF cr, USE op1, USE op2);
15143
15144 ins_cost(INSN_COST);
15145 format %{ "cmp $op1, $op2" %}
15146
15147 ins_encode(aarch64_enc_cmp(op1, op2));
15148
15149 ins_pipe(icmp_reg_reg);
15150 %}
15151
15152 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15153 %{
15154 match(Set cr (CmpL op1 zero));
15155
15156 effect(DEF cr, USE op1);
15157
15158 ins_cost(INSN_COST);
15159 format %{ "tst $op1" %}
15160
15161 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15162
15163 ins_pipe(icmp_reg_imm);
15164 %}
15165
15166 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15167 %{
15168 match(Set cr (CmpL op1 op2));
15169
15170 effect(DEF cr, USE op1);
15171
15172 ins_cost(INSN_COST);
15173 format %{ "cmp $op1, $op2" %}
15174
15175 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15176
15177 ins_pipe(icmp_reg_imm);
15178 %}
15179
15180 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15181 %{
15182 match(Set cr (CmpL op1 op2));
15183
15184 effect(DEF cr, USE op1);
15185
15186 ins_cost(INSN_COST * 2);
15187 format %{ "cmp $op1, $op2" %}
15188
15189 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15190
15191 ins_pipe(icmp_reg_imm);
15192 %}
15193
15194 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15195 %{
15196 match(Set cr (CmpUL op1 op2));
15197
15198 effect(DEF cr, USE op1, USE op2);
15199
15200 ins_cost(INSN_COST);
15201 format %{ "cmp $op1, $op2" %}
15202
15203 ins_encode(aarch64_enc_cmp(op1, op2));
15204
15205 ins_pipe(icmp_reg_reg);
15206 %}
15207
15208 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15209 %{
15210 match(Set cr (CmpUL op1 zero));
15211
15212 effect(DEF cr, USE op1);
15213
15214 ins_cost(INSN_COST);
15215 format %{ "tst $op1" %}
15216
15217 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15218
15219 ins_pipe(icmp_reg_imm);
15220 %}
15221
15222 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15223 %{
15224 match(Set cr (CmpUL op1 op2));
15225
15226 effect(DEF cr, USE op1);
15227
15228 ins_cost(INSN_COST);
15229 format %{ "cmp $op1, $op2" %}
15230
15231 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15232
15233 ins_pipe(icmp_reg_imm);
15234 %}
15235
15236 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15237 %{
15238 match(Set cr (CmpUL op1 op2));
15239
15240 effect(DEF cr, USE op1);
15241
15242 ins_cost(INSN_COST * 2);
15243 format %{ "cmp $op1, $op2" %}
15244
15245 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15246
15247 ins_pipe(icmp_reg_imm);
15248 %}
15249
15250 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15251 %{
15252 match(Set cr (CmpP op1 op2));
15253
15254 effect(DEF cr, USE op1, USE op2);
15255
15256 ins_cost(INSN_COST);
15257 format %{ "cmp $op1, $op2\t // ptr" %}
15258
15259 ins_encode(aarch64_enc_cmpp(op1, op2));
15260
15261 ins_pipe(icmp_reg_reg);
15262 %}
15263
15264 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15265 %{
15266 match(Set cr (CmpN op1 op2));
15267
15268 effect(DEF cr, USE op1, USE op2);
15269
15270 ins_cost(INSN_COST);
15271 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15272
15273 ins_encode(aarch64_enc_cmpn(op1, op2));
15274
15275 ins_pipe(icmp_reg_reg);
15276 %}
15277
15278 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15279 %{
15280 match(Set cr (CmpP op1 zero));
15281
15282 effect(DEF cr, USE op1, USE zero);
15283
15284 ins_cost(INSN_COST);
15285 format %{ "cmp $op1, 0\t // ptr" %}
15286
15287 ins_encode(aarch64_enc_testp(op1));
15288
15289 ins_pipe(icmp_reg_imm);
15290 %}
15291
15292 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15293 %{
15294 match(Set cr (CmpN op1 zero));
15295
15296 effect(DEF cr, USE op1, USE zero);
15297
15298 ins_cost(INSN_COST);
15299 format %{ "cmp $op1, 0\t // compressed ptr" %}
15300
15301 ins_encode(aarch64_enc_testn(op1));
15302
15303 ins_pipe(icmp_reg_imm);
15304 %}
15305
15306 // FP comparisons
15307 //
15308 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15309 // using normal cmpOp. See declaration of rFlagsReg for details.
15310
15311 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15312 %{
15313 match(Set cr (CmpF src1 src2));
15314
15315 ins_cost(3 * INSN_COST);
15316 format %{ "fcmps $src1, $src2" %}
15317
15318 ins_encode %{
15319 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15320 %}
15321
15322 ins_pipe(pipe_class_compare);
15323 %}
15324
15325 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15326 %{
15327 match(Set cr (CmpF src1 src2));
15328
15329 ins_cost(3 * INSN_COST);
15330 format %{ "fcmps $src1, 0.0" %}
15331
15332 ins_encode %{
15333 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15334 %}
15335
15336 ins_pipe(pipe_class_compare);
15337 %}
15338 // FROM HERE
15339
15340 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15341 %{
15342 match(Set cr (CmpD src1 src2));
15343
15344 ins_cost(3 * INSN_COST);
15345 format %{ "fcmpd $src1, $src2" %}
15346
15347 ins_encode %{
15348 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15349 %}
15350
15351 ins_pipe(pipe_class_compare);
15352 %}
15353
15354 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15355 %{
15356 match(Set cr (CmpD src1 src2));
15357
15358 ins_cost(3 * INSN_COST);
15359 format %{ "fcmpd $src1, 0.0" %}
15360
15361 ins_encode %{
15362 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15363 %}
15364
15365 ins_pipe(pipe_class_compare);
15366 %}
15367
15368 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15369 %{
15370 match(Set dst (CmpF3 src1 src2));
15371 effect(KILL cr);
15372
15373 ins_cost(5 * INSN_COST);
15374 format %{ "fcmps $src1, $src2\n\t"
15375 "csinvw($dst, zr, zr, eq\n\t"
15376 "csnegw($dst, $dst, $dst, lt)"
15377 %}
15378
15379 ins_encode %{
15380 Label done;
15381 FloatRegister s1 = as_FloatRegister($src1$$reg);
15382 FloatRegister s2 = as_FloatRegister($src2$$reg);
15383 Register d = as_Register($dst$$reg);
15384 __ fcmps(s1, s2);
15385 // installs 0 if EQ else -1
15386 __ csinvw(d, zr, zr, Assembler::EQ);
15387 // keeps -1 if less or unordered else installs 1
15388 __ csnegw(d, d, d, Assembler::LT);
15389 __ bind(done);
15390 %}
15391
15392 ins_pipe(pipe_class_default);
15393
15394 %}
15395
15396 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15397 %{
15398 match(Set dst (CmpD3 src1 src2));
15399 effect(KILL cr);
15400
15401 ins_cost(5 * INSN_COST);
15402 format %{ "fcmpd $src1, $src2\n\t"
15403 "csinvw($dst, zr, zr, eq\n\t"
15404 "csnegw($dst, $dst, $dst, lt)"
15405 %}
15406
15407 ins_encode %{
15408 Label done;
15409 FloatRegister s1 = as_FloatRegister($src1$$reg);
15410 FloatRegister s2 = as_FloatRegister($src2$$reg);
15411 Register d = as_Register($dst$$reg);
15412 __ fcmpd(s1, s2);
15413 // installs 0 if EQ else -1
15414 __ csinvw(d, zr, zr, Assembler::EQ);
15415 // keeps -1 if less or unordered else installs 1
15416 __ csnegw(d, d, d, Assembler::LT);
15417 __ bind(done);
15418 %}
15419 ins_pipe(pipe_class_default);
15420
15421 %}
15422
15423 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15424 %{
15425 match(Set dst (CmpF3 src1 zero));
15426 effect(KILL cr);
15427
15428 ins_cost(5 * INSN_COST);
15429 format %{ "fcmps $src1, 0.0\n\t"
15430 "csinvw($dst, zr, zr, eq\n\t"
15431 "csnegw($dst, $dst, $dst, lt)"
15432 %}
15433
15434 ins_encode %{
15435 Label done;
15436 FloatRegister s1 = as_FloatRegister($src1$$reg);
15437 Register d = as_Register($dst$$reg);
15438 __ fcmps(s1, 0.0);
15439 // installs 0 if EQ else -1
15440 __ csinvw(d, zr, zr, Assembler::EQ);
15441 // keeps -1 if less or unordered else installs 1
15442 __ csnegw(d, d, d, Assembler::LT);
15443 __ bind(done);
15444 %}
15445
15446 ins_pipe(pipe_class_default);
15447
15448 %}
15449
15450 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15451 %{
15452 match(Set dst (CmpD3 src1 zero));
15453 effect(KILL cr);
15454
15455 ins_cost(5 * INSN_COST);
15456 format %{ "fcmpd $src1, 0.0\n\t"
15457 "csinvw($dst, zr, zr, eq\n\t"
15458 "csnegw($dst, $dst, $dst, lt)"
15459 %}
15460
15461 ins_encode %{
15462 Label done;
15463 FloatRegister s1 = as_FloatRegister($src1$$reg);
15464 Register d = as_Register($dst$$reg);
15465 __ fcmpd(s1, 0.0);
15466 // installs 0 if EQ else -1
15467 __ csinvw(d, zr, zr, Assembler::EQ);
15468 // keeps -1 if less or unordered else installs 1
15469 __ csnegw(d, d, d, Assembler::LT);
15470 __ bind(done);
15471 %}
15472 ins_pipe(pipe_class_default);
15473
15474 %}
15475
15476 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15477 %{
15478 match(Set dst (CmpLTMask p q));
15479 effect(KILL cr);
15480
15481 ins_cost(3 * INSN_COST);
15482
15483 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15484 "csetw $dst, lt\n\t"
15485 "subw $dst, zr, $dst"
15486 %}
15487
15488 ins_encode %{
15489 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15490 __ csetw(as_Register($dst$$reg), Assembler::LT);
15491 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15492 %}
15493
15494 ins_pipe(ialu_reg_reg);
15495 %}
15496
15497 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15498 %{
15499 match(Set dst (CmpLTMask src zero));
15500 effect(KILL cr);
15501
15502 ins_cost(INSN_COST);
15503
15504 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15505
15506 ins_encode %{
15507 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15508 %}
15509
15510 ins_pipe(ialu_reg_shift);
15511 %}
15512
15513 // ============================================================================
15514 // Max and Min
15515
15516 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15517
15518 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15519 %{
15520 effect(DEF cr, USE src);
15521 ins_cost(INSN_COST);
15522 format %{ "cmpw $src, 0" %}
15523
15524 ins_encode %{
15525 __ cmpw($src$$Register, 0);
15526 %}
15527 ins_pipe(icmp_reg_imm);
15528 %}
15529
15530 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15531 %{
15532 match(Set dst (MinI src1 src2));
15533 ins_cost(INSN_COST * 3);
15534
15535 expand %{
15536 rFlagsReg cr;
15537 compI_reg_reg(cr, src1, src2);
15538 cmovI_reg_reg_lt(dst, src1, src2, cr);
15539 %}
15540 %}
15541
15542 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15543 %{
15544 match(Set dst (MaxI src1 src2));
15545 ins_cost(INSN_COST * 3);
15546
15547 expand %{
15548 rFlagsReg cr;
15549 compI_reg_reg(cr, src1, src2);
15550 cmovI_reg_reg_gt(dst, src1, src2, cr);
15551 %}
15552 %}
15553
15554
15555 // ============================================================================
15556 // Branch Instructions
15557
15558 // Direct Branch.
15559 instruct branch(label lbl)
15560 %{
15561 match(Goto);
15562
15563 effect(USE lbl);
15564
15565 ins_cost(BRANCH_COST);
15566 format %{ "b $lbl" %}
15567
15568 ins_encode(aarch64_enc_b(lbl));
15569
15570 ins_pipe(pipe_branch);
15571 %}
15572
15573 // Conditional Near Branch
15574 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15575 %{
15576 // Same match rule as `branchConFar'.
15577 match(If cmp cr);
15578
15579 effect(USE lbl);
15580
15581 ins_cost(BRANCH_COST);
15582 // If set to 1 this indicates that the current instruction is a
15583 // short variant of a long branch. This avoids using this
15584 // instruction in first-pass matching. It will then only be used in
15585 // the `Shorten_branches' pass.
15586 // ins_short_branch(1);
15587 format %{ "b$cmp $lbl" %}
15588
15589 ins_encode(aarch64_enc_br_con(cmp, lbl));
15590
15591 ins_pipe(pipe_branch_cond);
15592 %}
15593
15594 // Conditional Near Branch Unsigned
15595 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15596 %{
15597 // Same match rule as `branchConFar'.
15598 match(If cmp cr);
15599
15600 effect(USE lbl);
15601
15602 ins_cost(BRANCH_COST);
15603 // If set to 1 this indicates that the current instruction is a
15604 // short variant of a long branch. This avoids using this
15605 // instruction in first-pass matching. It will then only be used in
15606 // the `Shorten_branches' pass.
15607 // ins_short_branch(1);
15608 format %{ "b$cmp $lbl\t# unsigned" %}
15609
15610 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15611
15612 ins_pipe(pipe_branch_cond);
15613 %}
15614
15615 // Make use of CBZ and CBNZ. These instructions, as well as being
15616 // shorter than (cmp; branch), have the additional benefit of not
15617 // killing the flags.
15618
15619 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15620 match(If cmp (CmpI op1 op2));
15621 effect(USE labl);
15622
15623 ins_cost(BRANCH_COST);
15624 format %{ "cbw$cmp $op1, $labl" %}
15625 ins_encode %{
15626 Label* L = $labl$$label;
15627 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15628 if (cond == Assembler::EQ)
15629 __ cbzw($op1$$Register, *L);
15630 else
15631 __ cbnzw($op1$$Register, *L);
15632 %}
15633 ins_pipe(pipe_cmp_branch);
15634 %}
15635
15636 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15637 match(If cmp (CmpL op1 op2));
15638 effect(USE labl);
15639
15640 ins_cost(BRANCH_COST);
15641 format %{ "cb$cmp $op1, $labl" %}
15642 ins_encode %{
15643 Label* L = $labl$$label;
15644 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15645 if (cond == Assembler::EQ)
15646 __ cbz($op1$$Register, *L);
15647 else
15648 __ cbnz($op1$$Register, *L);
15649 %}
15650 ins_pipe(pipe_cmp_branch);
15651 %}
15652
15653 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15654 match(If cmp (CmpP op1 op2));
15655 effect(USE labl);
15656
15657 ins_cost(BRANCH_COST);
15658 format %{ "cb$cmp $op1, $labl" %}
15659 ins_encode %{
15660 Label* L = $labl$$label;
15661 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15662 if (cond == Assembler::EQ)
15663 __ cbz($op1$$Register, *L);
15664 else
15665 __ cbnz($op1$$Register, *L);
15666 %}
15667 ins_pipe(pipe_cmp_branch);
15668 %}
15669
15670 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15671 match(If cmp (CmpN op1 op2));
15672 effect(USE labl);
15673
15674 ins_cost(BRANCH_COST);
15675 format %{ "cbw$cmp $op1, $labl" %}
15676 ins_encode %{
15677 Label* L = $labl$$label;
15678 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15679 if (cond == Assembler::EQ)
15680 __ cbzw($op1$$Register, *L);
15681 else
15682 __ cbnzw($op1$$Register, *L);
15683 %}
15684 ins_pipe(pipe_cmp_branch);
15685 %}
15686
15687 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15688 match(If cmp (CmpP (DecodeN oop) zero));
15689 effect(USE labl);
15690
15691 ins_cost(BRANCH_COST);
15692 format %{ "cb$cmp $oop, $labl" %}
15693 ins_encode %{
15694 Label* L = $labl$$label;
15695 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15696 if (cond == Assembler::EQ)
15697 __ cbzw($oop$$Register, *L);
15698 else
15699 __ cbnzw($oop$$Register, *L);
15700 %}
15701 ins_pipe(pipe_cmp_branch);
15702 %}
15703
15704 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15705 match(If cmp (CmpU op1 op2));
15706 effect(USE labl);
15707
15708 ins_cost(BRANCH_COST);
15709 format %{ "cbw$cmp $op1, $labl" %}
15710 ins_encode %{
15711 Label* L = $labl$$label;
15712 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15713 if (cond == Assembler::EQ || cond == Assembler::LS) {
15714 __ cbzw($op1$$Register, *L);
15715 } else {
15716 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15717 __ cbnzw($op1$$Register, *L);
15718 }
15719 %}
15720 ins_pipe(pipe_cmp_branch);
15721 %}
15722
15723 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15724 match(If cmp (CmpUL op1 op2));
15725 effect(USE labl);
15726
15727 ins_cost(BRANCH_COST);
15728 format %{ "cb$cmp $op1, $labl" %}
15729 ins_encode %{
15730 Label* L = $labl$$label;
15731 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15732 if (cond == Assembler::EQ || cond == Assembler::LS) {
15733 __ cbz($op1$$Register, *L);
15734 } else {
15735 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15736 __ cbnz($op1$$Register, *L);
15737 }
15738 %}
15739 ins_pipe(pipe_cmp_branch);
15740 %}
15741
15742 // Test bit and Branch
15743
15744 // Patterns for short (< 32KiB) variants
15745 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15746 match(If cmp (CmpL op1 op2));
15747 effect(USE labl);
15748
15749 ins_cost(BRANCH_COST);
15750 format %{ "cb$cmp $op1, $labl # long" %}
15751 ins_encode %{
15752 Label* L = $labl$$label;
15753 Assembler::Condition cond =
15754 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15755 __ tbr(cond, $op1$$Register, 63, *L);
15756 %}
15757 ins_pipe(pipe_cmp_branch);
15758 ins_short_branch(1);
15759 %}
15760
15761 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15762 match(If cmp (CmpI op1 op2));
15763 effect(USE labl);
15764
15765 ins_cost(BRANCH_COST);
15766 format %{ "cb$cmp $op1, $labl # int" %}
15767 ins_encode %{
15768 Label* L = $labl$$label;
15769 Assembler::Condition cond =
15770 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15771 __ tbr(cond, $op1$$Register, 31, *L);
15772 %}
15773 ins_pipe(pipe_cmp_branch);
15774 ins_short_branch(1);
15775 %}
15776
15777 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15778 match(If cmp (CmpL (AndL op1 op2) op3));
15779 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15780 effect(USE labl);
15781
15782 ins_cost(BRANCH_COST);
15783 format %{ "tb$cmp $op1, $op2, $labl" %}
15784 ins_encode %{
15785 Label* L = $labl$$label;
15786 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15787 int bit = exact_log2_long($op2$$constant);
15788 __ tbr(cond, $op1$$Register, bit, *L);
15789 %}
15790 ins_pipe(pipe_cmp_branch);
15791 ins_short_branch(1);
15792 %}
15793
15794 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15795 match(If cmp (CmpI (AndI op1 op2) op3));
15796 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15797 effect(USE labl);
15798
15799 ins_cost(BRANCH_COST);
15800 format %{ "tb$cmp $op1, $op2, $labl" %}
15801 ins_encode %{
15802 Label* L = $labl$$label;
15803 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15804 int bit = exact_log2((juint)$op2$$constant);
15805 __ tbr(cond, $op1$$Register, bit, *L);
15806 %}
15807 ins_pipe(pipe_cmp_branch);
15808 ins_short_branch(1);
15809 %}
15810
15811 // And far variants
15812 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15813 match(If cmp (CmpL op1 op2));
15814 effect(USE labl);
15815
15816 ins_cost(BRANCH_COST);
15817 format %{ "cb$cmp $op1, $labl # long" %}
15818 ins_encode %{
15819 Label* L = $labl$$label;
15820 Assembler::Condition cond =
15821 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15822 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15823 %}
15824 ins_pipe(pipe_cmp_branch);
15825 %}
15826
15827 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15828 match(If cmp (CmpI op1 op2));
15829 effect(USE labl);
15830
15831 ins_cost(BRANCH_COST);
15832 format %{ "cb$cmp $op1, $labl # int" %}
15833 ins_encode %{
15834 Label* L = $labl$$label;
15835 Assembler::Condition cond =
15836 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15837 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15838 %}
15839 ins_pipe(pipe_cmp_branch);
15840 %}
15841
15842 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15843 match(If cmp (CmpL (AndL op1 op2) op3));
15844 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15845 effect(USE labl);
15846
15847 ins_cost(BRANCH_COST);
15848 format %{ "tb$cmp $op1, $op2, $labl" %}
15849 ins_encode %{
15850 Label* L = $labl$$label;
15851 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15852 int bit = exact_log2_long($op2$$constant);
15853 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15854 %}
15855 ins_pipe(pipe_cmp_branch);
15856 %}
15857
15858 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15859 match(If cmp (CmpI (AndI op1 op2) op3));
15860 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15861 effect(USE labl);
15862
15863 ins_cost(BRANCH_COST);
15864 format %{ "tb$cmp $op1, $op2, $labl" %}
15865 ins_encode %{
15866 Label* L = $labl$$label;
15867 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15868 int bit = exact_log2((juint)$op2$$constant);
15869 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15870 %}
15871 ins_pipe(pipe_cmp_branch);
15872 %}
15873
15874 // Test bits
15875
15876 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15877 match(Set cr (CmpL (AndL op1 op2) op3));
15878 predicate(Assembler::operand_valid_for_logical_immediate
15879 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15880
15881 ins_cost(INSN_COST);
15882 format %{ "tst $op1, $op2 # long" %}
15883 ins_encode %{
15884 __ tst($op1$$Register, $op2$$constant);
15885 %}
15886 ins_pipe(ialu_reg_reg);
15887 %}
15888
15889 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15890 match(Set cr (CmpI (AndI op1 op2) op3));
15891 predicate(Assembler::operand_valid_for_logical_immediate
15892 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15893
15894 ins_cost(INSN_COST);
15895 format %{ "tst $op1, $op2 # int" %}
15896 ins_encode %{
15897 __ tstw($op1$$Register, $op2$$constant);
15898 %}
15899 ins_pipe(ialu_reg_reg);
15900 %}
15901
15902 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15903 match(Set cr (CmpL (AndL op1 op2) op3));
15904
15905 ins_cost(INSN_COST);
15906 format %{ "tst $op1, $op2 # long" %}
15907 ins_encode %{
15908 __ tst($op1$$Register, $op2$$Register);
15909 %}
15910 ins_pipe(ialu_reg_reg);
15911 %}
15912
15913 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15914 match(Set cr (CmpI (AndI op1 op2) op3));
15915
15916 ins_cost(INSN_COST);
15917 format %{ "tstw $op1, $op2 # int" %}
15918 ins_encode %{
15919 __ tstw($op1$$Register, $op2$$Register);
15920 %}
15921 ins_pipe(ialu_reg_reg);
15922 %}
15923
15924
15925 // Conditional Far Branch
15926 // Conditional Far Branch Unsigned
15927 // TODO: fixme
15928
15929 // counted loop end branch near
15930 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15931 %{
15932 match(CountedLoopEnd cmp cr);
15933
15934 effect(USE lbl);
15935
15936 ins_cost(BRANCH_COST);
15937 // short variant.
15938 // ins_short_branch(1);
15939 format %{ "b$cmp $lbl \t// counted loop end" %}
15940
15941 ins_encode(aarch64_enc_br_con(cmp, lbl));
15942
15943 ins_pipe(pipe_branch);
15944 %}
15945
15946 // counted loop end branch far
15947 // TODO: fixme
15948
15949 // ============================================================================
15950 // inlined locking and unlocking
15951
15952 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15953 %{
15954 predicate(LockingMode != LM_LIGHTWEIGHT);
15955 match(Set cr (FastLock object box));
15956 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15957
15958 ins_cost(5 * INSN_COST);
15959 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15960
15961 ins_encode %{
15962 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15963 %}
15964
15965 ins_pipe(pipe_serial);
15966 %}
15967
15968 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
15969 %{
15970 predicate(LockingMode != LM_LIGHTWEIGHT);
15971 match(Set cr (FastUnlock object box));
15972 effect(TEMP tmp, TEMP tmp2);
15973
15974 ins_cost(5 * INSN_COST);
15975 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
15976
15977 ins_encode %{
15978 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
15979 %}
15980
15981 ins_pipe(pipe_serial);
15982 %}
15983
15984 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15985 %{
15986 predicate(LockingMode == LM_LIGHTWEIGHT);
15987 match(Set cr (FastLock object box));
15988 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15989
15990 ins_cost(5 * INSN_COST);
15991 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15992
15993 ins_encode %{
15994 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15995 %}
15996
15997 ins_pipe(pipe_serial);
15998 %}
15999
16000 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16001 %{
16002 predicate(LockingMode == LM_LIGHTWEIGHT);
16003 match(Set cr (FastUnlock object box));
16004 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16005
16006 ins_cost(5 * INSN_COST);
16007 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16008
16009 ins_encode %{
16010 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16011 %}
16012
16013 ins_pipe(pipe_serial);
16014 %}
16015
16016 // ============================================================================
16017 // Safepoint Instructions
16018
16019 // TODO
16020 // provide a near and far version of this code
16021
16022 instruct safePoint(rFlagsReg cr, iRegP poll)
16023 %{
16024 match(SafePoint poll);
16025 effect(KILL cr);
16026
16027 format %{
16028 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16029 %}
16030 ins_encode %{
16031 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16032 %}
16033 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16034 %}
16035
16036
16037 // ============================================================================
16038 // Procedure Call/Return Instructions
16039
16040 // Call Java Static Instruction
16041
16042 instruct CallStaticJavaDirect(method meth)
16043 %{
16044 match(CallStaticJava);
16045
16046 effect(USE meth);
16047
16048 ins_cost(CALL_COST);
16049
16050 format %{ "call,static $meth \t// ==> " %}
16051
16052 ins_encode(aarch64_enc_java_static_call(meth),
16053 aarch64_enc_call_epilog);
16054
16055 ins_pipe(pipe_class_call);
16056 %}
16057
16058 // TO HERE
16059
16060 // Call Java Dynamic Instruction
16061 instruct CallDynamicJavaDirect(method meth)
16062 %{
16063 match(CallDynamicJava);
16064
16065 effect(USE meth);
16066
16067 ins_cost(CALL_COST);
16068
16069 format %{ "CALL,dynamic $meth \t// ==> " %}
16070
16071 ins_encode(aarch64_enc_java_dynamic_call(meth),
16072 aarch64_enc_call_epilog);
16073
16074 ins_pipe(pipe_class_call);
16075 %}
16076
16077 // Call Runtime Instruction
16078
16079 instruct CallRuntimeDirect(method meth)
16080 %{
16081 match(CallRuntime);
16082
16083 effect(USE meth);
16084
16085 ins_cost(CALL_COST);
16086
16087 format %{ "CALL, runtime $meth" %}
16088
16089 ins_encode( aarch64_enc_java_to_runtime(meth) );
16090
16091 ins_pipe(pipe_class_call);
16092 %}
16093
16094 // Call Runtime Instruction
16095
16096 instruct CallLeafDirect(method meth)
16097 %{
16098 match(CallLeaf);
16099
16100 effect(USE meth);
16101
16102 ins_cost(CALL_COST);
16103
16104 format %{ "CALL, runtime leaf $meth" %}
16105
16106 ins_encode( aarch64_enc_java_to_runtime(meth) );
16107
16108 ins_pipe(pipe_class_call);
16109 %}
16110
16111 // Call Runtime Instruction
16112
16113 instruct CallLeafNoFPDirect(method meth)
16114 %{
16115 match(CallLeafNoFP);
16116
16117 effect(USE meth);
16118
16119 ins_cost(CALL_COST);
16120
16121 format %{ "CALL, runtime leaf nofp $meth" %}
16122
16123 ins_encode( aarch64_enc_java_to_runtime(meth) );
16124
16125 ins_pipe(pipe_class_call);
16126 %}
16127
16128 // Tail Call; Jump from runtime stub to Java code.
16129 // Also known as an 'interprocedural jump'.
16130 // Target of jump will eventually return to caller.
16131 // TailJump below removes the return address.
16132 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16133 // emitted just above the TailCall which has reset rfp to the caller state.
16134 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16135 %{
16136 match(TailCall jump_target method_ptr);
16137
16138 ins_cost(CALL_COST);
16139
16140 format %{ "br $jump_target\t# $method_ptr holds method" %}
16141
16142 ins_encode(aarch64_enc_tail_call(jump_target));
16143
16144 ins_pipe(pipe_class_call);
16145 %}
16146
16147 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16148 %{
16149 match(TailJump jump_target ex_oop);
16150
16151 ins_cost(CALL_COST);
16152
16153 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16154
16155 ins_encode(aarch64_enc_tail_jmp(jump_target));
16156
16157 ins_pipe(pipe_class_call);
16158 %}
16159
16160 // Create exception oop: created by stack-crawling runtime code.
16161 // Created exception is now available to this handler, and is setup
16162 // just prior to jumping to this handler. No code emitted.
16163 // TODO check
16164 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16165 instruct CreateException(iRegP_R0 ex_oop)
16166 %{
16167 match(Set ex_oop (CreateEx));
16168
16169 format %{ " -- \t// exception oop; no code emitted" %}
16170
16171 size(0);
16172
16173 ins_encode( /*empty*/ );
16174
16175 ins_pipe(pipe_class_empty);
16176 %}
16177
16178 // Rethrow exception: The exception oop will come in the first
16179 // argument position. Then JUMP (not call) to the rethrow stub code.
16180 instruct RethrowException() %{
16181 match(Rethrow);
16182 ins_cost(CALL_COST);
16183
16184 format %{ "b rethrow_stub" %}
16185
16186 ins_encode( aarch64_enc_rethrow() );
16187
16188 ins_pipe(pipe_class_call);
16189 %}
16190
16191
16192 // Return Instruction
16193 // epilog node loads ret address into lr as part of frame pop
16194 instruct Ret()
16195 %{
16196 match(Return);
16197
16198 format %{ "ret\t// return register" %}
16199
16200 ins_encode( aarch64_enc_ret() );
16201
16202 ins_pipe(pipe_branch);
16203 %}
16204
16205 // Die now.
16206 instruct ShouldNotReachHere() %{
16207 match(Halt);
16208
16209 ins_cost(CALL_COST);
16210 format %{ "ShouldNotReachHere" %}
16211
16212 ins_encode %{
16213 if (is_reachable()) {
16214 __ stop(_halt_reason);
16215 }
16216 %}
16217
16218 ins_pipe(pipe_class_default);
16219 %}
16220
16221 // ============================================================================
16222 // Partial Subtype Check
16223 //
16224 // superklass array for an instance of the superklass. Set a hidden
16225 // internal cache on a hit (cache is checked with exposed code in
16226 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16227 // encoding ALSO sets flags.
16228
16229 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16230 %{
16231 match(Set result (PartialSubtypeCheck sub super));
16232 effect(KILL cr, KILL temp);
16233
16234 ins_cost(1100); // slightly larger than the next version
16235 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16236
16237 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16238
16239 opcode(0x1); // Force zero of result reg on hit
16240
16241 ins_pipe(pipe_class_memory);
16242 %}
16243
16244 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16245 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16246 rFlagsReg cr)
16247 %{
16248 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16249 predicate(UseSecondarySupersTable);
16250 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16251
16252 ins_cost(700); // smaller than the next version
16253 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16254
16255 ins_encode %{
16256 bool success = false;
16257 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16258 if (InlineSecondarySupersTest) {
16259 success = __ lookup_secondary_supers_table($sub$$Register, $super_reg$$Register,
16260 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16261 $vtemp$$FloatRegister,
16262 $result$$Register,
16263 super_klass_slot);
16264 } else {
16265 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16266 success = (call != nullptr);
16267 }
16268 if (!success) {
16269 ciEnv::current()->record_failure("CodeCache is full");
16270 return;
16271 }
16272 %}
16273
16274 ins_pipe(pipe_class_memory);
16275 %}
16276
16277 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16278 %{
16279 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16280 effect(KILL temp, KILL result);
16281
16282 ins_cost(1100); // slightly larger than the next version
16283 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16284
16285 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16286
16287 opcode(0x0); // Don't zero result reg on hit
16288
16289 ins_pipe(pipe_class_memory);
16290 %}
16291
16292 // Intrisics for String.compareTo()
16293
16294 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16295 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16296 %{
16297 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16298 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16299 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16300
16301 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16302 ins_encode %{
16303 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16304 __ string_compare($str1$$Register, $str2$$Register,
16305 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16306 $tmp1$$Register, $tmp2$$Register,
16307 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16308 %}
16309 ins_pipe(pipe_class_memory);
16310 %}
16311
16312 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16313 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16314 %{
16315 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16316 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16317 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16318
16319 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16320 ins_encode %{
16321 __ string_compare($str1$$Register, $str2$$Register,
16322 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16323 $tmp1$$Register, $tmp2$$Register,
16324 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16325 %}
16326 ins_pipe(pipe_class_memory);
16327 %}
16328
16329 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16330 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16331 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16332 %{
16333 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16334 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16335 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16336 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16337
16338 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16339 ins_encode %{
16340 __ string_compare($str1$$Register, $str2$$Register,
16341 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16342 $tmp1$$Register, $tmp2$$Register,
16343 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16344 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16345 %}
16346 ins_pipe(pipe_class_memory);
16347 %}
16348
16349 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16350 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16351 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16352 %{
16353 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16354 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16355 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16356 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16357
16358 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16359 ins_encode %{
16360 __ string_compare($str1$$Register, $str2$$Register,
16361 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16362 $tmp1$$Register, $tmp2$$Register,
16363 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16364 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16365 %}
16366 ins_pipe(pipe_class_memory);
16367 %}
16368
16369 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16370 // these string_compare variants as NEON register type for convenience so that the prototype of
16371 // string_compare can be shared with all variants.
16372
16373 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16374 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16375 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16376 pRegGov_P1 pgtmp2, rFlagsReg cr)
16377 %{
16378 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16379 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16380 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16381 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16382
16383 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16384 ins_encode %{
16385 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16386 __ string_compare($str1$$Register, $str2$$Register,
16387 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16388 $tmp1$$Register, $tmp2$$Register,
16389 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16390 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16391 StrIntrinsicNode::LL);
16392 %}
16393 ins_pipe(pipe_class_memory);
16394 %}
16395
16396 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16397 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16398 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16399 pRegGov_P1 pgtmp2, rFlagsReg cr)
16400 %{
16401 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16402 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16403 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16404 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16405
16406 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16407 ins_encode %{
16408 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16409 __ string_compare($str1$$Register, $str2$$Register,
16410 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16411 $tmp1$$Register, $tmp2$$Register,
16412 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16413 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16414 StrIntrinsicNode::LU);
16415 %}
16416 ins_pipe(pipe_class_memory);
16417 %}
16418
16419 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16420 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16421 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16422 pRegGov_P1 pgtmp2, rFlagsReg cr)
16423 %{
16424 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16425 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16426 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16427 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16428
16429 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16430 ins_encode %{
16431 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16432 __ string_compare($str1$$Register, $str2$$Register,
16433 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16434 $tmp1$$Register, $tmp2$$Register,
16435 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16436 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16437 StrIntrinsicNode::UL);
16438 %}
16439 ins_pipe(pipe_class_memory);
16440 %}
16441
16442 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16443 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16444 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16445 pRegGov_P1 pgtmp2, rFlagsReg cr)
16446 %{
16447 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16448 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16449 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16450 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16451
16452 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16453 ins_encode %{
16454 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16455 __ string_compare($str1$$Register, $str2$$Register,
16456 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16457 $tmp1$$Register, $tmp2$$Register,
16458 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16459 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16460 StrIntrinsicNode::UU);
16461 %}
16462 ins_pipe(pipe_class_memory);
16463 %}
16464
16465 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16466 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16467 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16468 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16469 %{
16470 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16471 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16472 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16473 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16474 TEMP vtmp0, TEMP vtmp1, KILL cr);
16475 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16476 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16477
16478 ins_encode %{
16479 __ string_indexof($str1$$Register, $str2$$Register,
16480 $cnt1$$Register, $cnt2$$Register,
16481 $tmp1$$Register, $tmp2$$Register,
16482 $tmp3$$Register, $tmp4$$Register,
16483 $tmp5$$Register, $tmp6$$Register,
16484 -1, $result$$Register, StrIntrinsicNode::UU);
16485 %}
16486 ins_pipe(pipe_class_memory);
16487 %}
16488
16489 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16490 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16491 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16492 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16493 %{
16494 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16495 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16496 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16497 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16498 TEMP vtmp0, TEMP vtmp1, KILL cr);
16499 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16500 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16501
16502 ins_encode %{
16503 __ string_indexof($str1$$Register, $str2$$Register,
16504 $cnt1$$Register, $cnt2$$Register,
16505 $tmp1$$Register, $tmp2$$Register,
16506 $tmp3$$Register, $tmp4$$Register,
16507 $tmp5$$Register, $tmp6$$Register,
16508 -1, $result$$Register, StrIntrinsicNode::LL);
16509 %}
16510 ins_pipe(pipe_class_memory);
16511 %}
16512
16513 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16514 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16515 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16516 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16517 %{
16518 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16519 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16520 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16521 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16522 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16523 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16524 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16525
16526 ins_encode %{
16527 __ string_indexof($str1$$Register, $str2$$Register,
16528 $cnt1$$Register, $cnt2$$Register,
16529 $tmp1$$Register, $tmp2$$Register,
16530 $tmp3$$Register, $tmp4$$Register,
16531 $tmp5$$Register, $tmp6$$Register,
16532 -1, $result$$Register, StrIntrinsicNode::UL);
16533 %}
16534 ins_pipe(pipe_class_memory);
16535 %}
16536
16537 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16538 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16539 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16540 %{
16541 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16542 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16543 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16544 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16545 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16546 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16547
16548 ins_encode %{
16549 int icnt2 = (int)$int_cnt2$$constant;
16550 __ string_indexof($str1$$Register, $str2$$Register,
16551 $cnt1$$Register, zr,
16552 $tmp1$$Register, $tmp2$$Register,
16553 $tmp3$$Register, $tmp4$$Register, zr, zr,
16554 icnt2, $result$$Register, StrIntrinsicNode::UU);
16555 %}
16556 ins_pipe(pipe_class_memory);
16557 %}
16558
16559 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16560 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16561 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16562 %{
16563 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16564 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16565 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16566 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16567 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16568 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16569
16570 ins_encode %{
16571 int icnt2 = (int)$int_cnt2$$constant;
16572 __ string_indexof($str1$$Register, $str2$$Register,
16573 $cnt1$$Register, zr,
16574 $tmp1$$Register, $tmp2$$Register,
16575 $tmp3$$Register, $tmp4$$Register, zr, zr,
16576 icnt2, $result$$Register, StrIntrinsicNode::LL);
16577 %}
16578 ins_pipe(pipe_class_memory);
16579 %}
16580
16581 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16582 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16583 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16584 %{
16585 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16586 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16587 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16588 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16589 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16590 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16591
16592 ins_encode %{
16593 int icnt2 = (int)$int_cnt2$$constant;
16594 __ string_indexof($str1$$Register, $str2$$Register,
16595 $cnt1$$Register, zr,
16596 $tmp1$$Register, $tmp2$$Register,
16597 $tmp3$$Register, $tmp4$$Register, zr, zr,
16598 icnt2, $result$$Register, StrIntrinsicNode::UL);
16599 %}
16600 ins_pipe(pipe_class_memory);
16601 %}
16602
16603 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16604 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16605 iRegINoSp tmp3, rFlagsReg cr)
16606 %{
16607 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16608 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16609 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16610 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16611
16612 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16613
16614 ins_encode %{
16615 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16616 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16617 $tmp3$$Register);
16618 %}
16619 ins_pipe(pipe_class_memory);
16620 %}
16621
16622 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16623 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16624 iRegINoSp tmp3, rFlagsReg cr)
16625 %{
16626 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16627 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16628 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16629 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16630
16631 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16632
16633 ins_encode %{
16634 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16635 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16636 $tmp3$$Register);
16637 %}
16638 ins_pipe(pipe_class_memory);
16639 %}
16640
16641 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16642 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16643 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16644 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16645 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16646 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16647 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16648 ins_encode %{
16649 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16650 $result$$Register, $ztmp1$$FloatRegister,
16651 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16652 $ptmp$$PRegister, true /* isL */);
16653 %}
16654 ins_pipe(pipe_class_memory);
16655 %}
16656
16657 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16658 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16659 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16660 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16661 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16662 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16663 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16664 ins_encode %{
16665 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16666 $result$$Register, $ztmp1$$FloatRegister,
16667 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16668 $ptmp$$PRegister, false /* isL */);
16669 %}
16670 ins_pipe(pipe_class_memory);
16671 %}
16672
16673 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16674 iRegI_R0 result, rFlagsReg cr)
16675 %{
16676 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16677 match(Set result (StrEquals (Binary str1 str2) cnt));
16678 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16679
16680 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16681 ins_encode %{
16682 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16683 __ string_equals($str1$$Register, $str2$$Register,
16684 $result$$Register, $cnt$$Register);
16685 %}
16686 ins_pipe(pipe_class_memory);
16687 %}
16688
16689 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16690 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16691 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16692 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16693 iRegP_R10 tmp, rFlagsReg cr)
16694 %{
16695 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16696 match(Set result (AryEq ary1 ary2));
16697 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16698 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16699 TEMP vtmp6, TEMP vtmp7, KILL cr);
16700
16701 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16702 ins_encode %{
16703 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16704 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16705 $result$$Register, $tmp$$Register, 1);
16706 if (tpc == nullptr) {
16707 ciEnv::current()->record_failure("CodeCache is full");
16708 return;
16709 }
16710 %}
16711 ins_pipe(pipe_class_memory);
16712 %}
16713
16714 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16715 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16716 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16717 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16718 iRegP_R10 tmp, rFlagsReg cr)
16719 %{
16720 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16721 match(Set result (AryEq ary1 ary2));
16722 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16723 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16724 TEMP vtmp6, TEMP vtmp7, KILL cr);
16725
16726 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16727 ins_encode %{
16728 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16729 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16730 $result$$Register, $tmp$$Register, 2);
16731 if (tpc == nullptr) {
16732 ciEnv::current()->record_failure("CodeCache is full");
16733 return;
16734 }
16735 %}
16736 ins_pipe(pipe_class_memory);
16737 %}
16738
16739 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16740 %{
16741 match(Set result (CountPositives ary1 len));
16742 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16743 format %{ "count positives byte[] $ary1,$len -> $result" %}
16744 ins_encode %{
16745 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16746 if (tpc == nullptr) {
16747 ciEnv::current()->record_failure("CodeCache is full");
16748 return;
16749 }
16750 %}
16751 ins_pipe( pipe_slow );
16752 %}
16753
16754 // fast char[] to byte[] compression
16755 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16756 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16757 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16758 iRegI_R0 result, rFlagsReg cr)
16759 %{
16760 match(Set result (StrCompressedCopy src (Binary dst len)));
16761 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16762 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16763
16764 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16765 ins_encode %{
16766 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16767 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16768 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16769 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16770 %}
16771 ins_pipe(pipe_slow);
16772 %}
16773
16774 // fast byte[] to char[] inflation
16775 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16776 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16777 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16778 %{
16779 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16780 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16781 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16782 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16783
16784 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16785 ins_encode %{
16786 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16787 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16788 $vtmp2$$FloatRegister, $tmp$$Register);
16789 if (tpc == nullptr) {
16790 ciEnv::current()->record_failure("CodeCache is full");
16791 return;
16792 }
16793 %}
16794 ins_pipe(pipe_class_memory);
16795 %}
16796
16797 // encode char[] to byte[] in ISO_8859_1
16798 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16799 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16800 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16801 iRegI_R0 result, rFlagsReg cr)
16802 %{
16803 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16804 match(Set result (EncodeISOArray src (Binary dst len)));
16805 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16806 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16807
16808 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16809 ins_encode %{
16810 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16811 $result$$Register, false,
16812 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16813 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16814 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16815 %}
16816 ins_pipe(pipe_class_memory);
16817 %}
16818
16819 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16820 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16821 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16822 iRegI_R0 result, rFlagsReg cr)
16823 %{
16824 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16825 match(Set result (EncodeISOArray src (Binary dst len)));
16826 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16827 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16828
16829 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16830 ins_encode %{
16831 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16832 $result$$Register, true,
16833 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16834 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16835 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16836 %}
16837 ins_pipe(pipe_class_memory);
16838 %}
16839
16840 //----------------------------- CompressBits/ExpandBits ------------------------
16841
16842 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16843 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16844 match(Set dst (CompressBits src mask));
16845 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16846 format %{ "mov $tsrc, $src\n\t"
16847 "mov $tmask, $mask\n\t"
16848 "bext $tdst, $tsrc, $tmask\n\t"
16849 "mov $dst, $tdst"
16850 %}
16851 ins_encode %{
16852 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16853 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16854 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16855 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16856 %}
16857 ins_pipe(pipe_slow);
16858 %}
16859
16860 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16861 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16862 match(Set dst (CompressBits (LoadI mem) mask));
16863 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16864 format %{ "ldrs $tsrc, $mem\n\t"
16865 "ldrs $tmask, $mask\n\t"
16866 "bext $tdst, $tsrc, $tmask\n\t"
16867 "mov $dst, $tdst"
16868 %}
16869 ins_encode %{
16870 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16871 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16872 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16873 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16874 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16875 %}
16876 ins_pipe(pipe_slow);
16877 %}
16878
16879 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16880 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16881 match(Set dst (CompressBits src mask));
16882 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16883 format %{ "mov $tsrc, $src\n\t"
16884 "mov $tmask, $mask\n\t"
16885 "bext $tdst, $tsrc, $tmask\n\t"
16886 "mov $dst, $tdst"
16887 %}
16888 ins_encode %{
16889 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16890 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16891 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16892 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16893 %}
16894 ins_pipe(pipe_slow);
16895 %}
16896
16897 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16898 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16899 match(Set dst (CompressBits (LoadL mem) mask));
16900 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16901 format %{ "ldrd $tsrc, $mem\n\t"
16902 "ldrd $tmask, $mask\n\t"
16903 "bext $tdst, $tsrc, $tmask\n\t"
16904 "mov $dst, $tdst"
16905 %}
16906 ins_encode %{
16907 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16908 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16909 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16910 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16911 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16912 %}
16913 ins_pipe(pipe_slow);
16914 %}
16915
16916 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16917 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16918 match(Set dst (ExpandBits src mask));
16919 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16920 format %{ "mov $tsrc, $src\n\t"
16921 "mov $tmask, $mask\n\t"
16922 "bdep $tdst, $tsrc, $tmask\n\t"
16923 "mov $dst, $tdst"
16924 %}
16925 ins_encode %{
16926 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16927 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16928 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16929 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16930 %}
16931 ins_pipe(pipe_slow);
16932 %}
16933
16934 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16935 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16936 match(Set dst (ExpandBits (LoadI mem) mask));
16937 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16938 format %{ "ldrs $tsrc, $mem\n\t"
16939 "ldrs $tmask, $mask\n\t"
16940 "bdep $tdst, $tsrc, $tmask\n\t"
16941 "mov $dst, $tdst"
16942 %}
16943 ins_encode %{
16944 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16945 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16946 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16947 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16948 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16949 %}
16950 ins_pipe(pipe_slow);
16951 %}
16952
16953 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16954 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16955 match(Set dst (ExpandBits src mask));
16956 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16957 format %{ "mov $tsrc, $src\n\t"
16958 "mov $tmask, $mask\n\t"
16959 "bdep $tdst, $tsrc, $tmask\n\t"
16960 "mov $dst, $tdst"
16961 %}
16962 ins_encode %{
16963 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16964 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16965 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16966 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16967 %}
16968 ins_pipe(pipe_slow);
16969 %}
16970
16971
16972 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16973 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16974 match(Set dst (ExpandBits (LoadL mem) mask));
16975 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16976 format %{ "ldrd $tsrc, $mem\n\t"
16977 "ldrd $tmask, $mask\n\t"
16978 "bdep $tdst, $tsrc, $tmask\n\t"
16979 "mov $dst, $tdst"
16980 %}
16981 ins_encode %{
16982 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16983 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16984 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16985 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16986 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16987 %}
16988 ins_pipe(pipe_slow);
16989 %}
16990
16991 // ============================================================================
16992 // This name is KNOWN by the ADLC and cannot be changed.
16993 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
16994 // for this guy.
16995 instruct tlsLoadP(thread_RegP dst)
16996 %{
16997 match(Set dst (ThreadLocal));
16998
16999 ins_cost(0);
17000
17001 format %{ " -- \t// $dst=Thread::current(), empty" %}
17002
17003 size(0);
17004
17005 ins_encode( /*empty*/ );
17006
17007 ins_pipe(pipe_class_empty);
17008 %}
17009
17010 //----------PEEPHOLE RULES-----------------------------------------------------
17011 // These must follow all instruction definitions as they use the names
17012 // defined in the instructions definitions.
17013 //
17014 // peepmatch ( root_instr_name [preceding_instruction]* );
17015 //
17016 // peepconstraint %{
17017 // (instruction_number.operand_name relational_op instruction_number.operand_name
17018 // [, ...] );
17019 // // instruction numbers are zero-based using left to right order in peepmatch
17020 //
17021 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17022 // // provide an instruction_number.operand_name for each operand that appears
17023 // // in the replacement instruction's match rule
17024 //
17025 // ---------VM FLAGS---------------------------------------------------------
17026 //
17027 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17028 //
17029 // Each peephole rule is given an identifying number starting with zero and
17030 // increasing by one in the order seen by the parser. An individual peephole
17031 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17032 // on the command-line.
17033 //
17034 // ---------CURRENT LIMITATIONS----------------------------------------------
17035 //
17036 // Only match adjacent instructions in same basic block
17037 // Only equality constraints
17038 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17039 // Only one replacement instruction
17040 //
17041 // ---------EXAMPLE----------------------------------------------------------
17042 //
17043 // // pertinent parts of existing instructions in architecture description
17044 // instruct movI(iRegINoSp dst, iRegI src)
17045 // %{
17046 // match(Set dst (CopyI src));
17047 // %}
17048 //
17049 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17050 // %{
17051 // match(Set dst (AddI dst src));
17052 // effect(KILL cr);
17053 // %}
17054 //
17055 // // Change (inc mov) to lea
17056 // peephole %{
17057 // // increment preceded by register-register move
17058 // peepmatch ( incI_iReg movI );
17059 // // require that the destination register of the increment
17060 // // match the destination register of the move
17061 // peepconstraint ( 0.dst == 1.dst );
17062 // // construct a replacement instruction that sets
17063 // // the destination to ( move's source register + one )
17064 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17065 // %}
17066 //
17067
17068 // Implementation no longer uses movX instructions since
17069 // machine-independent system no longer uses CopyX nodes.
17070 //
17071 // peephole
17072 // %{
17073 // peepmatch (incI_iReg movI);
17074 // peepconstraint (0.dst == 1.dst);
17075 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17076 // %}
17077
17078 // peephole
17079 // %{
17080 // peepmatch (decI_iReg movI);
17081 // peepconstraint (0.dst == 1.dst);
17082 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17083 // %}
17084
17085 // peephole
17086 // %{
17087 // peepmatch (addI_iReg_imm movI);
17088 // peepconstraint (0.dst == 1.dst);
17089 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17090 // %}
17091
17092 // peephole
17093 // %{
17094 // peepmatch (incL_iReg movL);
17095 // peepconstraint (0.dst == 1.dst);
17096 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17097 // %}
17098
17099 // peephole
17100 // %{
17101 // peepmatch (decL_iReg movL);
17102 // peepconstraint (0.dst == 1.dst);
17103 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17104 // %}
17105
17106 // peephole
17107 // %{
17108 // peepmatch (addL_iReg_imm movL);
17109 // peepconstraint (0.dst == 1.dst);
17110 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17111 // %}
17112
17113 // peephole
17114 // %{
17115 // peepmatch (addP_iReg_imm movP);
17116 // peepconstraint (0.dst == 1.dst);
17117 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17118 // %}
17119
17120 // // Change load of spilled value to only a spill
17121 // instruct storeI(memory mem, iRegI src)
17122 // %{
17123 // match(Set mem (StoreI mem src));
17124 // %}
17125 //
17126 // instruct loadI(iRegINoSp dst, memory mem)
17127 // %{
17128 // match(Set dst (LoadI mem));
17129 // %}
17130 //
17131
17132 //----------SMARTSPILL RULES---------------------------------------------------
17133 // These must follow all instruction definitions as they use the names
17134 // defined in the instructions definitions.
17135
17136 // Local Variables:
17137 // mode: c++
17138 // End: