1 //
2 // Copyright (c) 2003, 2024, Oracle and/or its affiliates. All rights reserved.
3 // Copyright (c) 2014, 2024, Red Hat, Inc. All rights reserved.
4 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
5 //
6 // This code is free software; you can redistribute it and/or modify it
7 // under the terms of the GNU General Public License version 2 only, as
8 // published by the Free Software Foundation.
9 //
10 // This code is distributed in the hope that it will be useful, but WITHOUT
11 // ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 // FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
13 // version 2 for more details (a copy is included in the LICENSE file that
14 // accompanied this code).
15 //
16 // You should have received a copy of the GNU General Public License version
17 // 2 along with this work; if not, write to the Free Software Foundation,
18 // Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19 //
20 // Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
21 // or visit www.oracle.com if you need additional information or have any
22 // questions.
23 //
24 //
25
26 // AArch64 Architecture Description File
27
28 //----------REGISTER DEFINITION BLOCK------------------------------------------
29 // This information is used by the matcher and the register allocator to
30 // describe individual registers and classes of registers within the target
31 // architecture.
32
33 register %{
34 //----------Architecture Description Register Definitions----------------------
35 // General Registers
36 // "reg_def" name ( register save type, C convention save type,
37 // ideal register type, encoding );
38 // Register Save Types:
39 //
40 // NS = No-Save: The register allocator assumes that these registers
41 // can be used without saving upon entry to the method, &
42 // that they do not need to be saved at call sites.
43 //
44 // SOC = Save-On-Call: The register allocator assumes that these registers
45 // can be used without saving upon entry to the method,
46 // but that they must be saved at call sites.
47 //
48 // SOE = Save-On-Entry: The register allocator assumes that these registers
49 // must be saved before using them upon entry to the
50 // method, but they do not need to be saved at call
51 // sites.
52 //
53 // AS = Always-Save: The register allocator assumes that these registers
54 // must be saved before using them upon entry to the
55 // method, & that they must be saved at call sites.
56 //
57 // Ideal Register Type is used to determine how to save & restore a
58 // register. Op_RegI will get spilled with LoadI/StoreI, Op_RegP will get
59 // spilled with LoadP/StoreP. If the register supports both, use Op_RegI.
60 //
61 // The encoding number is the actual bit-pattern placed into the opcodes.
62
63 // We must define the 64 bit int registers in two 32 bit halves, the
64 // real lower register and a virtual upper half register. upper halves
65 // are used by the register allocator but are not actually supplied as
66 // operands to memory ops.
67 //
68 // follow the C1 compiler in making registers
69 //
70 // r0-r7,r10-r26 volatile (caller save)
71 // r27-r32 system (no save, no allocate)
72 // r8-r9 non-allocatable (so we can use them as scratch regs)
73 //
74 // as regards Java usage. we don't use any callee save registers
75 // because this makes it difficult to de-optimise a frame (see comment
76 // in x86 implementation of Deoptimization::unwind_callee_save_values)
77 //
78
79 // General Registers
80
81 reg_def R0 ( SOC, SOC, Op_RegI, 0, r0->as_VMReg() );
82 reg_def R0_H ( SOC, SOC, Op_RegI, 0, r0->as_VMReg()->next() );
83 reg_def R1 ( SOC, SOC, Op_RegI, 1, r1->as_VMReg() );
84 reg_def R1_H ( SOC, SOC, Op_RegI, 1, r1->as_VMReg()->next() );
85 reg_def R2 ( SOC, SOC, Op_RegI, 2, r2->as_VMReg() );
86 reg_def R2_H ( SOC, SOC, Op_RegI, 2, r2->as_VMReg()->next() );
87 reg_def R3 ( SOC, SOC, Op_RegI, 3, r3->as_VMReg() );
88 reg_def R3_H ( SOC, SOC, Op_RegI, 3, r3->as_VMReg()->next() );
89 reg_def R4 ( SOC, SOC, Op_RegI, 4, r4->as_VMReg() );
90 reg_def R4_H ( SOC, SOC, Op_RegI, 4, r4->as_VMReg()->next() );
91 reg_def R5 ( SOC, SOC, Op_RegI, 5, r5->as_VMReg() );
92 reg_def R5_H ( SOC, SOC, Op_RegI, 5, r5->as_VMReg()->next() );
93 reg_def R6 ( SOC, SOC, Op_RegI, 6, r6->as_VMReg() );
94 reg_def R6_H ( SOC, SOC, Op_RegI, 6, r6->as_VMReg()->next() );
95 reg_def R7 ( SOC, SOC, Op_RegI, 7, r7->as_VMReg() );
96 reg_def R7_H ( SOC, SOC, Op_RegI, 7, r7->as_VMReg()->next() );
97 reg_def R8 ( NS, SOC, Op_RegI, 8, r8->as_VMReg() ); // rscratch1, non-allocatable
98 reg_def R8_H ( NS, SOC, Op_RegI, 8, r8->as_VMReg()->next() );
99 reg_def R9 ( NS, SOC, Op_RegI, 9, r9->as_VMReg() ); // rscratch2, non-allocatable
100 reg_def R9_H ( NS, SOC, Op_RegI, 9, r9->as_VMReg()->next() );
101 reg_def R10 ( SOC, SOC, Op_RegI, 10, r10->as_VMReg() );
102 reg_def R10_H ( SOC, SOC, Op_RegI, 10, r10->as_VMReg()->next());
103 reg_def R11 ( SOC, SOC, Op_RegI, 11, r11->as_VMReg() );
104 reg_def R11_H ( SOC, SOC, Op_RegI, 11, r11->as_VMReg()->next());
105 reg_def R12 ( SOC, SOC, Op_RegI, 12, r12->as_VMReg() );
106 reg_def R12_H ( SOC, SOC, Op_RegI, 12, r12->as_VMReg()->next());
107 reg_def R13 ( SOC, SOC, Op_RegI, 13, r13->as_VMReg() );
108 reg_def R13_H ( SOC, SOC, Op_RegI, 13, r13->as_VMReg()->next());
109 reg_def R14 ( SOC, SOC, Op_RegI, 14, r14->as_VMReg() );
110 reg_def R14_H ( SOC, SOC, Op_RegI, 14, r14->as_VMReg()->next());
111 reg_def R15 ( SOC, SOC, Op_RegI, 15, r15->as_VMReg() );
112 reg_def R15_H ( SOC, SOC, Op_RegI, 15, r15->as_VMReg()->next());
113 reg_def R16 ( SOC, SOC, Op_RegI, 16, r16->as_VMReg() );
114 reg_def R16_H ( SOC, SOC, Op_RegI, 16, r16->as_VMReg()->next());
115 reg_def R17 ( SOC, SOC, Op_RegI, 17, r17->as_VMReg() );
116 reg_def R17_H ( SOC, SOC, Op_RegI, 17, r17->as_VMReg()->next());
117 reg_def R18 ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg() );
118 reg_def R18_H ( SOC, SOC, Op_RegI, 18, r18_tls->as_VMReg()->next());
119 reg_def R19 ( SOC, SOE, Op_RegI, 19, r19->as_VMReg() );
120 reg_def R19_H ( SOC, SOE, Op_RegI, 19, r19->as_VMReg()->next());
121 reg_def R20 ( SOC, SOE, Op_RegI, 20, r20->as_VMReg() ); // caller esp
122 reg_def R20_H ( SOC, SOE, Op_RegI, 20, r20->as_VMReg()->next());
123 reg_def R21 ( SOC, SOE, Op_RegI, 21, r21->as_VMReg() );
124 reg_def R21_H ( SOC, SOE, Op_RegI, 21, r21->as_VMReg()->next());
125 reg_def R22 ( SOC, SOE, Op_RegI, 22, r22->as_VMReg() );
126 reg_def R22_H ( SOC, SOE, Op_RegI, 22, r22->as_VMReg()->next());
127 reg_def R23 ( SOC, SOE, Op_RegI, 23, r23->as_VMReg() );
128 reg_def R23_H ( SOC, SOE, Op_RegI, 23, r23->as_VMReg()->next());
129 reg_def R24 ( SOC, SOE, Op_RegI, 24, r24->as_VMReg() );
130 reg_def R24_H ( SOC, SOE, Op_RegI, 24, r24->as_VMReg()->next());
131 reg_def R25 ( SOC, SOE, Op_RegI, 25, r25->as_VMReg() );
132 reg_def R25_H ( SOC, SOE, Op_RegI, 25, r25->as_VMReg()->next());
133 reg_def R26 ( SOC, SOE, Op_RegI, 26, r26->as_VMReg() );
134 reg_def R26_H ( SOC, SOE, Op_RegI, 26, r26->as_VMReg()->next());
135 reg_def R27 ( SOC, SOE, Op_RegI, 27, r27->as_VMReg() ); // heapbase
136 reg_def R27_H ( SOC, SOE, Op_RegI, 27, r27->as_VMReg()->next());
137 reg_def R28 ( NS, SOE, Op_RegI, 28, r28->as_VMReg() ); // thread
138 reg_def R28_H ( NS, SOE, Op_RegI, 28, r28->as_VMReg()->next());
139 reg_def R29 ( NS, NS, Op_RegI, 29, r29->as_VMReg() ); // fp
140 reg_def R29_H ( NS, NS, Op_RegI, 29, r29->as_VMReg()->next());
141 reg_def R30 ( NS, NS, Op_RegI, 30, r30->as_VMReg() ); // lr
142 reg_def R30_H ( NS, NS, Op_RegI, 30, r30->as_VMReg()->next());
143 reg_def R31 ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg() ); // sp
144 reg_def R31_H ( NS, NS, Op_RegI, 31, r31_sp->as_VMReg()->next());
145
146 // ----------------------------
147 // Float/Double/Vector Registers
148 // ----------------------------
149
150 // Double Registers
151
152 // The rules of ADL require that double registers be defined in pairs.
153 // Each pair must be two 32-bit values, but not necessarily a pair of
154 // single float registers. In each pair, ADLC-assigned register numbers
155 // must be adjacent, with the lower number even. Finally, when the
156 // CPU stores such a register pair to memory, the word associated with
157 // the lower ADLC-assigned number must be stored to the lower address.
158
159 // AArch64 has 32 floating-point registers. Each can store a vector of
160 // single or double precision floating-point values up to 8 * 32
161 // floats, 4 * 64 bit floats or 2 * 128 bit floats. We currently only
162 // use the first float or double element of the vector.
163
164 // for Java use float registers v0-v15 are always save on call whereas
165 // the platform ABI treats v8-v15 as callee save). float registers
166 // v16-v31 are SOC as per the platform spec
167
168 // For SVE vector registers, we simply extend vector register size to 8
169 // 'logical' slots. This is nominally 256 bits but it actually covers
170 // all possible 'physical' SVE vector register lengths from 128 ~ 2048
171 // bits. The 'physical' SVE vector register length is detected during
172 // startup, so the register allocator is able to identify the correct
173 // number of bytes needed for an SVE spill/unspill.
174 // Note that a vector register with 4 slots denotes a 128-bit NEON
175 // register allowing it to be distinguished from the corresponding SVE
176 // vector register when the SVE vector length is 128 bits.
177
178 reg_def V0 ( SOC, SOC, Op_RegF, 0, v0->as_VMReg() );
179 reg_def V0_H ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next() );
180 reg_def V0_J ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(2) );
181 reg_def V0_K ( SOC, SOC, Op_RegF, 0, v0->as_VMReg()->next(3) );
182
183 reg_def V1 ( SOC, SOC, Op_RegF, 1, v1->as_VMReg() );
184 reg_def V1_H ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next() );
185 reg_def V1_J ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(2) );
186 reg_def V1_K ( SOC, SOC, Op_RegF, 1, v1->as_VMReg()->next(3) );
187
188 reg_def V2 ( SOC, SOC, Op_RegF, 2, v2->as_VMReg() );
189 reg_def V2_H ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next() );
190 reg_def V2_J ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(2) );
191 reg_def V2_K ( SOC, SOC, Op_RegF, 2, v2->as_VMReg()->next(3) );
192
193 reg_def V3 ( SOC, SOC, Op_RegF, 3, v3->as_VMReg() );
194 reg_def V3_H ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next() );
195 reg_def V3_J ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(2) );
196 reg_def V3_K ( SOC, SOC, Op_RegF, 3, v3->as_VMReg()->next(3) );
197
198 reg_def V4 ( SOC, SOC, Op_RegF, 4, v4->as_VMReg() );
199 reg_def V4_H ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next() );
200 reg_def V4_J ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(2) );
201 reg_def V4_K ( SOC, SOC, Op_RegF, 4, v4->as_VMReg()->next(3) );
202
203 reg_def V5 ( SOC, SOC, Op_RegF, 5, v5->as_VMReg() );
204 reg_def V5_H ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next() );
205 reg_def V5_J ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(2) );
206 reg_def V5_K ( SOC, SOC, Op_RegF, 5, v5->as_VMReg()->next(3) );
207
208 reg_def V6 ( SOC, SOC, Op_RegF, 6, v6->as_VMReg() );
209 reg_def V6_H ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next() );
210 reg_def V6_J ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(2) );
211 reg_def V6_K ( SOC, SOC, Op_RegF, 6, v6->as_VMReg()->next(3) );
212
213 reg_def V7 ( SOC, SOC, Op_RegF, 7, v7->as_VMReg() );
214 reg_def V7_H ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next() );
215 reg_def V7_J ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(2) );
216 reg_def V7_K ( SOC, SOC, Op_RegF, 7, v7->as_VMReg()->next(3) );
217
218 reg_def V8 ( SOC, SOE, Op_RegF, 8, v8->as_VMReg() );
219 reg_def V8_H ( SOC, SOE, Op_RegF, 8, v8->as_VMReg()->next() );
220 reg_def V8_J ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(2) );
221 reg_def V8_K ( SOC, SOC, Op_RegF, 8, v8->as_VMReg()->next(3) );
222
223 reg_def V9 ( SOC, SOE, Op_RegF, 9, v9->as_VMReg() );
224 reg_def V9_H ( SOC, SOE, Op_RegF, 9, v9->as_VMReg()->next() );
225 reg_def V9_J ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(2) );
226 reg_def V9_K ( SOC, SOC, Op_RegF, 9, v9->as_VMReg()->next(3) );
227
228 reg_def V10 ( SOC, SOE, Op_RegF, 10, v10->as_VMReg() );
229 reg_def V10_H ( SOC, SOE, Op_RegF, 10, v10->as_VMReg()->next() );
230 reg_def V10_J ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(2) );
231 reg_def V10_K ( SOC, SOC, Op_RegF, 10, v10->as_VMReg()->next(3) );
232
233 reg_def V11 ( SOC, SOE, Op_RegF, 11, v11->as_VMReg() );
234 reg_def V11_H ( SOC, SOE, Op_RegF, 11, v11->as_VMReg()->next() );
235 reg_def V11_J ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(2) );
236 reg_def V11_K ( SOC, SOC, Op_RegF, 11, v11->as_VMReg()->next(3) );
237
238 reg_def V12 ( SOC, SOE, Op_RegF, 12, v12->as_VMReg() );
239 reg_def V12_H ( SOC, SOE, Op_RegF, 12, v12->as_VMReg()->next() );
240 reg_def V12_J ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(2) );
241 reg_def V12_K ( SOC, SOC, Op_RegF, 12, v12->as_VMReg()->next(3) );
242
243 reg_def V13 ( SOC, SOE, Op_RegF, 13, v13->as_VMReg() );
244 reg_def V13_H ( SOC, SOE, Op_RegF, 13, v13->as_VMReg()->next() );
245 reg_def V13_J ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(2) );
246 reg_def V13_K ( SOC, SOC, Op_RegF, 13, v13->as_VMReg()->next(3) );
247
248 reg_def V14 ( SOC, SOE, Op_RegF, 14, v14->as_VMReg() );
249 reg_def V14_H ( SOC, SOE, Op_RegF, 14, v14->as_VMReg()->next() );
250 reg_def V14_J ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(2) );
251 reg_def V14_K ( SOC, SOC, Op_RegF, 14, v14->as_VMReg()->next(3) );
252
253 reg_def V15 ( SOC, SOE, Op_RegF, 15, v15->as_VMReg() );
254 reg_def V15_H ( SOC, SOE, Op_RegF, 15, v15->as_VMReg()->next() );
255 reg_def V15_J ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(2) );
256 reg_def V15_K ( SOC, SOC, Op_RegF, 15, v15->as_VMReg()->next(3) );
257
258 reg_def V16 ( SOC, SOC, Op_RegF, 16, v16->as_VMReg() );
259 reg_def V16_H ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next() );
260 reg_def V16_J ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(2) );
261 reg_def V16_K ( SOC, SOC, Op_RegF, 16, v16->as_VMReg()->next(3) );
262
263 reg_def V17 ( SOC, SOC, Op_RegF, 17, v17->as_VMReg() );
264 reg_def V17_H ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next() );
265 reg_def V17_J ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(2) );
266 reg_def V17_K ( SOC, SOC, Op_RegF, 17, v17->as_VMReg()->next(3) );
267
268 reg_def V18 ( SOC, SOC, Op_RegF, 18, v18->as_VMReg() );
269 reg_def V18_H ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next() );
270 reg_def V18_J ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(2) );
271 reg_def V18_K ( SOC, SOC, Op_RegF, 18, v18->as_VMReg()->next(3) );
272
273 reg_def V19 ( SOC, SOC, Op_RegF, 19, v19->as_VMReg() );
274 reg_def V19_H ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next() );
275 reg_def V19_J ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(2) );
276 reg_def V19_K ( SOC, SOC, Op_RegF, 19, v19->as_VMReg()->next(3) );
277
278 reg_def V20 ( SOC, SOC, Op_RegF, 20, v20->as_VMReg() );
279 reg_def V20_H ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next() );
280 reg_def V20_J ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(2) );
281 reg_def V20_K ( SOC, SOC, Op_RegF, 20, v20->as_VMReg()->next(3) );
282
283 reg_def V21 ( SOC, SOC, Op_RegF, 21, v21->as_VMReg() );
284 reg_def V21_H ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next() );
285 reg_def V21_J ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(2) );
286 reg_def V21_K ( SOC, SOC, Op_RegF, 21, v21->as_VMReg()->next(3) );
287
288 reg_def V22 ( SOC, SOC, Op_RegF, 22, v22->as_VMReg() );
289 reg_def V22_H ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next() );
290 reg_def V22_J ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(2) );
291 reg_def V22_K ( SOC, SOC, Op_RegF, 22, v22->as_VMReg()->next(3) );
292
293 reg_def V23 ( SOC, SOC, Op_RegF, 23, v23->as_VMReg() );
294 reg_def V23_H ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next() );
295 reg_def V23_J ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(2) );
296 reg_def V23_K ( SOC, SOC, Op_RegF, 23, v23->as_VMReg()->next(3) );
297
298 reg_def V24 ( SOC, SOC, Op_RegF, 24, v24->as_VMReg() );
299 reg_def V24_H ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next() );
300 reg_def V24_J ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(2) );
301 reg_def V24_K ( SOC, SOC, Op_RegF, 24, v24->as_VMReg()->next(3) );
302
303 reg_def V25 ( SOC, SOC, Op_RegF, 25, v25->as_VMReg() );
304 reg_def V25_H ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next() );
305 reg_def V25_J ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(2) );
306 reg_def V25_K ( SOC, SOC, Op_RegF, 25, v25->as_VMReg()->next(3) );
307
308 reg_def V26 ( SOC, SOC, Op_RegF, 26, v26->as_VMReg() );
309 reg_def V26_H ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next() );
310 reg_def V26_J ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(2) );
311 reg_def V26_K ( SOC, SOC, Op_RegF, 26, v26->as_VMReg()->next(3) );
312
313 reg_def V27 ( SOC, SOC, Op_RegF, 27, v27->as_VMReg() );
314 reg_def V27_H ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next() );
315 reg_def V27_J ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(2) );
316 reg_def V27_K ( SOC, SOC, Op_RegF, 27, v27->as_VMReg()->next(3) );
317
318 reg_def V28 ( SOC, SOC, Op_RegF, 28, v28->as_VMReg() );
319 reg_def V28_H ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next() );
320 reg_def V28_J ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(2) );
321 reg_def V28_K ( SOC, SOC, Op_RegF, 28, v28->as_VMReg()->next(3) );
322
323 reg_def V29 ( SOC, SOC, Op_RegF, 29, v29->as_VMReg() );
324 reg_def V29_H ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next() );
325 reg_def V29_J ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(2) );
326 reg_def V29_K ( SOC, SOC, Op_RegF, 29, v29->as_VMReg()->next(3) );
327
328 reg_def V30 ( SOC, SOC, Op_RegF, 30, v30->as_VMReg() );
329 reg_def V30_H ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next() );
330 reg_def V30_J ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(2) );
331 reg_def V30_K ( SOC, SOC, Op_RegF, 30, v30->as_VMReg()->next(3) );
332
333 reg_def V31 ( SOC, SOC, Op_RegF, 31, v31->as_VMReg() );
334 reg_def V31_H ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next() );
335 reg_def V31_J ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(2) );
336 reg_def V31_K ( SOC, SOC, Op_RegF, 31, v31->as_VMReg()->next(3) );
337
338 // ----------------------------
339 // SVE Predicate Registers
340 // ----------------------------
341 reg_def P0 (SOC, SOC, Op_RegVectMask, 0, p0->as_VMReg());
342 reg_def P1 (SOC, SOC, Op_RegVectMask, 1, p1->as_VMReg());
343 reg_def P2 (SOC, SOC, Op_RegVectMask, 2, p2->as_VMReg());
344 reg_def P3 (SOC, SOC, Op_RegVectMask, 3, p3->as_VMReg());
345 reg_def P4 (SOC, SOC, Op_RegVectMask, 4, p4->as_VMReg());
346 reg_def P5 (SOC, SOC, Op_RegVectMask, 5, p5->as_VMReg());
347 reg_def P6 (SOC, SOC, Op_RegVectMask, 6, p6->as_VMReg());
348 reg_def P7 (SOC, SOC, Op_RegVectMask, 7, p7->as_VMReg());
349 reg_def P8 (SOC, SOC, Op_RegVectMask, 8, p8->as_VMReg());
350 reg_def P9 (SOC, SOC, Op_RegVectMask, 9, p9->as_VMReg());
351 reg_def P10 (SOC, SOC, Op_RegVectMask, 10, p10->as_VMReg());
352 reg_def P11 (SOC, SOC, Op_RegVectMask, 11, p11->as_VMReg());
353 reg_def P12 (SOC, SOC, Op_RegVectMask, 12, p12->as_VMReg());
354 reg_def P13 (SOC, SOC, Op_RegVectMask, 13, p13->as_VMReg());
355 reg_def P14 (SOC, SOC, Op_RegVectMask, 14, p14->as_VMReg());
356 reg_def P15 (SOC, SOC, Op_RegVectMask, 15, p15->as_VMReg());
357
358 // ----------------------------
359 // Special Registers
360 // ----------------------------
361
362 // the AArch64 CSPR status flag register is not directly accessible as
363 // instruction operand. the FPSR status flag register is a system
364 // register which can be written/read using MSR/MRS but again does not
365 // appear as an operand (a code identifying the FSPR occurs as an
366 // immediate value in the instruction).
367
368 reg_def RFLAGS(SOC, SOC, 0, 32, VMRegImpl::Bad());
369
370 // Specify priority of register selection within phases of register
371 // allocation. Highest priority is first. A useful heuristic is to
372 // give registers a low priority when they are required by machine
373 // instructions, like EAX and EDX on I486, and choose no-save registers
374 // before save-on-call, & save-on-call before save-on-entry. Registers
375 // which participate in fixed calling sequences should come last.
376 // Registers which are used as pairs must fall on an even boundary.
377
378 alloc_class chunk0(
379 // volatiles
380 R10, R10_H,
381 R11, R11_H,
382 R12, R12_H,
383 R13, R13_H,
384 R14, R14_H,
385 R15, R15_H,
386 R16, R16_H,
387 R17, R17_H,
388 R18, R18_H,
389
390 // arg registers
391 R0, R0_H,
392 R1, R1_H,
393 R2, R2_H,
394 R3, R3_H,
395 R4, R4_H,
396 R5, R5_H,
397 R6, R6_H,
398 R7, R7_H,
399
400 // non-volatiles
401 R19, R19_H,
402 R20, R20_H,
403 R21, R21_H,
404 R22, R22_H,
405 R23, R23_H,
406 R24, R24_H,
407 R25, R25_H,
408 R26, R26_H,
409
410 // non-allocatable registers
411
412 R27, R27_H, // heapbase
413 R28, R28_H, // thread
414 R29, R29_H, // fp
415 R30, R30_H, // lr
416 R31, R31_H, // sp
417 R8, R8_H, // rscratch1
418 R9, R9_H, // rscratch2
419 );
420
421 alloc_class chunk1(
422
423 // no save
424 V16, V16_H, V16_J, V16_K,
425 V17, V17_H, V17_J, V17_K,
426 V18, V18_H, V18_J, V18_K,
427 V19, V19_H, V19_J, V19_K,
428 V20, V20_H, V20_J, V20_K,
429 V21, V21_H, V21_J, V21_K,
430 V22, V22_H, V22_J, V22_K,
431 V23, V23_H, V23_J, V23_K,
432 V24, V24_H, V24_J, V24_K,
433 V25, V25_H, V25_J, V25_K,
434 V26, V26_H, V26_J, V26_K,
435 V27, V27_H, V27_J, V27_K,
436 V28, V28_H, V28_J, V28_K,
437 V29, V29_H, V29_J, V29_K,
438 V30, V30_H, V30_J, V30_K,
439 V31, V31_H, V31_J, V31_K,
440
441 // arg registers
442 V0, V0_H, V0_J, V0_K,
443 V1, V1_H, V1_J, V1_K,
444 V2, V2_H, V2_J, V2_K,
445 V3, V3_H, V3_J, V3_K,
446 V4, V4_H, V4_J, V4_K,
447 V5, V5_H, V5_J, V5_K,
448 V6, V6_H, V6_J, V6_K,
449 V7, V7_H, V7_J, V7_K,
450
451 // non-volatiles
452 V8, V8_H, V8_J, V8_K,
453 V9, V9_H, V9_J, V9_K,
454 V10, V10_H, V10_J, V10_K,
455 V11, V11_H, V11_J, V11_K,
456 V12, V12_H, V12_J, V12_K,
457 V13, V13_H, V13_J, V13_K,
458 V14, V14_H, V14_J, V14_K,
459 V15, V15_H, V15_J, V15_K,
460 );
461
462 alloc_class chunk2 (
463 // Governing predicates for load/store and arithmetic
464 P0,
465 P1,
466 P2,
467 P3,
468 P4,
469 P5,
470 P6,
471
472 // Extra predicates
473 P8,
474 P9,
475 P10,
476 P11,
477 P12,
478 P13,
479 P14,
480 P15,
481
482 // Preserved for all-true predicate
483 P7,
484 );
485
486 alloc_class chunk3(RFLAGS);
487
488 //----------Architecture Description Register Classes--------------------------
489 // Several register classes are automatically defined based upon information in
490 // this architecture description.
491 // 1) reg_class inline_cache_reg ( /* as def'd in frame section */ )
492 // 2) reg_class stack_slots( /* one chunk of stack-based "registers" */ )
493 //
494
495 // Class for all 32 bit general purpose registers
496 reg_class all_reg32(
497 R0,
498 R1,
499 R2,
500 R3,
501 R4,
502 R5,
503 R6,
504 R7,
505 R10,
506 R11,
507 R12,
508 R13,
509 R14,
510 R15,
511 R16,
512 R17,
513 R18,
514 R19,
515 R20,
516 R21,
517 R22,
518 R23,
519 R24,
520 R25,
521 R26,
522 R27,
523 R28,
524 R29,
525 R30,
526 R31
527 );
528
529
530 // Class for all 32 bit integer registers (excluding SP which
531 // will never be used as an integer register)
532 reg_class any_reg32 %{
533 return _ANY_REG32_mask;
534 %}
535
536 // Singleton class for R0 int register
537 reg_class int_r0_reg(R0);
538
539 // Singleton class for R2 int register
540 reg_class int_r2_reg(R2);
541
542 // Singleton class for R3 int register
543 reg_class int_r3_reg(R3);
544
545 // Singleton class for R4 int register
546 reg_class int_r4_reg(R4);
547
548 // Singleton class for R31 int register
549 reg_class int_r31_reg(R31);
550
551 // Class for all 64 bit general purpose registers
552 reg_class all_reg(
553 R0, R0_H,
554 R1, R1_H,
555 R2, R2_H,
556 R3, R3_H,
557 R4, R4_H,
558 R5, R5_H,
559 R6, R6_H,
560 R7, R7_H,
561 R10, R10_H,
562 R11, R11_H,
563 R12, R12_H,
564 R13, R13_H,
565 R14, R14_H,
566 R15, R15_H,
567 R16, R16_H,
568 R17, R17_H,
569 R18, R18_H,
570 R19, R19_H,
571 R20, R20_H,
572 R21, R21_H,
573 R22, R22_H,
574 R23, R23_H,
575 R24, R24_H,
576 R25, R25_H,
577 R26, R26_H,
578 R27, R27_H,
579 R28, R28_H,
580 R29, R29_H,
581 R30, R30_H,
582 R31, R31_H
583 );
584
585 // Class for all long integer registers (including SP)
586 reg_class any_reg %{
587 return _ANY_REG_mask;
588 %}
589
590 // Class for non-allocatable 32 bit registers
591 reg_class non_allocatable_reg32(
592 #ifdef R18_RESERVED
593 // See comment in register_aarch64.hpp
594 R18, // tls on Windows
595 #endif
596 R28, // thread
597 R30, // lr
598 R31 // sp
599 );
600
601 // Class for non-allocatable 64 bit registers
602 reg_class non_allocatable_reg(
603 #ifdef R18_RESERVED
604 // See comment in register_aarch64.hpp
605 R18, R18_H, // tls on Windows, platform register on macOS
606 #endif
607 R28, R28_H, // thread
608 R30, R30_H, // lr
609 R31, R31_H // sp
610 );
611
612 // Class for all non-special integer registers
613 reg_class no_special_reg32 %{
614 return _NO_SPECIAL_REG32_mask;
615 %}
616
617 // Class for all non-special long integer registers
618 reg_class no_special_reg %{
619 return _NO_SPECIAL_REG_mask;
620 %}
621
622 // Class for 64 bit register r0
623 reg_class r0_reg(
624 R0, R0_H
625 );
626
627 // Class for 64 bit register r1
628 reg_class r1_reg(
629 R1, R1_H
630 );
631
632 // Class for 64 bit register r2
633 reg_class r2_reg(
634 R2, R2_H
635 );
636
637 // Class for 64 bit register r3
638 reg_class r3_reg(
639 R3, R3_H
640 );
641
642 // Class for 64 bit register r4
643 reg_class r4_reg(
644 R4, R4_H
645 );
646
647 // Class for 64 bit register r5
648 reg_class r5_reg(
649 R5, R5_H
650 );
651
652 // Class for 64 bit register r10
653 reg_class r10_reg(
654 R10, R10_H
655 );
656
657 // Class for 64 bit register r11
658 reg_class r11_reg(
659 R11, R11_H
660 );
661
662 // Class for method register
663 reg_class method_reg(
664 R12, R12_H
665 );
666
667 // Class for thread register
668 reg_class thread_reg(
669 R28, R28_H
670 );
671
672 // Class for frame pointer register
673 reg_class fp_reg(
674 R29, R29_H
675 );
676
677 // Class for link register
678 reg_class lr_reg(
679 R30, R30_H
680 );
681
682 // Class for long sp register
683 reg_class sp_reg(
684 R31, R31_H
685 );
686
687 // Class for all pointer registers
688 reg_class ptr_reg %{
689 return _PTR_REG_mask;
690 %}
691
692 // Class for all non_special pointer registers
693 reg_class no_special_ptr_reg %{
694 return _NO_SPECIAL_PTR_REG_mask;
695 %}
696
697 // Class for all non_special pointer registers (excluding rfp)
698 reg_class no_special_no_rfp_ptr_reg %{
699 return _NO_SPECIAL_NO_RFP_PTR_REG_mask;
700 %}
701
702 // Class for all float registers
703 reg_class float_reg(
704 V0,
705 V1,
706 V2,
707 V3,
708 V4,
709 V5,
710 V6,
711 V7,
712 V8,
713 V9,
714 V10,
715 V11,
716 V12,
717 V13,
718 V14,
719 V15,
720 V16,
721 V17,
722 V18,
723 V19,
724 V20,
725 V21,
726 V22,
727 V23,
728 V24,
729 V25,
730 V26,
731 V27,
732 V28,
733 V29,
734 V30,
735 V31
736 );
737
738 // Double precision float registers have virtual `high halves' that
739 // are needed by the allocator.
740 // Class for all double registers
741 reg_class double_reg(
742 V0, V0_H,
743 V1, V1_H,
744 V2, V2_H,
745 V3, V3_H,
746 V4, V4_H,
747 V5, V5_H,
748 V6, V6_H,
749 V7, V7_H,
750 V8, V8_H,
751 V9, V9_H,
752 V10, V10_H,
753 V11, V11_H,
754 V12, V12_H,
755 V13, V13_H,
756 V14, V14_H,
757 V15, V15_H,
758 V16, V16_H,
759 V17, V17_H,
760 V18, V18_H,
761 V19, V19_H,
762 V20, V20_H,
763 V21, V21_H,
764 V22, V22_H,
765 V23, V23_H,
766 V24, V24_H,
767 V25, V25_H,
768 V26, V26_H,
769 V27, V27_H,
770 V28, V28_H,
771 V29, V29_H,
772 V30, V30_H,
773 V31, V31_H
774 );
775
776 // Class for all SVE vector registers.
777 reg_class vectora_reg (
778 V0, V0_H, V0_J, V0_K,
779 V1, V1_H, V1_J, V1_K,
780 V2, V2_H, V2_J, V2_K,
781 V3, V3_H, V3_J, V3_K,
782 V4, V4_H, V4_J, V4_K,
783 V5, V5_H, V5_J, V5_K,
784 V6, V6_H, V6_J, V6_K,
785 V7, V7_H, V7_J, V7_K,
786 V8, V8_H, V8_J, V8_K,
787 V9, V9_H, V9_J, V9_K,
788 V10, V10_H, V10_J, V10_K,
789 V11, V11_H, V11_J, V11_K,
790 V12, V12_H, V12_J, V12_K,
791 V13, V13_H, V13_J, V13_K,
792 V14, V14_H, V14_J, V14_K,
793 V15, V15_H, V15_J, V15_K,
794 V16, V16_H, V16_J, V16_K,
795 V17, V17_H, V17_J, V17_K,
796 V18, V18_H, V18_J, V18_K,
797 V19, V19_H, V19_J, V19_K,
798 V20, V20_H, V20_J, V20_K,
799 V21, V21_H, V21_J, V21_K,
800 V22, V22_H, V22_J, V22_K,
801 V23, V23_H, V23_J, V23_K,
802 V24, V24_H, V24_J, V24_K,
803 V25, V25_H, V25_J, V25_K,
804 V26, V26_H, V26_J, V26_K,
805 V27, V27_H, V27_J, V27_K,
806 V28, V28_H, V28_J, V28_K,
807 V29, V29_H, V29_J, V29_K,
808 V30, V30_H, V30_J, V30_K,
809 V31, V31_H, V31_J, V31_K,
810 );
811
812 // Class for all 64bit vector registers
813 reg_class vectord_reg(
814 V0, V0_H,
815 V1, V1_H,
816 V2, V2_H,
817 V3, V3_H,
818 V4, V4_H,
819 V5, V5_H,
820 V6, V6_H,
821 V7, V7_H,
822 V8, V8_H,
823 V9, V9_H,
824 V10, V10_H,
825 V11, V11_H,
826 V12, V12_H,
827 V13, V13_H,
828 V14, V14_H,
829 V15, V15_H,
830 V16, V16_H,
831 V17, V17_H,
832 V18, V18_H,
833 V19, V19_H,
834 V20, V20_H,
835 V21, V21_H,
836 V22, V22_H,
837 V23, V23_H,
838 V24, V24_H,
839 V25, V25_H,
840 V26, V26_H,
841 V27, V27_H,
842 V28, V28_H,
843 V29, V29_H,
844 V30, V30_H,
845 V31, V31_H
846 );
847
848 // Class for all 128bit vector registers
849 reg_class vectorx_reg(
850 V0, V0_H, V0_J, V0_K,
851 V1, V1_H, V1_J, V1_K,
852 V2, V2_H, V2_J, V2_K,
853 V3, V3_H, V3_J, V3_K,
854 V4, V4_H, V4_J, V4_K,
855 V5, V5_H, V5_J, V5_K,
856 V6, V6_H, V6_J, V6_K,
857 V7, V7_H, V7_J, V7_K,
858 V8, V8_H, V8_J, V8_K,
859 V9, V9_H, V9_J, V9_K,
860 V10, V10_H, V10_J, V10_K,
861 V11, V11_H, V11_J, V11_K,
862 V12, V12_H, V12_J, V12_K,
863 V13, V13_H, V13_J, V13_K,
864 V14, V14_H, V14_J, V14_K,
865 V15, V15_H, V15_J, V15_K,
866 V16, V16_H, V16_J, V16_K,
867 V17, V17_H, V17_J, V17_K,
868 V18, V18_H, V18_J, V18_K,
869 V19, V19_H, V19_J, V19_K,
870 V20, V20_H, V20_J, V20_K,
871 V21, V21_H, V21_J, V21_K,
872 V22, V22_H, V22_J, V22_K,
873 V23, V23_H, V23_J, V23_K,
874 V24, V24_H, V24_J, V24_K,
875 V25, V25_H, V25_J, V25_K,
876 V26, V26_H, V26_J, V26_K,
877 V27, V27_H, V27_J, V27_K,
878 V28, V28_H, V28_J, V28_K,
879 V29, V29_H, V29_J, V29_K,
880 V30, V30_H, V30_J, V30_K,
881 V31, V31_H, V31_J, V31_K
882 );
883
884 // Class for 128 bit register v0
885 reg_class v0_reg(
886 V0, V0_H
887 );
888
889 // Class for 128 bit register v1
890 reg_class v1_reg(
891 V1, V1_H
892 );
893
894 // Class for 128 bit register v2
895 reg_class v2_reg(
896 V2, V2_H
897 );
898
899 // Class for 128 bit register v3
900 reg_class v3_reg(
901 V3, V3_H
902 );
903
904 // Class for 128 bit register v4
905 reg_class v4_reg(
906 V4, V4_H
907 );
908
909 // Class for 128 bit register v5
910 reg_class v5_reg(
911 V5, V5_H
912 );
913
914 // Class for 128 bit register v6
915 reg_class v6_reg(
916 V6, V6_H
917 );
918
919 // Class for 128 bit register v7
920 reg_class v7_reg(
921 V7, V7_H
922 );
923
924 // Class for 128 bit register v8
925 reg_class v8_reg(
926 V8, V8_H
927 );
928
929 // Class for 128 bit register v9
930 reg_class v9_reg(
931 V9, V9_H
932 );
933
934 // Class for 128 bit register v10
935 reg_class v10_reg(
936 V10, V10_H
937 );
938
939 // Class for 128 bit register v11
940 reg_class v11_reg(
941 V11, V11_H
942 );
943
944 // Class for 128 bit register v12
945 reg_class v12_reg(
946 V12, V12_H
947 );
948
949 // Class for 128 bit register v13
950 reg_class v13_reg(
951 V13, V13_H
952 );
953
954 // Class for 128 bit register v14
955 reg_class v14_reg(
956 V14, V14_H
957 );
958
959 // Class for 128 bit register v15
960 reg_class v15_reg(
961 V15, V15_H
962 );
963
964 // Class for 128 bit register v16
965 reg_class v16_reg(
966 V16, V16_H
967 );
968
969 // Class for 128 bit register v17
970 reg_class v17_reg(
971 V17, V17_H
972 );
973
974 // Class for 128 bit register v18
975 reg_class v18_reg(
976 V18, V18_H
977 );
978
979 // Class for 128 bit register v19
980 reg_class v19_reg(
981 V19, V19_H
982 );
983
984 // Class for 128 bit register v20
985 reg_class v20_reg(
986 V20, V20_H
987 );
988
989 // Class for 128 bit register v21
990 reg_class v21_reg(
991 V21, V21_H
992 );
993
994 // Class for 128 bit register v22
995 reg_class v22_reg(
996 V22, V22_H
997 );
998
999 // Class for 128 bit register v23
1000 reg_class v23_reg(
1001 V23, V23_H
1002 );
1003
1004 // Class for 128 bit register v24
1005 reg_class v24_reg(
1006 V24, V24_H
1007 );
1008
1009 // Class for 128 bit register v25
1010 reg_class v25_reg(
1011 V25, V25_H
1012 );
1013
1014 // Class for 128 bit register v26
1015 reg_class v26_reg(
1016 V26, V26_H
1017 );
1018
1019 // Class for 128 bit register v27
1020 reg_class v27_reg(
1021 V27, V27_H
1022 );
1023
1024 // Class for 128 bit register v28
1025 reg_class v28_reg(
1026 V28, V28_H
1027 );
1028
1029 // Class for 128 bit register v29
1030 reg_class v29_reg(
1031 V29, V29_H
1032 );
1033
1034 // Class for 128 bit register v30
1035 reg_class v30_reg(
1036 V30, V30_H
1037 );
1038
1039 // Class for 128 bit register v31
1040 reg_class v31_reg(
1041 V31, V31_H
1042 );
1043
1044 // Class for all SVE predicate registers.
1045 reg_class pr_reg (
1046 P0,
1047 P1,
1048 P2,
1049 P3,
1050 P4,
1051 P5,
1052 P6,
1053 // P7, non-allocatable, preserved with all elements preset to TRUE.
1054 P8,
1055 P9,
1056 P10,
1057 P11,
1058 P12,
1059 P13,
1060 P14,
1061 P15
1062 );
1063
1064 // Class for SVE governing predicate registers, which are used
1065 // to determine the active elements of a predicated instruction.
1066 reg_class gov_pr (
1067 P0,
1068 P1,
1069 P2,
1070 P3,
1071 P4,
1072 P5,
1073 P6,
1074 // P7, non-allocatable, preserved with all elements preset to TRUE.
1075 );
1076
1077 reg_class p0_reg(P0);
1078 reg_class p1_reg(P1);
1079
1080 // Singleton class for condition codes
1081 reg_class int_flags(RFLAGS);
1082
1083 %}
1084
1085 //----------DEFINITION BLOCK---------------------------------------------------
1086 // Define name --> value mappings to inform the ADLC of an integer valued name
1087 // Current support includes integer values in the range [0, 0x7FFFFFFF]
1088 // Format:
1089 // int_def <name> ( <int_value>, <expression>);
1090 // Generated Code in ad_<arch>.hpp
1091 // #define <name> (<expression>)
1092 // // value == <int_value>
1093 // Generated code in ad_<arch>.cpp adlc_verification()
1094 // assert( <name> == <int_value>, "Expect (<expression>) to equal <int_value>");
1095 //
1096
1097 // we follow the ppc-aix port in using a simple cost model which ranks
1098 // register operations as cheap, memory ops as more expensive and
1099 // branches as most expensive. the first two have a low as well as a
1100 // normal cost. huge cost appears to be a way of saying don't do
1101 // something
1102
1103 definitions %{
1104 // The default cost (of a register move instruction).
1105 int_def INSN_COST ( 100, 100);
1106 int_def BRANCH_COST ( 200, 2 * INSN_COST);
1107 int_def CALL_COST ( 200, 2 * INSN_COST);
1108 int_def VOLATILE_REF_COST ( 1000, 10 * INSN_COST);
1109 %}
1110
1111
1112 //----------SOURCE BLOCK-------------------------------------------------------
1113 // This is a block of C++ code which provides values, functions, and
1114 // definitions necessary in the rest of the architecture description
1115
1116 source_hpp %{
1117
1118 #include "asm/macroAssembler.hpp"
1119 #include "gc/shared/barrierSetAssembler.hpp"
1120 #include "gc/shared/cardTable.hpp"
1121 #include "gc/shared/cardTableBarrierSet.hpp"
1122 #include "gc/shared/collectedHeap.hpp"
1123 #include "opto/addnode.hpp"
1124 #include "opto/convertnode.hpp"
1125 #include "runtime/objectMonitor.hpp"
1126
1127 extern RegMask _ANY_REG32_mask;
1128 extern RegMask _ANY_REG_mask;
1129 extern RegMask _PTR_REG_mask;
1130 extern RegMask _NO_SPECIAL_REG32_mask;
1131 extern RegMask _NO_SPECIAL_REG_mask;
1132 extern RegMask _NO_SPECIAL_PTR_REG_mask;
1133 extern RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1134
1135 class CallStubImpl {
1136
1137 //--------------------------------------------------------------
1138 //---< Used for optimization in Compile::shorten_branches >---
1139 //--------------------------------------------------------------
1140
1141 public:
1142 // Size of call trampoline stub.
1143 static uint size_call_trampoline() {
1144 return 0; // no call trampolines on this platform
1145 }
1146
1147 // number of relocations needed by a call trampoline stub
1148 static uint reloc_call_trampoline() {
1149 return 0; // no call trampolines on this platform
1150 }
1151 };
1152
1153 class HandlerImpl {
1154
1155 public:
1156
1157 static int emit_exception_handler(C2_MacroAssembler *masm);
1158 static int emit_deopt_handler(C2_MacroAssembler* masm);
1159
1160 static uint size_exception_handler() {
1161 return MacroAssembler::far_codestub_branch_size();
1162 }
1163
1164 static uint size_deopt_handler() {
1165 // count one adr and one far branch instruction
1166 return NativeInstruction::instruction_size + MacroAssembler::far_codestub_branch_size();
1167 }
1168 };
1169
1170 class Node::PD {
1171 public:
1172 enum NodeFlags {
1173 _last_flag = Node::_last_flag
1174 };
1175 };
1176
1177 bool is_CAS(int opcode, bool maybe_volatile);
1178
1179 // predicates controlling emit of ldr<x>/ldar<x> and associated dmb
1180
1181 bool unnecessary_acquire(const Node *barrier);
1182 bool needs_acquiring_load(const Node *load);
1183
1184 // predicates controlling emit of str<x>/stlr<x> and associated dmbs
1185
1186 bool unnecessary_release(const Node *barrier);
1187 bool unnecessary_volatile(const Node *barrier);
1188 bool needs_releasing_store(const Node *store);
1189
1190 // predicate controlling translation of CompareAndSwapX
1191 bool needs_acquiring_load_exclusive(const Node *load);
1192
1193 // predicate controlling addressing modes
1194 bool size_fits_all_mem_uses(AddPNode* addp, int shift);
1195
1196 // Convert BootTest condition to Assembler condition.
1197 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
1198 Assembler::Condition to_assembler_cond(BoolTest::mask cond);
1199 %}
1200
1201 source %{
1202
1203 // Derived RegMask with conditionally allocatable registers
1204
1205 void PhaseOutput::pd_perform_mach_node_analysis() {
1206 }
1207
1208 int MachNode::pd_alignment_required() const {
1209 return 1;
1210 }
1211
1212 int MachNode::compute_padding(int current_offset) const {
1213 return 0;
1214 }
1215
1216 RegMask _ANY_REG32_mask;
1217 RegMask _ANY_REG_mask;
1218 RegMask _PTR_REG_mask;
1219 RegMask _NO_SPECIAL_REG32_mask;
1220 RegMask _NO_SPECIAL_REG_mask;
1221 RegMask _NO_SPECIAL_PTR_REG_mask;
1222 RegMask _NO_SPECIAL_NO_RFP_PTR_REG_mask;
1223
1224 void reg_mask_init() {
1225 // We derive below RegMask(s) from the ones which are auto-generated from
1226 // adlc register classes to make AArch64 rheapbase (r27) and rfp (r29)
1227 // registers conditionally reserved.
1228
1229 _ANY_REG32_mask = _ALL_REG32_mask;
1230 _ANY_REG32_mask.Remove(OptoReg::as_OptoReg(r31_sp->as_VMReg()));
1231
1232 _ANY_REG_mask = _ALL_REG_mask;
1233
1234 _PTR_REG_mask = _ALL_REG_mask;
1235
1236 _NO_SPECIAL_REG32_mask = _ALL_REG32_mask;
1237 _NO_SPECIAL_REG32_mask.SUBTRACT(_NON_ALLOCATABLE_REG32_mask);
1238
1239 _NO_SPECIAL_REG_mask = _ALL_REG_mask;
1240 _NO_SPECIAL_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1241
1242 _NO_SPECIAL_PTR_REG_mask = _ALL_REG_mask;
1243 _NO_SPECIAL_PTR_REG_mask.SUBTRACT(_NON_ALLOCATABLE_REG_mask);
1244
1245 // r27 is not allocatable when compressed oops is on and heapbase is not
1246 // zero, compressed klass pointers doesn't use r27 after JDK-8234794
1247 if (UseCompressedOops && (CompressedOops::base() != nullptr)) {
1248 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1249 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1250 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r27->as_VMReg()));
1251 }
1252
1253 // r29 is not allocatable when PreserveFramePointer is on
1254 if (PreserveFramePointer) {
1255 _NO_SPECIAL_REG32_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1256 _NO_SPECIAL_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1257 _NO_SPECIAL_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1258 }
1259
1260 _NO_SPECIAL_NO_RFP_PTR_REG_mask = _NO_SPECIAL_PTR_REG_mask;
1261 _NO_SPECIAL_NO_RFP_PTR_REG_mask.Remove(OptoReg::as_OptoReg(r29->as_VMReg()));
1262 }
1263
1264 // Optimizaton of volatile gets and puts
1265 // -------------------------------------
1266 //
1267 // AArch64 has ldar<x> and stlr<x> instructions which we can safely
1268 // use to implement volatile reads and writes. For a volatile read
1269 // we simply need
1270 //
1271 // ldar<x>
1272 //
1273 // and for a volatile write we need
1274 //
1275 // stlr<x>
1276 //
1277 // Alternatively, we can implement them by pairing a normal
1278 // load/store with a memory barrier. For a volatile read we need
1279 //
1280 // ldr<x>
1281 // dmb ishld
1282 //
1283 // for a volatile write
1284 //
1285 // dmb ish
1286 // str<x>
1287 // dmb ish
1288 //
1289 // We can also use ldaxr and stlxr to implement compare and swap CAS
1290 // sequences. These are normally translated to an instruction
1291 // sequence like the following
1292 //
1293 // dmb ish
1294 // retry:
1295 // ldxr<x> rval raddr
1296 // cmp rval rold
1297 // b.ne done
1298 // stlxr<x> rval, rnew, rold
1299 // cbnz rval retry
1300 // done:
1301 // cset r0, eq
1302 // dmb ishld
1303 //
1304 // Note that the exclusive store is already using an stlxr
1305 // instruction. That is required to ensure visibility to other
1306 // threads of the exclusive write (assuming it succeeds) before that
1307 // of any subsequent writes.
1308 //
1309 // The following instruction sequence is an improvement on the above
1310 //
1311 // retry:
1312 // ldaxr<x> rval raddr
1313 // cmp rval rold
1314 // b.ne done
1315 // stlxr<x> rval, rnew, rold
1316 // cbnz rval retry
1317 // done:
1318 // cset r0, eq
1319 //
1320 // We don't need the leading dmb ish since the stlxr guarantees
1321 // visibility of prior writes in the case that the swap is
1322 // successful. Crucially we don't have to worry about the case where
1323 // the swap is not successful since no valid program should be
1324 // relying on visibility of prior changes by the attempting thread
1325 // in the case where the CAS fails.
1326 //
1327 // Similarly, we don't need the trailing dmb ishld if we substitute
1328 // an ldaxr instruction since that will provide all the guarantees we
1329 // require regarding observation of changes made by other threads
1330 // before any change to the CAS address observed by the load.
1331 //
1332 // In order to generate the desired instruction sequence we need to
1333 // be able to identify specific 'signature' ideal graph node
1334 // sequences which i) occur as a translation of a volatile reads or
1335 // writes or CAS operations and ii) do not occur through any other
1336 // translation or graph transformation. We can then provide
1337 // alternative aldc matching rules which translate these node
1338 // sequences to the desired machine code sequences. Selection of the
1339 // alternative rules can be implemented by predicates which identify
1340 // the relevant node sequences.
1341 //
1342 // The ideal graph generator translates a volatile read to the node
1343 // sequence
1344 //
1345 // LoadX[mo_acquire]
1346 // MemBarAcquire
1347 //
1348 // As a special case when using the compressed oops optimization we
1349 // may also see this variant
1350 //
1351 // LoadN[mo_acquire]
1352 // DecodeN
1353 // MemBarAcquire
1354 //
1355 // A volatile write is translated to the node sequence
1356 //
1357 // MemBarRelease
1358 // StoreX[mo_release] {CardMark}-optional
1359 // MemBarVolatile
1360 //
1361 // n.b. the above node patterns are generated with a strict
1362 // 'signature' configuration of input and output dependencies (see
1363 // the predicates below for exact details). The card mark may be as
1364 // simple as a few extra nodes or, in a few GC configurations, may
1365 // include more complex control flow between the leading and
1366 // trailing memory barriers. However, whatever the card mark
1367 // configuration these signatures are unique to translated volatile
1368 // reads/stores -- they will not appear as a result of any other
1369 // bytecode translation or inlining nor as a consequence of
1370 // optimizing transforms.
1371 //
1372 // We also want to catch inlined unsafe volatile gets and puts and
1373 // be able to implement them using either ldar<x>/stlr<x> or some
1374 // combination of ldr<x>/stlr<x> and dmb instructions.
1375 //
1376 // Inlined unsafe volatiles puts manifest as a minor variant of the
1377 // normal volatile put node sequence containing an extra cpuorder
1378 // membar
1379 //
1380 // MemBarRelease
1381 // MemBarCPUOrder
1382 // StoreX[mo_release] {CardMark}-optional
1383 // MemBarCPUOrder
1384 // MemBarVolatile
1385 //
1386 // n.b. as an aside, a cpuorder membar is not itself subject to
1387 // matching and translation by adlc rules. However, the rule
1388 // predicates need to detect its presence in order to correctly
1389 // select the desired adlc rules.
1390 //
1391 // Inlined unsafe volatile gets manifest as a slightly different
1392 // node sequence to a normal volatile get because of the
1393 // introduction of some CPUOrder memory barriers to bracket the
1394 // Load. However, but the same basic skeleton of a LoadX feeding a
1395 // MemBarAcquire, possibly through an optional DecodeN, is still
1396 // present
1397 //
1398 // MemBarCPUOrder
1399 // || \\
1400 // MemBarCPUOrder LoadX[mo_acquire]
1401 // || |
1402 // || {DecodeN} optional
1403 // || /
1404 // MemBarAcquire
1405 //
1406 // In this case the acquire membar does not directly depend on the
1407 // load. However, we can be sure that the load is generated from an
1408 // inlined unsafe volatile get if we see it dependent on this unique
1409 // sequence of membar nodes. Similarly, given an acquire membar we
1410 // can know that it was added because of an inlined unsafe volatile
1411 // get if it is fed and feeds a cpuorder membar and if its feed
1412 // membar also feeds an acquiring load.
1413 //
1414 // Finally an inlined (Unsafe) CAS operation is translated to the
1415 // following ideal graph
1416 //
1417 // MemBarRelease
1418 // MemBarCPUOrder
1419 // CompareAndSwapX {CardMark}-optional
1420 // MemBarCPUOrder
1421 // MemBarAcquire
1422 //
1423 // So, where we can identify these volatile read and write
1424 // signatures we can choose to plant either of the above two code
1425 // sequences. For a volatile read we can simply plant a normal
1426 // ldr<x> and translate the MemBarAcquire to a dmb. However, we can
1427 // also choose to inhibit translation of the MemBarAcquire and
1428 // inhibit planting of the ldr<x>, instead planting an ldar<x>.
1429 //
1430 // When we recognise a volatile store signature we can choose to
1431 // plant at a dmb ish as a translation for the MemBarRelease, a
1432 // normal str<x> and then a dmb ish for the MemBarVolatile.
1433 // Alternatively, we can inhibit translation of the MemBarRelease
1434 // and MemBarVolatile and instead plant a simple stlr<x>
1435 // instruction.
1436 //
1437 // when we recognise a CAS signature we can choose to plant a dmb
1438 // ish as a translation for the MemBarRelease, the conventional
1439 // macro-instruction sequence for the CompareAndSwap node (which
1440 // uses ldxr<x>) and then a dmb ishld for the MemBarAcquire.
1441 // Alternatively, we can elide generation of the dmb instructions
1442 // and plant the alternative CompareAndSwap macro-instruction
1443 // sequence (which uses ldaxr<x>).
1444 //
1445 // Of course, the above only applies when we see these signature
1446 // configurations. We still want to plant dmb instructions in any
1447 // other cases where we may see a MemBarAcquire, MemBarRelease or
1448 // MemBarVolatile. For example, at the end of a constructor which
1449 // writes final/volatile fields we will see a MemBarRelease
1450 // instruction and this needs a 'dmb ish' lest we risk the
1451 // constructed object being visible without making the
1452 // final/volatile field writes visible.
1453 //
1454 // n.b. the translation rules below which rely on detection of the
1455 // volatile signatures and insert ldar<x> or stlr<x> are failsafe.
1456 // If we see anything other than the signature configurations we
1457 // always just translate the loads and stores to ldr<x> and str<x>
1458 // and translate acquire, release and volatile membars to the
1459 // relevant dmb instructions.
1460 //
1461
1462 // is_CAS(int opcode, bool maybe_volatile)
1463 //
1464 // return true if opcode is one of the possible CompareAndSwapX
1465 // values otherwise false.
1466
1467 bool is_CAS(int opcode, bool maybe_volatile)
1468 {
1469 switch(opcode) {
1470 // We handle these
1471 case Op_CompareAndSwapI:
1472 case Op_CompareAndSwapL:
1473 case Op_CompareAndSwapP:
1474 case Op_CompareAndSwapN:
1475 case Op_ShenandoahCompareAndSwapP:
1476 case Op_ShenandoahCompareAndSwapN:
1477 case Op_CompareAndSwapB:
1478 case Op_CompareAndSwapS:
1479 case Op_GetAndSetI:
1480 case Op_GetAndSetL:
1481 case Op_GetAndSetP:
1482 case Op_GetAndSetN:
1483 case Op_GetAndAddI:
1484 case Op_GetAndAddL:
1485 return true;
1486 case Op_CompareAndExchangeI:
1487 case Op_CompareAndExchangeN:
1488 case Op_CompareAndExchangeB:
1489 case Op_CompareAndExchangeS:
1490 case Op_CompareAndExchangeL:
1491 case Op_CompareAndExchangeP:
1492 case Op_WeakCompareAndSwapB:
1493 case Op_WeakCompareAndSwapS:
1494 case Op_WeakCompareAndSwapI:
1495 case Op_WeakCompareAndSwapL:
1496 case Op_WeakCompareAndSwapP:
1497 case Op_WeakCompareAndSwapN:
1498 case Op_ShenandoahWeakCompareAndSwapP:
1499 case Op_ShenandoahWeakCompareAndSwapN:
1500 case Op_ShenandoahCompareAndExchangeP:
1501 case Op_ShenandoahCompareAndExchangeN:
1502 return maybe_volatile;
1503 default:
1504 return false;
1505 }
1506 }
1507
1508 // helper to determine the maximum number of Phi nodes we may need to
1509 // traverse when searching from a card mark membar for the merge mem
1510 // feeding a trailing membar or vice versa
1511
1512 // predicates controlling emit of ldr<x>/ldar<x>
1513
1514 bool unnecessary_acquire(const Node *barrier)
1515 {
1516 assert(barrier->is_MemBar(), "expecting a membar");
1517
1518 MemBarNode* mb = barrier->as_MemBar();
1519
1520 if (mb->trailing_load()) {
1521 return true;
1522 }
1523
1524 if (mb->trailing_load_store()) {
1525 Node* load_store = mb->in(MemBarNode::Precedent);
1526 assert(load_store->is_LoadStore(), "unexpected graph shape");
1527 return is_CAS(load_store->Opcode(), true);
1528 }
1529
1530 return false;
1531 }
1532
1533 bool needs_acquiring_load(const Node *n)
1534 {
1535 assert(n->is_Load(), "expecting a load");
1536 LoadNode *ld = n->as_Load();
1537 return ld->is_acquire();
1538 }
1539
1540 bool unnecessary_release(const Node *n)
1541 {
1542 assert((n->is_MemBar() &&
1543 n->Opcode() == Op_MemBarRelease),
1544 "expecting a release membar");
1545
1546 MemBarNode *barrier = n->as_MemBar();
1547 if (!barrier->leading()) {
1548 return false;
1549 } else {
1550 Node* trailing = barrier->trailing_membar();
1551 MemBarNode* trailing_mb = trailing->as_MemBar();
1552 assert(trailing_mb->trailing(), "Not a trailing membar?");
1553 assert(trailing_mb->leading_membar() == n, "inconsistent leading/trailing membars");
1554
1555 Node* mem = trailing_mb->in(MemBarNode::Precedent);
1556 if (mem->is_Store()) {
1557 assert(mem->as_Store()->is_release(), "");
1558 assert(trailing_mb->Opcode() == Op_MemBarVolatile, "");
1559 return true;
1560 } else {
1561 assert(mem->is_LoadStore(), "");
1562 assert(trailing_mb->Opcode() == Op_MemBarAcquire, "");
1563 return is_CAS(mem->Opcode(), true);
1564 }
1565 }
1566 return false;
1567 }
1568
1569 bool unnecessary_volatile(const Node *n)
1570 {
1571 // assert n->is_MemBar();
1572 MemBarNode *mbvol = n->as_MemBar();
1573
1574 bool release = mbvol->trailing_store();
1575 assert(!release || (mbvol->in(MemBarNode::Precedent)->is_Store() && mbvol->in(MemBarNode::Precedent)->as_Store()->is_release()), "");
1576 #ifdef ASSERT
1577 if (release) {
1578 Node* leading = mbvol->leading_membar();
1579 assert(leading->Opcode() == Op_MemBarRelease, "");
1580 assert(leading->as_MemBar()->leading_store(), "");
1581 assert(leading->as_MemBar()->trailing_membar() == mbvol, "");
1582 }
1583 #endif
1584
1585 return release;
1586 }
1587
1588 // predicates controlling emit of str<x>/stlr<x>
1589
1590 bool needs_releasing_store(const Node *n)
1591 {
1592 // assert n->is_Store();
1593 StoreNode *st = n->as_Store();
1594 return st->trailing_membar() != nullptr;
1595 }
1596
1597 // predicate controlling translation of CAS
1598 //
1599 // returns true if CAS needs to use an acquiring load otherwise false
1600
1601 bool needs_acquiring_load_exclusive(const Node *n)
1602 {
1603 assert(is_CAS(n->Opcode(), true), "expecting a compare and swap");
1604 LoadStoreNode* ldst = n->as_LoadStore();
1605 if (is_CAS(n->Opcode(), false)) {
1606 assert(ldst->trailing_membar() != nullptr, "expected trailing membar");
1607 } else {
1608 return ldst->trailing_membar() != nullptr;
1609 }
1610
1611 // so we can just return true here
1612 return true;
1613 }
1614
1615 #define __ masm->
1616
1617 // advance declarations for helper functions to convert register
1618 // indices to register objects
1619
1620 // the ad file has to provide implementations of certain methods
1621 // expected by the generic code
1622 //
1623 // REQUIRED FUNCTIONALITY
1624
1625 //=============================================================================
1626
1627 // !!!!! Special hack to get all types of calls to specify the byte offset
1628 // from the start of the call to the point where the return address
1629 // will point.
1630
1631 int MachCallStaticJavaNode::ret_addr_offset()
1632 {
1633 // call should be a simple bl
1634 int off = 4;
1635 return off;
1636 }
1637
1638 int MachCallDynamicJavaNode::ret_addr_offset()
1639 {
1640 return 16; // movz, movk, movk, bl
1641 }
1642
1643 int MachCallRuntimeNode::ret_addr_offset() {
1644 // for generated stubs the call will be
1645 // bl(addr)
1646 // or with far branches
1647 // bl(trampoline_stub)
1648 // for real runtime callouts it will be six instructions
1649 // see aarch64_enc_java_to_runtime
1650 // adr(rscratch2, retaddr)
1651 // lea(rscratch1, RuntimeAddress(addr)
1652 // stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)))
1653 // blr(rscratch1)
1654 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1655 if (cb) {
1656 return 1 * NativeInstruction::instruction_size;
1657 } else {
1658 return 6 * NativeInstruction::instruction_size;
1659 }
1660 }
1661
1662 //=============================================================================
1663
1664 #ifndef PRODUCT
1665 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1666 st->print("BREAKPOINT");
1667 }
1668 #endif
1669
1670 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1671 __ brk(0);
1672 }
1673
1674 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1675 return MachNode::size(ra_);
1676 }
1677
1678 //=============================================================================
1679
1680 #ifndef PRODUCT
1681 void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const {
1682 st->print("nop \t# %d bytes pad for loops and calls", _count);
1683 }
1684 #endif
1685
1686 void MachNopNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc*) const {
1687 for (int i = 0; i < _count; i++) {
1688 __ nop();
1689 }
1690 }
1691
1692 uint MachNopNode::size(PhaseRegAlloc*) const {
1693 return _count * NativeInstruction::instruction_size;
1694 }
1695
1696 //=============================================================================
1697 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
1698
1699 int ConstantTable::calculate_table_base_offset() const {
1700 return 0; // absolute addressing, no offset
1701 }
1702
1703 bool MachConstantBaseNode::requires_postalloc_expand() const { return false; }
1704 void MachConstantBaseNode::postalloc_expand(GrowableArray <Node *> *nodes, PhaseRegAlloc *ra_) {
1705 ShouldNotReachHere();
1706 }
1707
1708 void MachConstantBaseNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const {
1709 // Empty encoding
1710 }
1711
1712 uint MachConstantBaseNode::size(PhaseRegAlloc* ra_) const {
1713 return 0;
1714 }
1715
1716 #ifndef PRODUCT
1717 void MachConstantBaseNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
1718 st->print("-- \t// MachConstantBaseNode (empty encoding)");
1719 }
1720 #endif
1721
1722 #ifndef PRODUCT
1723 void MachPrologNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1724 Compile* C = ra_->C;
1725
1726 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1727
1728 if (C->output()->need_stack_bang(framesize))
1729 st->print("# stack bang size=%d\n\t", framesize);
1730
1731 if (VM_Version::use_rop_protection()) {
1732 st->print("ldr zr, [lr]\n\t");
1733 st->print("paciaz\n\t");
1734 }
1735 if (framesize < ((1 << 9) + 2 * wordSize)) {
1736 st->print("sub sp, sp, #%d\n\t", framesize);
1737 st->print("stp rfp, lr, [sp, #%d]", framesize - 2 * wordSize);
1738 if (PreserveFramePointer) st->print("\n\tadd rfp, sp, #%d", framesize - 2 * wordSize);
1739 } else {
1740 st->print("stp lr, rfp, [sp, #%d]!\n\t", -(2 * wordSize));
1741 if (PreserveFramePointer) st->print("mov rfp, sp\n\t");
1742 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1743 st->print("sub sp, sp, rscratch1");
1744 }
1745 if (C->stub_function() == nullptr && BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1746 st->print("\n\t");
1747 st->print("ldr rscratch1, [guard]\n\t");
1748 st->print("dmb ishld\n\t");
1749 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1750 st->print("cmp rscratch1, rscratch2\n\t");
1751 st->print("b.eq skip");
1752 st->print("\n\t");
1753 st->print("blr #nmethod_entry_barrier_stub\n\t");
1754 st->print("b skip\n\t");
1755 st->print("guard: int\n\t");
1756 st->print("\n\t");
1757 st->print("skip:\n\t");
1758 }
1759 }
1760 #endif
1761
1762 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1763 Compile* C = ra_->C;
1764
1765 // n.b. frame size includes space for return pc and rfp
1766 const int framesize = C->output()->frame_size_in_bytes();
1767
1768 // insert a nop at the start of the prolog so we can patch in a
1769 // branch if we need to invalidate the method later
1770 __ nop();
1771
1772 if (C->clinit_barrier_on_entry()) {
1773 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1774
1775 Label L_skip_barrier;
1776
1777 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1778 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1779 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1780 __ bind(L_skip_barrier);
1781 }
1782
1783 if (C->max_vector_size() > 0) {
1784 __ reinitialize_ptrue();
1785 }
1786
1787 int bangsize = C->output()->bang_size_in_bytes();
1788 if (C->output()->need_stack_bang(bangsize))
1789 __ generate_stack_overflow_check(bangsize);
1790
1791 __ build_frame(framesize);
1792
1793 if (C->stub_function() == nullptr) {
1794 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1795 if (BarrierSet::barrier_set()->barrier_set_nmethod() != nullptr) {
1796 // Dummy labels for just measuring the code size
1797 Label dummy_slow_path;
1798 Label dummy_continuation;
1799 Label dummy_guard;
1800 Label* slow_path = &dummy_slow_path;
1801 Label* continuation = &dummy_continuation;
1802 Label* guard = &dummy_guard;
1803 if (!Compile::current()->output()->in_scratch_emit_size()) {
1804 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1805 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1806 Compile::current()->output()->add_stub(stub);
1807 slow_path = &stub->entry();
1808 continuation = &stub->continuation();
1809 guard = &stub->guard();
1810 }
1811 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1812 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1813 }
1814 }
1815
1816 if (VerifyStackAtCalls) {
1817 Unimplemented();
1818 }
1819
1820 C->output()->set_frame_complete(__ offset());
1821
1822 if (C->has_mach_constant_base_node()) {
1823 // NOTE: We set the table base offset here because users might be
1824 // emitted before MachConstantBaseNode.
1825 ConstantTable& constant_table = C->output()->constant_table();
1826 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1827 }
1828 }
1829
1830 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1831 {
1832 return MachNode::size(ra_); // too many variables; just compute it
1833 // the hard way
1834 }
1835
1836 int MachPrologNode::reloc() const
1837 {
1838 return 0;
1839 }
1840
1841 //=============================================================================
1842
1843 #ifndef PRODUCT
1844 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1845 Compile* C = ra_->C;
1846 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1847
1848 st->print("# pop frame %d\n\t",framesize);
1849
1850 if (framesize == 0) {
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1853 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1854 st->print("add sp, sp, #%d\n\t", framesize);
1855 } else {
1856 st->print("mov rscratch1, #%d\n\t", framesize - 2 * wordSize);
1857 st->print("add sp, sp, rscratch1\n\t");
1858 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1859 }
1860 if (VM_Version::use_rop_protection()) {
1861 st->print("autiaz\n\t");
1862 st->print("ldr zr, [lr]\n\t");
1863 }
1864
1865 if (do_polling() && C->is_method_compilation()) {
1866 st->print("# test polling word\n\t");
1867 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1868 st->print("cmp sp, rscratch1\n\t");
1869 st->print("bhi #slow_path");
1870 }
1871 }
1872 #endif
1873
1874 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 __ remove_frame(framesize);
1879
1880 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1881 __ reserved_stack_check();
1882 }
1883
1884 if (do_polling() && C->is_method_compilation()) {
1885 Label dummy_label;
1886 Label* code_stub = &dummy_label;
1887 if (!C->output()->in_scratch_emit_size()) {
1888 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1889 C->output()->add_stub(stub);
1890 code_stub = &stub->entry();
1891 }
1892 __ relocate(relocInfo::poll_return_type);
1893 __ safepoint_poll(*code_stub, true /* at_return */, false /* acquire */, true /* in_nmethod */);
1894 }
1895 }
1896
1897 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1898 // Variable size. Determine dynamically.
1899 return MachNode::size(ra_);
1900 }
1901
1902 int MachEpilogNode::reloc() const {
1903 // Return number of relocatable values contained in this instruction.
1904 return 1; // 1 for polling page.
1905 }
1906
1907 const Pipeline * MachEpilogNode::pipeline() const {
1908 return MachNode::pipeline_class();
1909 }
1910
1911 //=============================================================================
1912
1913 static enum RC rc_class(OptoReg::Name reg) {
1914
1915 if (reg == OptoReg::Bad) {
1916 return rc_bad;
1917 }
1918
1919 // we have 32 int registers * 2 halves
1920 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1921
1922 if (reg < slots_of_int_registers) {
1923 return rc_int;
1924 }
1925
1926 // we have 32 float register * 8 halves
1927 int slots_of_float_registers = FloatRegister::number_of_registers * FloatRegister::max_slots_per_register;
1928 if (reg < slots_of_int_registers + slots_of_float_registers) {
1929 return rc_float;
1930 }
1931
1932 int slots_of_predicate_registers = PRegister::number_of_registers * PRegister::max_slots_per_register;
1933 if (reg < slots_of_int_registers + slots_of_float_registers + slots_of_predicate_registers) {
1934 return rc_predicate;
1935 }
1936
1937 // Between predicate regs & stack is the flags.
1938 assert(OptoReg::is_stack(reg), "blow up if spilling flags");
1939
1940 return rc_stack;
1941 }
1942
1943 uint MachSpillCopyNode::implementation(C2_MacroAssembler *masm, PhaseRegAlloc *ra_, bool do_size, outputStream *st) const {
1944 Compile* C = ra_->C;
1945
1946 // Get registers to move.
1947 OptoReg::Name src_hi = ra_->get_reg_second(in(1));
1948 OptoReg::Name src_lo = ra_->get_reg_first(in(1));
1949 OptoReg::Name dst_hi = ra_->get_reg_second(this);
1950 OptoReg::Name dst_lo = ra_->get_reg_first(this);
1951
1952 enum RC src_hi_rc = rc_class(src_hi);
1953 enum RC src_lo_rc = rc_class(src_lo);
1954 enum RC dst_hi_rc = rc_class(dst_hi);
1955 enum RC dst_lo_rc = rc_class(dst_lo);
1956
1957 assert(src_lo != OptoReg::Bad && dst_lo != OptoReg::Bad, "must move at least 1 register");
1958
1959 if (src_hi != OptoReg::Bad && !bottom_type()->isa_vectmask()) {
1960 assert((src_lo&1)==0 && src_lo+1==src_hi &&
1961 (dst_lo&1)==0 && dst_lo+1==dst_hi,
1962 "expected aligned-adjacent pairs");
1963 }
1964
1965 if (src_lo == dst_lo && src_hi == dst_hi) {
1966 return 0; // Self copy, no move.
1967 }
1968
1969 bool is64 = (src_lo & 1) == 0 && src_lo + 1 == src_hi &&
1970 (dst_lo & 1) == 0 && dst_lo + 1 == dst_hi;
1971 int src_offset = ra_->reg2offset(src_lo);
1972 int dst_offset = ra_->reg2offset(dst_lo);
1973
1974 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
1975 uint ireg = ideal_reg();
1976 if (ireg == Op_VecA && masm) {
1977 int sve_vector_reg_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1978 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1979 // stack->stack
1980 __ spill_copy_sve_vector_stack_to_stack(src_offset, dst_offset,
1981 sve_vector_reg_size_in_bytes);
1982 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
1983 __ spill_sve_vector(as_FloatRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
1984 sve_vector_reg_size_in_bytes);
1985 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
1986 __ unspill_sve_vector(as_FloatRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
1987 sve_vector_reg_size_in_bytes);
1988 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
1989 __ sve_orr(as_FloatRegister(Matcher::_regEncode[dst_lo]),
1990 as_FloatRegister(Matcher::_regEncode[src_lo]),
1991 as_FloatRegister(Matcher::_regEncode[src_lo]));
1992 } else {
1993 ShouldNotReachHere();
1994 }
1995 } else if (masm) {
1996 assert(ireg == Op_VecD || ireg == Op_VecX, "must be 64 bit or 128 bit vector");
1997 assert((src_lo_rc != rc_int && dst_lo_rc != rc_int), "sanity");
1998 if (src_lo_rc == rc_stack && dst_lo_rc == rc_stack) {
1999 // stack->stack
2000 assert((src_offset & 7) == 0 && (dst_offset & 7) == 0, "unaligned stack offset");
2001 if (ireg == Op_VecD) {
2002 __ unspill(rscratch1, true, src_offset);
2003 __ spill(rscratch1, true, dst_offset);
2004 } else {
2005 __ spill_copy128(src_offset, dst_offset);
2006 }
2007 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_float) {
2008 __ mov(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2009 ireg == Op_VecD ? __ T8B : __ T16B,
2010 as_FloatRegister(Matcher::_regEncode[src_lo]));
2011 } else if (src_lo_rc == rc_float && dst_lo_rc == rc_stack) {
2012 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2013 ireg == Op_VecD ? __ D : __ Q,
2014 ra_->reg2offset(dst_lo));
2015 } else if (src_lo_rc == rc_stack && dst_lo_rc == rc_float) {
2016 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2017 ireg == Op_VecD ? __ D : __ Q,
2018 ra_->reg2offset(src_lo));
2019 } else {
2020 ShouldNotReachHere();
2021 }
2022 }
2023 } else if (masm) {
2024 switch (src_lo_rc) {
2025 case rc_int:
2026 if (dst_lo_rc == rc_int) { // gpr --> gpr copy
2027 if (is64) {
2028 __ mov(as_Register(Matcher::_regEncode[dst_lo]),
2029 as_Register(Matcher::_regEncode[src_lo]));
2030 } else {
2031 __ movw(as_Register(Matcher::_regEncode[dst_lo]),
2032 as_Register(Matcher::_regEncode[src_lo]));
2033 }
2034 } else if (dst_lo_rc == rc_float) { // gpr --> fpr copy
2035 if (is64) {
2036 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2037 as_Register(Matcher::_regEncode[src_lo]));
2038 } else {
2039 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2040 as_Register(Matcher::_regEncode[src_lo]));
2041 }
2042 } else { // gpr --> stack spill
2043 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2044 __ spill(as_Register(Matcher::_regEncode[src_lo]), is64, dst_offset);
2045 }
2046 break;
2047 case rc_float:
2048 if (dst_lo_rc == rc_int) { // fpr --> gpr copy
2049 if (is64) {
2050 __ fmovd(as_Register(Matcher::_regEncode[dst_lo]),
2051 as_FloatRegister(Matcher::_regEncode[src_lo]));
2052 } else {
2053 __ fmovs(as_Register(Matcher::_regEncode[dst_lo]),
2054 as_FloatRegister(Matcher::_regEncode[src_lo]));
2055 }
2056 } else if (dst_lo_rc == rc_float) { // fpr --> fpr copy
2057 if (is64) {
2058 __ fmovd(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2059 as_FloatRegister(Matcher::_regEncode[src_lo]));
2060 } else {
2061 __ fmovs(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2062 as_FloatRegister(Matcher::_regEncode[src_lo]));
2063 }
2064 } else { // fpr --> stack spill
2065 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2066 __ spill(as_FloatRegister(Matcher::_regEncode[src_lo]),
2067 is64 ? __ D : __ S, dst_offset);
2068 }
2069 break;
2070 case rc_stack:
2071 if (dst_lo_rc == rc_int) { // stack --> gpr load
2072 __ unspill(as_Register(Matcher::_regEncode[dst_lo]), is64, src_offset);
2073 } else if (dst_lo_rc == rc_float) { // stack --> fpr load
2074 __ unspill(as_FloatRegister(Matcher::_regEncode[dst_lo]),
2075 is64 ? __ D : __ S, src_offset);
2076 } else if (dst_lo_rc == rc_predicate) {
2077 __ unspill_sve_predicate(as_PRegister(Matcher::_regEncode[dst_lo]), ra_->reg2offset(src_lo),
2078 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2079 } else { // stack --> stack copy
2080 assert(dst_lo_rc == rc_stack, "spill to bad register class");
2081 if (ideal_reg() == Op_RegVectMask) {
2082 __ spill_copy_sve_predicate_stack_to_stack(src_offset, dst_offset,
2083 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2084 } else {
2085 __ unspill(rscratch1, is64, src_offset);
2086 __ spill(rscratch1, is64, dst_offset);
2087 }
2088 }
2089 break;
2090 case rc_predicate:
2091 if (dst_lo_rc == rc_predicate) {
2092 __ sve_mov(as_PRegister(Matcher::_regEncode[dst_lo]), as_PRegister(Matcher::_regEncode[src_lo]));
2093 } else if (dst_lo_rc == rc_stack) {
2094 __ spill_sve_predicate(as_PRegister(Matcher::_regEncode[src_lo]), ra_->reg2offset(dst_lo),
2095 Matcher::scalable_vector_reg_size(T_BYTE) >> 3);
2096 } else {
2097 assert(false, "bad src and dst rc_class combination.");
2098 ShouldNotReachHere();
2099 }
2100 break;
2101 default:
2102 assert(false, "bad rc_class for spill");
2103 ShouldNotReachHere();
2104 }
2105 }
2106
2107 if (st) {
2108 st->print("spill ");
2109 if (src_lo_rc == rc_stack) {
2110 st->print("[sp, #%d] -> ", ra_->reg2offset(src_lo));
2111 } else {
2112 st->print("%s -> ", Matcher::regName[src_lo]);
2113 }
2114 if (dst_lo_rc == rc_stack) {
2115 st->print("[sp, #%d]", ra_->reg2offset(dst_lo));
2116 } else {
2117 st->print("%s", Matcher::regName[dst_lo]);
2118 }
2119 if (bottom_type()->isa_vect() && !bottom_type()->isa_vectmask()) {
2120 int vsize = 0;
2121 switch (ideal_reg()) {
2122 case Op_VecD:
2123 vsize = 64;
2124 break;
2125 case Op_VecX:
2126 vsize = 128;
2127 break;
2128 case Op_VecA:
2129 vsize = Matcher::scalable_vector_reg_size(T_BYTE) * 8;
2130 break;
2131 default:
2132 assert(false, "bad register type for spill");
2133 ShouldNotReachHere();
2134 }
2135 st->print("\t# vector spill size = %d", vsize);
2136 } else if (ideal_reg() == Op_RegVectMask) {
2137 assert(Matcher::supports_scalable_vector(), "bad register type for spill");
2138 int vsize = Matcher::scalable_predicate_reg_slots() * 32;
2139 st->print("\t# predicate spill size = %d", vsize);
2140 } else {
2141 st->print("\t# spill size = %d", is64 ? 64 : 32);
2142 }
2143 }
2144
2145 return 0;
2146
2147 }
2148
2149 #ifndef PRODUCT
2150 void MachSpillCopyNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2151 if (!ra_)
2152 st->print("N%d = SpillCopy(N%d)", _idx, in(1)->_idx);
2153 else
2154 implementation(nullptr, ra_, false, st);
2155 }
2156 #endif
2157
2158 void MachSpillCopyNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2159 implementation(masm, ra_, false, nullptr);
2160 }
2161
2162 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const {
2163 return MachNode::size(ra_);
2164 }
2165
2166 //=============================================================================
2167
2168 #ifndef PRODUCT
2169 void BoxLockNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
2170 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2171 int reg = ra_->get_reg_first(this);
2172 st->print("add %s, rsp, #%d]\t# box lock",
2173 Matcher::regName[reg], offset);
2174 }
2175 #endif
2176
2177 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2178 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2179 int reg = ra_->get_encode(this);
2180
2181 // This add will handle any 24-bit signed offset. 24 bits allows an
2182 // 8 megabyte stack frame.
2183 __ add(as_Register(reg), sp, offset);
2184 }
2185
2186 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2187 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2188 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2189
2190 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2191 return NativeInstruction::instruction_size;
2192 } else {
2193 return 2 * NativeInstruction::instruction_size;
2194 }
2195 }
2196
2197 //=============================================================================
2198
2199 #ifndef PRODUCT
2200 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2201 {
2202 st->print_cr("# MachUEPNode");
2203 if (UseCompressedClassPointers) {
2204 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2205 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2206 st->print_cr("\tcmpw rscratch1, r10");
2207 } else {
2208 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2209 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2210 st->print_cr("\tcmp rscratch1, r10");
2211 }
2212 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2213 }
2214 #endif
2215
2216 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2217 {
2218 __ ic_check(InteriorEntryAlignment);
2219 }
2220
2221 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2222 {
2223 return MachNode::size(ra_);
2224 }
2225
2226 // REQUIRED EMIT CODE
2227
2228 //=============================================================================
2229
2230 // Emit exception handler code.
2231 int HandlerImpl::emit_exception_handler(C2_MacroAssembler* masm)
2232 {
2233 // mov rscratch1 #exception_blob_entry_point
2234 // br rscratch1
2235 // Note that the code buffer's insts_mark is always relative to insts.
2236 // That's why we must use the macroassembler to generate a handler.
2237 address base = __ start_a_stub(size_exception_handler());
2238 if (base == nullptr) {
2239 ciEnv::current()->record_failure("CodeCache is full");
2240 return 0; // CodeBuffer::expand failed
2241 }
2242 int offset = __ offset();
2243 __ far_jump(RuntimeAddress(OptoRuntime::exception_blob()->entry_point()));
2244 assert(__ offset() - offset <= (int) size_exception_handler(), "overflow");
2245 __ end_a_stub();
2246 return offset;
2247 }
2248
2249 // Emit deopt handler code.
2250 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2251 {
2252 // Note that the code buffer's insts_mark is always relative to insts.
2253 // That's why we must use the macroassembler to generate a handler.
2254 address base = __ start_a_stub(size_deopt_handler());
2255 if (base == nullptr) {
2256 ciEnv::current()->record_failure("CodeCache is full");
2257 return 0; // CodeBuffer::expand failed
2258 }
2259 int offset = __ offset();
2260
2261 __ adr(lr, __ pc());
2262 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2263
2264 assert(__ offset() - offset == (int) size_deopt_handler(), "overflow");
2265 __ end_a_stub();
2266 return offset;
2267 }
2268
2269 // REQUIRED MATCHER CODE
2270
2271 //=============================================================================
2272
2273 bool Matcher::match_rule_supported(int opcode) {
2274 if (!has_match_rule(opcode))
2275 return false;
2276
2277 switch (opcode) {
2278 case Op_OnSpinWait:
2279 return VM_Version::supports_on_spin_wait();
2280 case Op_CacheWB:
2281 case Op_CacheWBPreSync:
2282 case Op_CacheWBPostSync:
2283 if (!VM_Version::supports_data_cache_line_flush()) {
2284 return false;
2285 }
2286 break;
2287 case Op_ExpandBits:
2288 case Op_CompressBits:
2289 if (!VM_Version::supports_svebitperm()) {
2290 return false;
2291 }
2292 break;
2293 case Op_FmaF:
2294 case Op_FmaD:
2295 case Op_FmaVF:
2296 case Op_FmaVD:
2297 if (!UseFMA) {
2298 return false;
2299 }
2300 break;
2301 }
2302
2303 return true; // Per default match rules are supported.
2304 }
2305
2306 const RegMask* Matcher::predicate_reg_mask(void) {
2307 return &_PR_REG_mask;
2308 }
2309
2310 // Vector calling convention not yet implemented.
2311 bool Matcher::supports_vector_calling_convention(void) {
2312 return false;
2313 }
2314
2315 OptoRegPair Matcher::vector_return_value(uint ideal_reg) {
2316 Unimplemented();
2317 return OptoRegPair(0, 0);
2318 }
2319
2320 // Is this branch offset short enough that a short branch can be used?
2321 //
2322 // NOTE: If the platform does not provide any short branch variants, then
2323 // this method should return false for offset 0.
2324 bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) {
2325 // The passed offset is relative to address of the branch.
2326
2327 return (-32768 <= offset && offset < 32768);
2328 }
2329
2330 // Vector width in bytes.
2331 int Matcher::vector_width_in_bytes(BasicType bt) {
2332 // The MaxVectorSize should have been set by detecting SVE max vector register size.
2333 int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize);
2334 // Minimum 2 values in vector
2335 if (size < 2*type2aelembytes(bt)) size = 0;
2336 // But never < 4
2337 if (size < 4) size = 0;
2338 return size;
2339 }
2340
2341 // Limits on vector size (number of elements) loaded into vector.
2342 int Matcher::max_vector_size(const BasicType bt) {
2343 return vector_width_in_bytes(bt)/type2aelembytes(bt);
2344 }
2345
2346 int Matcher::min_vector_size(const BasicType bt) {
2347 int max_size = max_vector_size(bt);
2348 // Limit the min vector size to 8 bytes.
2349 int size = 8 / type2aelembytes(bt);
2350 if (bt == T_BYTE) {
2351 // To support vector api shuffle/rearrange.
2352 size = 4;
2353 } else if (bt == T_BOOLEAN) {
2354 // To support vector api load/store mask.
2355 size = 2;
2356 }
2357 if (size < 2) size = 2;
2358 return MIN2(size, max_size);
2359 }
2360
2361 int Matcher::max_vector_size_auto_vectorization(const BasicType bt) {
2362 return Matcher::max_vector_size(bt);
2363 }
2364
2365 // Actual max scalable vector register length.
2366 int Matcher::scalable_vector_reg_size(const BasicType bt) {
2367 return Matcher::max_vector_size(bt);
2368 }
2369
2370 // Vector ideal reg.
2371 uint Matcher::vector_ideal_reg(int len) {
2372 if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) {
2373 return Op_VecA;
2374 }
2375 switch(len) {
2376 // For 16-bit/32-bit mask vector, reuse VecD.
2377 case 2:
2378 case 4:
2379 case 8: return Op_VecD;
2380 case 16: return Op_VecX;
2381 }
2382 ShouldNotReachHere();
2383 return 0;
2384 }
2385
2386 MachOper* Matcher::pd_specialize_generic_vector_operand(MachOper* generic_opnd, uint ideal_reg, bool is_temp) {
2387 assert(Matcher::is_generic_vector(generic_opnd), "not generic");
2388 switch (ideal_reg) {
2389 case Op_VecA: return new vecAOper();
2390 case Op_VecD: return new vecDOper();
2391 case Op_VecX: return new vecXOper();
2392 }
2393 ShouldNotReachHere();
2394 return nullptr;
2395 }
2396
2397 bool Matcher::is_reg2reg_move(MachNode* m) {
2398 return false;
2399 }
2400
2401 bool Matcher::is_generic_vector(MachOper* opnd) {
2402 return opnd->opcode() == VREG;
2403 }
2404
2405 // Return whether or not this register is ever used as an argument.
2406 // This function is used on startup to build the trampoline stubs in
2407 // generateOptoStub. Registers not mentioned will be killed by the VM
2408 // call in the trampoline, and arguments in those registers not be
2409 // available to the callee.
2410 bool Matcher::can_be_java_arg(int reg)
2411 {
2412 return
2413 reg == R0_num || reg == R0_H_num ||
2414 reg == R1_num || reg == R1_H_num ||
2415 reg == R2_num || reg == R2_H_num ||
2416 reg == R3_num || reg == R3_H_num ||
2417 reg == R4_num || reg == R4_H_num ||
2418 reg == R5_num || reg == R5_H_num ||
2419 reg == R6_num || reg == R6_H_num ||
2420 reg == R7_num || reg == R7_H_num ||
2421 reg == V0_num || reg == V0_H_num ||
2422 reg == V1_num || reg == V1_H_num ||
2423 reg == V2_num || reg == V2_H_num ||
2424 reg == V3_num || reg == V3_H_num ||
2425 reg == V4_num || reg == V4_H_num ||
2426 reg == V5_num || reg == V5_H_num ||
2427 reg == V6_num || reg == V6_H_num ||
2428 reg == V7_num || reg == V7_H_num;
2429 }
2430
2431 bool Matcher::is_spillable_arg(int reg)
2432 {
2433 return can_be_java_arg(reg);
2434 }
2435
2436 uint Matcher::int_pressure_limit()
2437 {
2438 // JDK-8183543: When taking the number of available registers as int
2439 // register pressure threshold, the jtreg test:
2440 // test/hotspot/jtreg/compiler/regalloc/TestC2IntPressure.java
2441 // failed due to C2 compilation failure with
2442 // "COMPILE SKIPPED: failed spill-split-recycle sanity check".
2443 //
2444 // A derived pointer is live at CallNode and then is flagged by RA
2445 // as a spilled LRG. Spilling heuristics(Spill-USE) explicitly skip
2446 // derived pointers and lastly fail to spill after reaching maximum
2447 // number of iterations. Lowering the default pressure threshold to
2448 // (_NO_SPECIAL_REG32_mask.Size() minus 1) forces CallNode to become
2449 // a high register pressure area of the code so that split_DEF can
2450 // generate DefinitionSpillCopy for the derived pointer.
2451 uint default_int_pressure_threshold = _NO_SPECIAL_REG32_mask.Size() - 1;
2452 if (!PreserveFramePointer) {
2453 // When PreserveFramePointer is off, frame pointer is allocatable,
2454 // but different from other SOC registers, it is excluded from
2455 // fatproj's mask because its save type is No-Save. Decrease 1 to
2456 // ensure high pressure at fatproj when PreserveFramePointer is off.
2457 // See check_pressure_at_fatproj().
2458 default_int_pressure_threshold--;
2459 }
2460 return (INTPRESSURE == -1) ? default_int_pressure_threshold : INTPRESSURE;
2461 }
2462
2463 uint Matcher::float_pressure_limit()
2464 {
2465 // _FLOAT_REG_mask is generated by adlc from the float_reg register class.
2466 return (FLOATPRESSURE == -1) ? _FLOAT_REG_mask.Size() : FLOATPRESSURE;
2467 }
2468
2469 bool Matcher::use_asm_for_ldiv_by_con(jlong divisor) {
2470 return false;
2471 }
2472
2473 RegMask Matcher::divI_proj_mask() {
2474 ShouldNotReachHere();
2475 return RegMask();
2476 }
2477
2478 // Register for MODI projection of divmodI.
2479 RegMask Matcher::modI_proj_mask() {
2480 ShouldNotReachHere();
2481 return RegMask();
2482 }
2483
2484 // Register for DIVL projection of divmodL.
2485 RegMask Matcher::divL_proj_mask() {
2486 ShouldNotReachHere();
2487 return RegMask();
2488 }
2489
2490 // Register for MODL projection of divmodL.
2491 RegMask Matcher::modL_proj_mask() {
2492 ShouldNotReachHere();
2493 return RegMask();
2494 }
2495
2496 const RegMask Matcher::method_handle_invoke_SP_save_mask() {
2497 return FP_REG_mask();
2498 }
2499
2500 bool size_fits_all_mem_uses(AddPNode* addp, int shift) {
2501 for (DUIterator_Fast imax, i = addp->fast_outs(imax); i < imax; i++) {
2502 Node* u = addp->fast_out(i);
2503 if (u->is_LoadStore()) {
2504 // On AArch64, LoadStoreNodes (i.e. compare and swap
2505 // instructions) only take register indirect as an operand, so
2506 // any attempt to use an AddPNode as an input to a LoadStoreNode
2507 // must fail.
2508 return false;
2509 }
2510 if (u->is_Mem()) {
2511 int opsize = u->as_Mem()->memory_size();
2512 assert(opsize > 0, "unexpected memory operand size");
2513 if (u->as_Mem()->memory_size() != (1<<shift)) {
2514 return false;
2515 }
2516 }
2517 }
2518 return true;
2519 }
2520
2521 // Convert BootTest condition to Assembler condition.
2522 // Replicate the logic of cmpOpOper::ccode() and cmpOpUOper::ccode().
2523 Assembler::Condition to_assembler_cond(BoolTest::mask cond) {
2524 Assembler::Condition result;
2525 switch(cond) {
2526 case BoolTest::eq:
2527 result = Assembler::EQ; break;
2528 case BoolTest::ne:
2529 result = Assembler::NE; break;
2530 case BoolTest::le:
2531 result = Assembler::LE; break;
2532 case BoolTest::ge:
2533 result = Assembler::GE; break;
2534 case BoolTest::lt:
2535 result = Assembler::LT; break;
2536 case BoolTest::gt:
2537 result = Assembler::GT; break;
2538 case BoolTest::ule:
2539 result = Assembler::LS; break;
2540 case BoolTest::uge:
2541 result = Assembler::HS; break;
2542 case BoolTest::ult:
2543 result = Assembler::LO; break;
2544 case BoolTest::ugt:
2545 result = Assembler::HI; break;
2546 case BoolTest::overflow:
2547 result = Assembler::VS; break;
2548 case BoolTest::no_overflow:
2549 result = Assembler::VC; break;
2550 default:
2551 ShouldNotReachHere();
2552 return Assembler::Condition(-1);
2553 }
2554
2555 // Check conversion
2556 if (cond & BoolTest::unsigned_compare) {
2557 assert(cmpOpUOper((BoolTest::mask)((int)cond & ~(BoolTest::unsigned_compare))).ccode() == result, "Invalid conversion");
2558 } else {
2559 assert(cmpOpOper(cond).ccode() == result, "Invalid conversion");
2560 }
2561
2562 return result;
2563 }
2564
2565 // Binary src (Replicate con)
2566 static bool is_valid_sve_arith_imm_pattern(Node* n, Node* m) {
2567 if (n == nullptr || m == nullptr) {
2568 return false;
2569 }
2570
2571 if (UseSVE == 0 || m->Opcode() != Op_Replicate) {
2572 return false;
2573 }
2574
2575 Node* imm_node = m->in(1);
2576 if (!imm_node->is_Con()) {
2577 return false;
2578 }
2579
2580 const Type* t = imm_node->bottom_type();
2581 if (!(t->isa_int() || t->isa_long())) {
2582 return false;
2583 }
2584
2585 switch (n->Opcode()) {
2586 case Op_AndV:
2587 case Op_OrV:
2588 case Op_XorV: {
2589 Assembler::SIMD_RegVariant T = Assembler::elemType_to_regVariant(Matcher::vector_element_basic_type(n));
2590 uint64_t value = t->isa_long() ? (uint64_t)imm_node->get_long() : (uint64_t)imm_node->get_int();
2591 return Assembler::operand_valid_for_sve_logical_immediate(Assembler::regVariant_to_elemBits(T), value);
2592 }
2593 case Op_AddVB:
2594 return (imm_node->get_int() <= 255 && imm_node->get_int() >= -255);
2595 case Op_AddVS:
2596 case Op_AddVI:
2597 return Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)imm_node->get_int());
2598 case Op_AddVL:
2599 return Assembler::operand_valid_for_sve_add_sub_immediate(imm_node->get_long());
2600 default:
2601 return false;
2602 }
2603 }
2604
2605 // (XorV src (Replicate m1))
2606 // (XorVMask src (MaskAll m1))
2607 static bool is_vector_bitwise_not_pattern(Node* n, Node* m) {
2608 if (n != nullptr && m != nullptr) {
2609 return (n->Opcode() == Op_XorV || n->Opcode() == Op_XorVMask) &&
2610 VectorNode::is_all_ones_vector(m);
2611 }
2612 return false;
2613 }
2614
2615 // Should the matcher clone input 'm' of node 'n'?
2616 bool Matcher::pd_clone_node(Node* n, Node* m, Matcher::MStack& mstack) {
2617 if (is_vshift_con_pattern(n, m) ||
2618 is_vector_bitwise_not_pattern(n, m) ||
2619 is_valid_sve_arith_imm_pattern(n, m) ||
2620 is_encode_and_store_pattern(n, m)) {
2621 mstack.push(m, Visit);
2622 return true;
2623 }
2624 return false;
2625 }
2626
2627 // Should the Matcher clone shifts on addressing modes, expecting them
2628 // to be subsumed into complex addressing expressions or compute them
2629 // into registers?
2630 bool Matcher::pd_clone_address_expressions(AddPNode* m, Matcher::MStack& mstack, VectorSet& address_visited) {
2631 if (clone_base_plus_offset_address(m, mstack, address_visited)) {
2632 return true;
2633 }
2634
2635 Node *off = m->in(AddPNode::Offset);
2636 if (off->Opcode() == Op_LShiftL && off->in(2)->is_Con() &&
2637 size_fits_all_mem_uses(m, off->in(2)->get_int()) &&
2638 // Are there other uses besides address expressions?
2639 !is_visited(off)) {
2640 address_visited.set(off->_idx); // Flag as address_visited
2641 mstack.push(off->in(2), Visit);
2642 Node *conv = off->in(1);
2643 if (conv->Opcode() == Op_ConvI2L &&
2644 // Are there other uses besides address expressions?
2645 !is_visited(conv)) {
2646 address_visited.set(conv->_idx); // Flag as address_visited
2647 mstack.push(conv->in(1), Pre_Visit);
2648 } else {
2649 mstack.push(conv, Pre_Visit);
2650 }
2651 address_visited.test_set(m->_idx); // Flag as address_visited
2652 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2653 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2654 return true;
2655 } else if (off->Opcode() == Op_ConvI2L &&
2656 // Are there other uses besides address expressions?
2657 !is_visited(off)) {
2658 address_visited.test_set(m->_idx); // Flag as address_visited
2659 address_visited.set(off->_idx); // Flag as address_visited
2660 mstack.push(off->in(1), Pre_Visit);
2661 mstack.push(m->in(AddPNode::Address), Pre_Visit);
2662 mstack.push(m->in(AddPNode::Base), Pre_Visit);
2663 return true;
2664 }
2665 return false;
2666 }
2667
2668 #define MOV_VOLATILE(REG, BASE, INDEX, SCALE, DISP, SCRATCH, INSN) \
2669 { \
2670 guarantee(INDEX == -1, "mode not permitted for volatile"); \
2671 guarantee(DISP == 0, "mode not permitted for volatile"); \
2672 guarantee(SCALE == 0, "mode not permitted for volatile"); \
2673 __ INSN(REG, as_Register(BASE)); \
2674 }
2675
2676
2677 static Address mem2address(int opcode, Register base, int index, int size, int disp)
2678 {
2679 Address::extend scale;
2680
2681 // Hooboy, this is fugly. We need a way to communicate to the
2682 // encoder that the index needs to be sign extended, so we have to
2683 // enumerate all the cases.
2684 switch (opcode) {
2685 case INDINDEXSCALEDI2L:
2686 case INDINDEXSCALEDI2LN:
2687 case INDINDEXI2L:
2688 case INDINDEXI2LN:
2689 scale = Address::sxtw(size);
2690 break;
2691 default:
2692 scale = Address::lsl(size);
2693 }
2694
2695 if (index == -1) {
2696 return Address(base, disp);
2697 } else {
2698 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2699 return Address(base, as_Register(index), scale);
2700 }
2701 }
2702
2703
2704 typedef void (MacroAssembler::* mem_insn)(Register Rt, const Address &adr);
2705 typedef void (MacroAssembler::* mem_insn2)(Register Rt, Register adr);
2706 typedef void (MacroAssembler::* mem_float_insn)(FloatRegister Rt, const Address &adr);
2707 typedef void (MacroAssembler::* mem_vector_insn)(FloatRegister Rt,
2708 MacroAssembler::SIMD_RegVariant T, const Address &adr);
2709
2710 // Used for all non-volatile memory accesses. The use of
2711 // $mem->opcode() to discover whether this pattern uses sign-extended
2712 // offsets is something of a kludge.
2713 static void loadStore(C2_MacroAssembler* masm, mem_insn insn,
2714 Register reg, int opcode,
2715 Register base, int index, int scale, int disp,
2716 int size_in_memory)
2717 {
2718 Address addr = mem2address(opcode, base, index, scale, disp);
2719 if (addr.getMode() == Address::base_plus_offset) {
2720 /* Fix up any out-of-range offsets. */
2721 assert_different_registers(rscratch1, base);
2722 assert_different_registers(rscratch1, reg);
2723 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2724 }
2725 (masm->*insn)(reg, addr);
2726 }
2727
2728 static void loadStore(C2_MacroAssembler* masm, mem_float_insn insn,
2729 FloatRegister reg, int opcode,
2730 Register base, int index, int size, int disp,
2731 int size_in_memory)
2732 {
2733 Address::extend scale;
2734
2735 switch (opcode) {
2736 case INDINDEXSCALEDI2L:
2737 case INDINDEXSCALEDI2LN:
2738 scale = Address::sxtw(size);
2739 break;
2740 default:
2741 scale = Address::lsl(size);
2742 }
2743
2744 if (index == -1) {
2745 // Fix up any out-of-range offsets.
2746 assert_different_registers(rscratch1, base);
2747 Address addr = Address(base, disp);
2748 addr = __ legitimize_address(addr, size_in_memory, rscratch1);
2749 (masm->*insn)(reg, addr);
2750 } else {
2751 assert(disp == 0, "unsupported address mode: disp = %d", disp);
2752 (masm->*insn)(reg, Address(base, as_Register(index), scale));
2753 }
2754 }
2755
2756 static void loadStore(C2_MacroAssembler* masm, mem_vector_insn insn,
2757 FloatRegister reg, MacroAssembler::SIMD_RegVariant T,
2758 int opcode, Register base, int index, int size, int disp)
2759 {
2760 if (index == -1) {
2761 (masm->*insn)(reg, T, Address(base, disp));
2762 } else {
2763 assert(disp == 0, "unsupported address mode");
2764 (masm->*insn)(reg, T, Address(base, as_Register(index), Address::lsl(size)));
2765 }
2766 }
2767
2768 %}
2769
2770
2771
2772 //----------ENCODING BLOCK-----------------------------------------------------
2773 // This block specifies the encoding classes used by the compiler to
2774 // output byte streams. Encoding classes are parameterized macros
2775 // used by Machine Instruction Nodes in order to generate the bit
2776 // encoding of the instruction. Operands specify their base encoding
2777 // interface with the interface keyword. There are currently
2778 // supported four interfaces, REG_INTER, CONST_INTER, MEMORY_INTER, &
2779 // COND_INTER. REG_INTER causes an operand to generate a function
2780 // which returns its register number when queried. CONST_INTER causes
2781 // an operand to generate a function which returns the value of the
2782 // constant when queried. MEMORY_INTER causes an operand to generate
2783 // four functions which return the Base Register, the Index Register,
2784 // the Scale Value, and the Offset Value of the operand when queried.
2785 // COND_INTER causes an operand to generate six functions which return
2786 // the encoding code (ie - encoding bits for the instruction)
2787 // associated with each basic boolean condition for a conditional
2788 // instruction.
2789 //
2790 // Instructions specify two basic values for encoding. Again, a
2791 // function is available to check if the constant displacement is an
2792 // oop. They use the ins_encode keyword to specify their encoding
2793 // classes (which must be a sequence of enc_class names, and their
2794 // parameters, specified in the encoding block), and they use the
2795 // opcode keyword to specify, in order, their primary, secondary, and
2796 // tertiary opcode. Only the opcode sections which a particular
2797 // instruction needs for encoding need to be specified.
2798 encode %{
2799 // Build emit functions for each basic byte or larger field in the
2800 // intel encoding scheme (opcode, rm, sib, immediate), and call them
2801 // from C++ code in the enc_class source block. Emit functions will
2802 // live in the main source block for now. In future, we can
2803 // generalize this by adding a syntax that specifies the sizes of
2804 // fields in an order, so that the adlc can build the emit functions
2805 // automagically
2806
2807 // catch all for unimplemented encodings
2808 enc_class enc_unimplemented %{
2809 __ unimplemented("C2 catch all");
2810 %}
2811
2812 // BEGIN Non-volatile memory access
2813
2814 // This encoding class is generated automatically from ad_encode.m4.
2815 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2816 enc_class aarch64_enc_ldrsbw(iRegI dst, memory1 mem) %{
2817 Register dst_reg = as_Register($dst$$reg);
2818 loadStore(masm, &MacroAssembler::ldrsbw, dst_reg, $mem->opcode(),
2819 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2820 %}
2821
2822 // This encoding class is generated automatically from ad_encode.m4.
2823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2824 enc_class aarch64_enc_ldrsb(iRegI dst, memory1 mem) %{
2825 Register dst_reg = as_Register($dst$$reg);
2826 loadStore(masm, &MacroAssembler::ldrsb, dst_reg, $mem->opcode(),
2827 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2828 %}
2829
2830 // This encoding class is generated automatically from ad_encode.m4.
2831 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2832 enc_class aarch64_enc_ldrb(iRegI dst, memory1 mem) %{
2833 Register dst_reg = as_Register($dst$$reg);
2834 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2835 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2836 %}
2837
2838 // This encoding class is generated automatically from ad_encode.m4.
2839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2840 enc_class aarch64_enc_ldrb(iRegL dst, memory1 mem) %{
2841 Register dst_reg = as_Register($dst$$reg);
2842 loadStore(masm, &MacroAssembler::ldrb, dst_reg, $mem->opcode(),
2843 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2844 %}
2845
2846 // This encoding class is generated automatically from ad_encode.m4.
2847 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2848 enc_class aarch64_enc_ldrshw(iRegI dst, memory2 mem) %{
2849 Register dst_reg = as_Register($dst$$reg);
2850 loadStore(masm, &MacroAssembler::ldrshw, dst_reg, $mem->opcode(),
2851 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2852 %}
2853
2854 // This encoding class is generated automatically from ad_encode.m4.
2855 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2856 enc_class aarch64_enc_ldrsh(iRegI dst, memory2 mem) %{
2857 Register dst_reg = as_Register($dst$$reg);
2858 loadStore(masm, &MacroAssembler::ldrsh, dst_reg, $mem->opcode(),
2859 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2860 %}
2861
2862 // This encoding class is generated automatically from ad_encode.m4.
2863 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2864 enc_class aarch64_enc_ldrh(iRegI dst, memory2 mem) %{
2865 Register dst_reg = as_Register($dst$$reg);
2866 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2867 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2868 %}
2869
2870 // This encoding class is generated automatically from ad_encode.m4.
2871 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2872 enc_class aarch64_enc_ldrh(iRegL dst, memory2 mem) %{
2873 Register dst_reg = as_Register($dst$$reg);
2874 loadStore(masm, &MacroAssembler::ldrh, dst_reg, $mem->opcode(),
2875 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2876 %}
2877
2878 // This encoding class is generated automatically from ad_encode.m4.
2879 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2880 enc_class aarch64_enc_ldrw(iRegI dst, memory4 mem) %{
2881 Register dst_reg = as_Register($dst$$reg);
2882 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2883 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2884 %}
2885
2886 // This encoding class is generated automatically from ad_encode.m4.
2887 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2888 enc_class aarch64_enc_ldrw(iRegL dst, memory4 mem) %{
2889 Register dst_reg = as_Register($dst$$reg);
2890 loadStore(masm, &MacroAssembler::ldrw, dst_reg, $mem->opcode(),
2891 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2892 %}
2893
2894 // This encoding class is generated automatically from ad_encode.m4.
2895 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2896 enc_class aarch64_enc_ldrsw(iRegL dst, memory4 mem) %{
2897 Register dst_reg = as_Register($dst$$reg);
2898 loadStore(masm, &MacroAssembler::ldrsw, dst_reg, $mem->opcode(),
2899 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2900 %}
2901
2902 // This encoding class is generated automatically from ad_encode.m4.
2903 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2904 enc_class aarch64_enc_ldr(iRegL dst, memory8 mem) %{
2905 Register dst_reg = as_Register($dst$$reg);
2906 loadStore(masm, &MacroAssembler::ldr, dst_reg, $mem->opcode(),
2907 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2908 %}
2909
2910 // This encoding class is generated automatically from ad_encode.m4.
2911 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2912 enc_class aarch64_enc_ldrs(vRegF dst, memory4 mem) %{
2913 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2914 loadStore(masm, &MacroAssembler::ldrs, dst_reg, $mem->opcode(),
2915 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2916 %}
2917
2918 // This encoding class is generated automatically from ad_encode.m4.
2919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2920 enc_class aarch64_enc_ldrd(vRegD dst, memory8 mem) %{
2921 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
2922 loadStore(masm, &MacroAssembler::ldrd, dst_reg, $mem->opcode(),
2923 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2924 %}
2925
2926 // This encoding class is generated automatically from ad_encode.m4.
2927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2928 enc_class aarch64_enc_strb(iRegI src, memory1 mem) %{
2929 Register src_reg = as_Register($src$$reg);
2930 loadStore(masm, &MacroAssembler::strb, src_reg, $mem->opcode(),
2931 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2932 %}
2933
2934 // This encoding class is generated automatically from ad_encode.m4.
2935 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2936 enc_class aarch64_enc_strb0(memory1 mem) %{
2937 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
2938 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
2939 %}
2940
2941 // This encoding class is generated automatically from ad_encode.m4.
2942 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2943 enc_class aarch64_enc_strh(iRegI src, memory2 mem) %{
2944 Register src_reg = as_Register($src$$reg);
2945 loadStore(masm, &MacroAssembler::strh, src_reg, $mem->opcode(),
2946 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2947 %}
2948
2949 // This encoding class is generated automatically from ad_encode.m4.
2950 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2951 enc_class aarch64_enc_strh0(memory2 mem) %{
2952 loadStore(masm, &MacroAssembler::strh, zr, $mem->opcode(),
2953 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 2);
2954 %}
2955
2956 // This encoding class is generated automatically from ad_encode.m4.
2957 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2958 enc_class aarch64_enc_strw(iRegI src, memory4 mem) %{
2959 Register src_reg = as_Register($src$$reg);
2960 loadStore(masm, &MacroAssembler::strw, src_reg, $mem->opcode(),
2961 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2962 %}
2963
2964 // This encoding class is generated automatically from ad_encode.m4.
2965 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2966 enc_class aarch64_enc_strw0(memory4 mem) %{
2967 loadStore(masm, &MacroAssembler::strw, zr, $mem->opcode(),
2968 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2969 %}
2970
2971 // This encoding class is generated automatically from ad_encode.m4.
2972 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2973 enc_class aarch64_enc_str(iRegL src, memory8 mem) %{
2974 Register src_reg = as_Register($src$$reg);
2975 // we sometimes get asked to store the stack pointer into the
2976 // current thread -- we cannot do that directly on AArch64
2977 if (src_reg == r31_sp) {
2978 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
2979 __ mov(rscratch2, sp);
2980 src_reg = rscratch2;
2981 }
2982 loadStore(masm, &MacroAssembler::str, src_reg, $mem->opcode(),
2983 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2984 %}
2985
2986 // This encoding class is generated automatically from ad_encode.m4.
2987 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2988 enc_class aarch64_enc_str0(memory8 mem) %{
2989 loadStore(masm, &MacroAssembler::str, zr, $mem->opcode(),
2990 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
2991 %}
2992
2993 // This encoding class is generated automatically from ad_encode.m4.
2994 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
2995 enc_class aarch64_enc_strs(vRegF src, memory4 mem) %{
2996 FloatRegister src_reg = as_FloatRegister($src$$reg);
2997 loadStore(masm, &MacroAssembler::strs, src_reg, $mem->opcode(),
2998 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
2999 %}
3000
3001 // This encoding class is generated automatically from ad_encode.m4.
3002 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3003 enc_class aarch64_enc_strd(vRegD src, memory8 mem) %{
3004 FloatRegister src_reg = as_FloatRegister($src$$reg);
3005 loadStore(masm, &MacroAssembler::strd, src_reg, $mem->opcode(),
3006 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
3007 %}
3008
3009 // This encoding class is generated automatically from ad_encode.m4.
3010 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
3011 enc_class aarch64_enc_strb0_ordered(memory4 mem) %{
3012 __ membar(Assembler::StoreStore);
3013 loadStore(masm, &MacroAssembler::strb, zr, $mem->opcode(),
3014 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 1);
3015 %}
3016
3017 // END Non-volatile memory access
3018
3019 // Vector loads and stores
3020 enc_class aarch64_enc_ldrvH(vReg dst, memory mem) %{
3021 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3022 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::H,
3023 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3024 %}
3025
3026 enc_class aarch64_enc_ldrvS(vReg dst, memory mem) %{
3027 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3028 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::S,
3029 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3030 %}
3031
3032 enc_class aarch64_enc_ldrvD(vReg dst, memory mem) %{
3033 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3034 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::D,
3035 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3036 %}
3037
3038 enc_class aarch64_enc_ldrvQ(vReg dst, memory mem) %{
3039 FloatRegister dst_reg = as_FloatRegister($dst$$reg);
3040 loadStore(masm, &MacroAssembler::ldr, dst_reg, MacroAssembler::Q,
3041 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3042 %}
3043
3044 enc_class aarch64_enc_strvH(vReg src, memory mem) %{
3045 FloatRegister src_reg = as_FloatRegister($src$$reg);
3046 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::H,
3047 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3048 %}
3049
3050 enc_class aarch64_enc_strvS(vReg src, memory mem) %{
3051 FloatRegister src_reg = as_FloatRegister($src$$reg);
3052 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::S,
3053 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3054 %}
3055
3056 enc_class aarch64_enc_strvD(vReg src, memory mem) %{
3057 FloatRegister src_reg = as_FloatRegister($src$$reg);
3058 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::D,
3059 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3060 %}
3061
3062 enc_class aarch64_enc_strvQ(vReg src, memory mem) %{
3063 FloatRegister src_reg = as_FloatRegister($src$$reg);
3064 loadStore(masm, &MacroAssembler::str, src_reg, MacroAssembler::Q,
3065 $mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
3066 %}
3067
3068 // volatile loads and stores
3069
3070 enc_class aarch64_enc_stlrb(iRegI src, memory mem) %{
3071 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3072 rscratch1, stlrb);
3073 %}
3074
3075 enc_class aarch64_enc_stlrb0(memory mem) %{
3076 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3077 rscratch1, stlrb);
3078 %}
3079
3080 enc_class aarch64_enc_stlrh(iRegI src, memory mem) %{
3081 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3082 rscratch1, stlrh);
3083 %}
3084
3085 enc_class aarch64_enc_stlrh0(memory mem) %{
3086 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3087 rscratch1, stlrh);
3088 %}
3089
3090 enc_class aarch64_enc_stlrw(iRegI src, memory mem) %{
3091 MOV_VOLATILE(as_Register($src$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3092 rscratch1, stlrw);
3093 %}
3094
3095 enc_class aarch64_enc_stlrw0(memory mem) %{
3096 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3097 rscratch1, stlrw);
3098 %}
3099
3100 enc_class aarch64_enc_ldarsbw(iRegI dst, memory mem) %{
3101 Register dst_reg = as_Register($dst$$reg);
3102 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3103 rscratch1, ldarb);
3104 __ sxtbw(dst_reg, dst_reg);
3105 %}
3106
3107 enc_class aarch64_enc_ldarsb(iRegL dst, memory mem) %{
3108 Register dst_reg = as_Register($dst$$reg);
3109 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3110 rscratch1, ldarb);
3111 __ sxtb(dst_reg, dst_reg);
3112 %}
3113
3114 enc_class aarch64_enc_ldarbw(iRegI dst, memory mem) %{
3115 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3116 rscratch1, ldarb);
3117 %}
3118
3119 enc_class aarch64_enc_ldarb(iRegL dst, memory mem) %{
3120 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3121 rscratch1, ldarb);
3122 %}
3123
3124 enc_class aarch64_enc_ldarshw(iRegI dst, memory mem) %{
3125 Register dst_reg = as_Register($dst$$reg);
3126 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3127 rscratch1, ldarh);
3128 __ sxthw(dst_reg, dst_reg);
3129 %}
3130
3131 enc_class aarch64_enc_ldarsh(iRegL dst, memory mem) %{
3132 Register dst_reg = as_Register($dst$$reg);
3133 MOV_VOLATILE(dst_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3134 rscratch1, ldarh);
3135 __ sxth(dst_reg, dst_reg);
3136 %}
3137
3138 enc_class aarch64_enc_ldarhw(iRegI dst, memory mem) %{
3139 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3140 rscratch1, ldarh);
3141 %}
3142
3143 enc_class aarch64_enc_ldarh(iRegL dst, memory mem) %{
3144 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3145 rscratch1, ldarh);
3146 %}
3147
3148 enc_class aarch64_enc_ldarw(iRegI dst, memory mem) %{
3149 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3150 rscratch1, ldarw);
3151 %}
3152
3153 enc_class aarch64_enc_ldarw(iRegL dst, memory mem) %{
3154 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3155 rscratch1, ldarw);
3156 %}
3157
3158 enc_class aarch64_enc_ldar(iRegL dst, memory mem) %{
3159 MOV_VOLATILE(as_Register($dst$$reg), $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3160 rscratch1, ldar);
3161 %}
3162
3163 enc_class aarch64_enc_fldars(vRegF dst, memory mem) %{
3164 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3165 rscratch1, ldarw);
3166 __ fmovs(as_FloatRegister($dst$$reg), rscratch1);
3167 %}
3168
3169 enc_class aarch64_enc_fldard(vRegD dst, memory mem) %{
3170 MOV_VOLATILE(rscratch1, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3171 rscratch1, ldar);
3172 __ fmovd(as_FloatRegister($dst$$reg), rscratch1);
3173 %}
3174
3175 enc_class aarch64_enc_stlr(iRegL src, memory mem) %{
3176 Register src_reg = as_Register($src$$reg);
3177 // we sometimes get asked to store the stack pointer into the
3178 // current thread -- we cannot do that directly on AArch64
3179 if (src_reg == r31_sp) {
3180 assert(as_Register($mem$$base) == rthread, "unexpected store for sp");
3181 __ mov(rscratch2, sp);
3182 src_reg = rscratch2;
3183 }
3184 MOV_VOLATILE(src_reg, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3185 rscratch1, stlr);
3186 %}
3187
3188 enc_class aarch64_enc_stlr0(memory mem) %{
3189 MOV_VOLATILE(zr, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3190 rscratch1, stlr);
3191 %}
3192
3193 enc_class aarch64_enc_fstlrs(vRegF src, memory mem) %{
3194 {
3195 FloatRegister src_reg = as_FloatRegister($src$$reg);
3196 __ fmovs(rscratch2, src_reg);
3197 }
3198 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3199 rscratch1, stlrw);
3200 %}
3201
3202 enc_class aarch64_enc_fstlrd(vRegD src, memory mem) %{
3203 {
3204 FloatRegister src_reg = as_FloatRegister($src$$reg);
3205 __ fmovd(rscratch2, src_reg);
3206 }
3207 MOV_VOLATILE(rscratch2, $mem$$base, $mem$$index, $mem$$scale, $mem$$disp,
3208 rscratch1, stlr);
3209 %}
3210
3211 // synchronized read/update encodings
3212
3213 enc_class aarch64_enc_ldaxr(iRegL dst, memory8 mem) %{
3214 Register dst_reg = as_Register($dst$$reg);
3215 Register base = as_Register($mem$$base);
3216 int index = $mem$$index;
3217 int scale = $mem$$scale;
3218 int disp = $mem$$disp;
3219 if (index == -1) {
3220 if (disp != 0) {
3221 __ lea(rscratch1, Address(base, disp));
3222 __ ldaxr(dst_reg, rscratch1);
3223 } else {
3224 // TODO
3225 // should we ever get anything other than this case?
3226 __ ldaxr(dst_reg, base);
3227 }
3228 } else {
3229 Register index_reg = as_Register(index);
3230 if (disp == 0) {
3231 __ lea(rscratch1, Address(base, index_reg, Address::lsl(scale)));
3232 __ ldaxr(dst_reg, rscratch1);
3233 } else {
3234 __ lea(rscratch1, Address(base, disp));
3235 __ lea(rscratch1, Address(rscratch1, index_reg, Address::lsl(scale)));
3236 __ ldaxr(dst_reg, rscratch1);
3237 }
3238 }
3239 %}
3240
3241 enc_class aarch64_enc_stlxr(iRegLNoSp src, memory8 mem) %{
3242 Register src_reg = as_Register($src$$reg);
3243 Register base = as_Register($mem$$base);
3244 int index = $mem$$index;
3245 int scale = $mem$$scale;
3246 int disp = $mem$$disp;
3247 if (index == -1) {
3248 if (disp != 0) {
3249 __ lea(rscratch2, Address(base, disp));
3250 __ stlxr(rscratch1, src_reg, rscratch2);
3251 } else {
3252 // TODO
3253 // should we ever get anything other than this case?
3254 __ stlxr(rscratch1, src_reg, base);
3255 }
3256 } else {
3257 Register index_reg = as_Register(index);
3258 if (disp == 0) {
3259 __ lea(rscratch2, Address(base, index_reg, Address::lsl(scale)));
3260 __ stlxr(rscratch1, src_reg, rscratch2);
3261 } else {
3262 __ lea(rscratch2, Address(base, disp));
3263 __ lea(rscratch2, Address(rscratch2, index_reg, Address::lsl(scale)));
3264 __ stlxr(rscratch1, src_reg, rscratch2);
3265 }
3266 }
3267 __ cmpw(rscratch1, zr);
3268 %}
3269
3270 enc_class aarch64_enc_cmpxchg(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3271 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3272 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3273 Assembler::xword, /*acquire*/ false, /*release*/ true,
3274 /*weak*/ false, noreg);
3275 %}
3276
3277 enc_class aarch64_enc_cmpxchgw(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3278 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3279 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3280 Assembler::word, /*acquire*/ false, /*release*/ true,
3281 /*weak*/ false, noreg);
3282 %}
3283
3284 enc_class aarch64_enc_cmpxchgs(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3285 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3286 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3287 Assembler::halfword, /*acquire*/ false, /*release*/ true,
3288 /*weak*/ false, noreg);
3289 %}
3290
3291 enc_class aarch64_enc_cmpxchgb(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3292 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3293 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3294 Assembler::byte, /*acquire*/ false, /*release*/ true,
3295 /*weak*/ false, noreg);
3296 %}
3297
3298
3299 // The only difference between aarch64_enc_cmpxchg and
3300 // aarch64_enc_cmpxchg_acq is that we use load-acquire in the
3301 // CompareAndSwap sequence to serve as a barrier on acquiring a
3302 // lock.
3303 enc_class aarch64_enc_cmpxchg_acq(memory mem, iRegLNoSp oldval, iRegLNoSp newval) %{
3304 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3305 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3306 Assembler::xword, /*acquire*/ true, /*release*/ true,
3307 /*weak*/ false, noreg);
3308 %}
3309
3310 enc_class aarch64_enc_cmpxchgw_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3311 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3312 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3313 Assembler::word, /*acquire*/ true, /*release*/ true,
3314 /*weak*/ false, noreg);
3315 %}
3316
3317 enc_class aarch64_enc_cmpxchgs_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3318 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3319 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3320 Assembler::halfword, /*acquire*/ true, /*release*/ true,
3321 /*weak*/ false, noreg);
3322 %}
3323
3324 enc_class aarch64_enc_cmpxchgb_acq(memory mem, iRegINoSp oldval, iRegINoSp newval) %{
3325 guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
3326 __ cmpxchg($mem$$base$$Register, $oldval$$Register, $newval$$Register,
3327 Assembler::byte, /*acquire*/ true, /*release*/ true,
3328 /*weak*/ false, noreg);
3329 %}
3330
3331 // auxiliary used for CompareAndSwapX to set result register
3332 enc_class aarch64_enc_cset_eq(iRegINoSp res) %{
3333 Register res_reg = as_Register($res$$reg);
3334 __ cset(res_reg, Assembler::EQ);
3335 %}
3336
3337 // prefetch encodings
3338
3339 enc_class aarch64_enc_prefetchw(memory mem) %{
3340 Register base = as_Register($mem$$base);
3341 int index = $mem$$index;
3342 int scale = $mem$$scale;
3343 int disp = $mem$$disp;
3344 if (index == -1) {
3345 // Fix up any out-of-range offsets.
3346 assert_different_registers(rscratch1, base);
3347 Address addr = Address(base, disp);
3348 addr = __ legitimize_address(addr, 8, rscratch1);
3349 __ prfm(addr, PSTL1KEEP);
3350 } else {
3351 Register index_reg = as_Register(index);
3352 if (disp == 0) {
3353 __ prfm(Address(base, index_reg, Address::lsl(scale)), PSTL1KEEP);
3354 } else {
3355 __ lea(rscratch1, Address(base, disp));
3356 __ prfm(Address(rscratch1, index_reg, Address::lsl(scale)), PSTL1KEEP);
3357 }
3358 }
3359 %}
3360
3361 // mov encodings
3362
3363 enc_class aarch64_enc_movw_imm(iRegI dst, immI src) %{
3364 uint32_t con = (uint32_t)$src$$constant;
3365 Register dst_reg = as_Register($dst$$reg);
3366 if (con == 0) {
3367 __ movw(dst_reg, zr);
3368 } else {
3369 __ movw(dst_reg, con);
3370 }
3371 %}
3372
3373 enc_class aarch64_enc_mov_imm(iRegL dst, immL src) %{
3374 Register dst_reg = as_Register($dst$$reg);
3375 uint64_t con = (uint64_t)$src$$constant;
3376 if (con == 0) {
3377 __ mov(dst_reg, zr);
3378 } else {
3379 __ mov(dst_reg, con);
3380 }
3381 %}
3382
3383 enc_class aarch64_enc_mov_p(iRegP dst, immP src) %{
3384 Register dst_reg = as_Register($dst$$reg);
3385 address con = (address)$src$$constant;
3386 if (con == nullptr || con == (address)1) {
3387 ShouldNotReachHere();
3388 } else {
3389 relocInfo::relocType rtype = $src->constant_reloc();
3390 if (rtype == relocInfo::oop_type) {
3391 __ movoop(dst_reg, (jobject)con);
3392 } else if (rtype == relocInfo::metadata_type) {
3393 __ mov_metadata(dst_reg, (Metadata*)con);
3394 } else {
3395 assert(rtype == relocInfo::none, "unexpected reloc type");
3396 if (! __ is_valid_AArch64_address(con) ||
3397 con < (address)(uintptr_t)os::vm_page_size()) {
3398 __ mov(dst_reg, con);
3399 } else {
3400 uint64_t offset;
3401 __ adrp(dst_reg, con, offset);
3402 __ add(dst_reg, dst_reg, offset);
3403 }
3404 }
3405 }
3406 %}
3407
3408 enc_class aarch64_enc_mov_p0(iRegP dst, immP0 src) %{
3409 Register dst_reg = as_Register($dst$$reg);
3410 __ mov(dst_reg, zr);
3411 %}
3412
3413 enc_class aarch64_enc_mov_p1(iRegP dst, immP_1 src) %{
3414 Register dst_reg = as_Register($dst$$reg);
3415 __ mov(dst_reg, (uint64_t)1);
3416 %}
3417
3418 enc_class aarch64_enc_mov_byte_map_base(iRegP dst, immByteMapBase src) %{
3419 __ load_byte_map_base($dst$$Register);
3420 %}
3421
3422 enc_class aarch64_enc_mov_n(iRegN dst, immN src) %{
3423 Register dst_reg = as_Register($dst$$reg);
3424 address con = (address)$src$$constant;
3425 if (con == nullptr) {
3426 ShouldNotReachHere();
3427 } else {
3428 relocInfo::relocType rtype = $src->constant_reloc();
3429 assert(rtype == relocInfo::oop_type, "unexpected reloc type");
3430 __ set_narrow_oop(dst_reg, (jobject)con);
3431 }
3432 %}
3433
3434 enc_class aarch64_enc_mov_n0(iRegN dst, immN0 src) %{
3435 Register dst_reg = as_Register($dst$$reg);
3436 __ mov(dst_reg, zr);
3437 %}
3438
3439 enc_class aarch64_enc_mov_nk(iRegN dst, immNKlass src) %{
3440 Register dst_reg = as_Register($dst$$reg);
3441 address con = (address)$src$$constant;
3442 if (con == nullptr) {
3443 ShouldNotReachHere();
3444 } else {
3445 relocInfo::relocType rtype = $src->constant_reloc();
3446 assert(rtype == relocInfo::metadata_type, "unexpected reloc type");
3447 __ set_narrow_klass(dst_reg, (Klass *)con);
3448 }
3449 %}
3450
3451 // arithmetic encodings
3452
3453 enc_class aarch64_enc_addsubw_imm(iRegI dst, iRegI src1, immIAddSub src2) %{
3454 Register dst_reg = as_Register($dst$$reg);
3455 Register src_reg = as_Register($src1$$reg);
3456 int32_t con = (int32_t)$src2$$constant;
3457 // add has primary == 0, subtract has primary == 1
3458 if ($primary) { con = -con; }
3459 if (con < 0) {
3460 __ subw(dst_reg, src_reg, -con);
3461 } else {
3462 __ addw(dst_reg, src_reg, con);
3463 }
3464 %}
3465
3466 enc_class aarch64_enc_addsub_imm(iRegL dst, iRegL src1, immLAddSub src2) %{
3467 Register dst_reg = as_Register($dst$$reg);
3468 Register src_reg = as_Register($src1$$reg);
3469 int32_t con = (int32_t)$src2$$constant;
3470 // add has primary == 0, subtract has primary == 1
3471 if ($primary) { con = -con; }
3472 if (con < 0) {
3473 __ sub(dst_reg, src_reg, -con);
3474 } else {
3475 __ add(dst_reg, src_reg, con);
3476 }
3477 %}
3478
3479 enc_class aarch64_enc_divw(iRegI dst, iRegI src1, iRegI src2) %{
3480 Register dst_reg = as_Register($dst$$reg);
3481 Register src1_reg = as_Register($src1$$reg);
3482 Register src2_reg = as_Register($src2$$reg);
3483 __ corrected_idivl(dst_reg, src1_reg, src2_reg, false, rscratch1);
3484 %}
3485
3486 enc_class aarch64_enc_div(iRegI dst, iRegI src1, iRegI src2) %{
3487 Register dst_reg = as_Register($dst$$reg);
3488 Register src1_reg = as_Register($src1$$reg);
3489 Register src2_reg = as_Register($src2$$reg);
3490 __ corrected_idivq(dst_reg, src1_reg, src2_reg, false, rscratch1);
3491 %}
3492
3493 enc_class aarch64_enc_modw(iRegI dst, iRegI src1, iRegI src2) %{
3494 Register dst_reg = as_Register($dst$$reg);
3495 Register src1_reg = as_Register($src1$$reg);
3496 Register src2_reg = as_Register($src2$$reg);
3497 __ corrected_idivl(dst_reg, src1_reg, src2_reg, true, rscratch1);
3498 %}
3499
3500 enc_class aarch64_enc_mod(iRegI dst, iRegI src1, iRegI src2) %{
3501 Register dst_reg = as_Register($dst$$reg);
3502 Register src1_reg = as_Register($src1$$reg);
3503 Register src2_reg = as_Register($src2$$reg);
3504 __ corrected_idivq(dst_reg, src1_reg, src2_reg, true, rscratch1);
3505 %}
3506
3507 // compare instruction encodings
3508
3509 enc_class aarch64_enc_cmpw(iRegI src1, iRegI src2) %{
3510 Register reg1 = as_Register($src1$$reg);
3511 Register reg2 = as_Register($src2$$reg);
3512 __ cmpw(reg1, reg2);
3513 %}
3514
3515 enc_class aarch64_enc_cmpw_imm_addsub(iRegI src1, immIAddSub src2) %{
3516 Register reg = as_Register($src1$$reg);
3517 int32_t val = $src2$$constant;
3518 if (val >= 0) {
3519 __ subsw(zr, reg, val);
3520 } else {
3521 __ addsw(zr, reg, -val);
3522 }
3523 %}
3524
3525 enc_class aarch64_enc_cmpw_imm(iRegI src1, immI src2) %{
3526 Register reg1 = as_Register($src1$$reg);
3527 uint32_t val = (uint32_t)$src2$$constant;
3528 __ movw(rscratch1, val);
3529 __ cmpw(reg1, rscratch1);
3530 %}
3531
3532 enc_class aarch64_enc_cmp(iRegL src1, iRegL src2) %{
3533 Register reg1 = as_Register($src1$$reg);
3534 Register reg2 = as_Register($src2$$reg);
3535 __ cmp(reg1, reg2);
3536 %}
3537
3538 enc_class aarch64_enc_cmp_imm_addsub(iRegL src1, immL12 src2) %{
3539 Register reg = as_Register($src1$$reg);
3540 int64_t val = $src2$$constant;
3541 if (val >= 0) {
3542 __ subs(zr, reg, val);
3543 } else if (val != -val) {
3544 __ adds(zr, reg, -val);
3545 } else {
3546 // aargh, Long.MIN_VALUE is a special case
3547 __ orr(rscratch1, zr, (uint64_t)val);
3548 __ subs(zr, reg, rscratch1);
3549 }
3550 %}
3551
3552 enc_class aarch64_enc_cmp_imm(iRegL src1, immL src2) %{
3553 Register reg1 = as_Register($src1$$reg);
3554 uint64_t val = (uint64_t)$src2$$constant;
3555 __ mov(rscratch1, val);
3556 __ cmp(reg1, rscratch1);
3557 %}
3558
3559 enc_class aarch64_enc_cmpp(iRegP src1, iRegP src2) %{
3560 Register reg1 = as_Register($src1$$reg);
3561 Register reg2 = as_Register($src2$$reg);
3562 __ cmp(reg1, reg2);
3563 %}
3564
3565 enc_class aarch64_enc_cmpn(iRegN src1, iRegN src2) %{
3566 Register reg1 = as_Register($src1$$reg);
3567 Register reg2 = as_Register($src2$$reg);
3568 __ cmpw(reg1, reg2);
3569 %}
3570
3571 enc_class aarch64_enc_testp(iRegP src) %{
3572 Register reg = as_Register($src$$reg);
3573 __ cmp(reg, zr);
3574 %}
3575
3576 enc_class aarch64_enc_testn(iRegN src) %{
3577 Register reg = as_Register($src$$reg);
3578 __ cmpw(reg, zr);
3579 %}
3580
3581 enc_class aarch64_enc_b(label lbl) %{
3582 Label *L = $lbl$$label;
3583 __ b(*L);
3584 %}
3585
3586 enc_class aarch64_enc_br_con(cmpOp cmp, label lbl) %{
3587 Label *L = $lbl$$label;
3588 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3589 %}
3590
3591 enc_class aarch64_enc_br_conU(cmpOpU cmp, label lbl) %{
3592 Label *L = $lbl$$label;
3593 __ br ((Assembler::Condition)$cmp$$cmpcode, *L);
3594 %}
3595
3596 enc_class aarch64_enc_partial_subtype_check(iRegP sub, iRegP super, iRegP temp, iRegP result)
3597 %{
3598 Register sub_reg = as_Register($sub$$reg);
3599 Register super_reg = as_Register($super$$reg);
3600 Register temp_reg = as_Register($temp$$reg);
3601 Register result_reg = as_Register($result$$reg);
3602
3603 Label miss;
3604 __ check_klass_subtype_slow_path(sub_reg, super_reg, temp_reg, result_reg,
3605 nullptr, &miss,
3606 /*set_cond_codes:*/ true);
3607 if ($primary) {
3608 __ mov(result_reg, zr);
3609 }
3610 __ bind(miss);
3611 %}
3612
3613 enc_class aarch64_enc_java_static_call(method meth) %{
3614 address addr = (address)$meth$$method;
3615 address call;
3616 if (!_method) {
3617 // A call to a runtime wrapper, e.g. new, new_typeArray_Java, uncommon_trap.
3618 call = __ trampoline_call(Address(addr, relocInfo::runtime_call_type));
3619 if (call == nullptr) {
3620 ciEnv::current()->record_failure("CodeCache is full");
3621 return;
3622 }
3623 } else if (_method->intrinsic_id() == vmIntrinsicID::_ensureMaterializedForStackWalk) {
3624 // The NOP here is purely to ensure that eliding a call to
3625 // JVM_EnsureMaterializedForStackWalk doesn't change the code size.
3626 __ nop();
3627 __ block_comment("call JVM_EnsureMaterializedForStackWalk (elided)");
3628 } else {
3629 int method_index = resolved_method_index(masm);
3630 RelocationHolder rspec = _optimized_virtual ? opt_virtual_call_Relocation::spec(method_index)
3631 : static_call_Relocation::spec(method_index);
3632 call = __ trampoline_call(Address(addr, rspec));
3633 if (call == nullptr) {
3634 ciEnv::current()->record_failure("CodeCache is full");
3635 return;
3636 }
3637 if (CodeBuffer::supports_shared_stubs() && _method->can_be_statically_bound()) {
3638 // Calls of the same statically bound method can share
3639 // a stub to the interpreter.
3640 __ code()->shared_stub_to_interp_for(_method, call - __ begin());
3641 } else {
3642 // Emit stub for static call
3643 address stub = CompiledDirectCall::emit_to_interp_stub(masm, call);
3644 if (stub == nullptr) {
3645 ciEnv::current()->record_failure("CodeCache is full");
3646 return;
3647 }
3648 }
3649 }
3650
3651 __ post_call_nop();
3652
3653 // Only non uncommon_trap calls need to reinitialize ptrue.
3654 if (Compile::current()->max_vector_size() > 0 && uncommon_trap_request() == 0) {
3655 __ reinitialize_ptrue();
3656 }
3657 %}
3658
3659 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3660 int method_index = resolved_method_index(masm);
3661 address call = __ ic_call((address)$meth$$method, method_index);
3662 if (call == nullptr) {
3663 ciEnv::current()->record_failure("CodeCache is full");
3664 return;
3665 }
3666 __ post_call_nop();
3667 if (Compile::current()->max_vector_size() > 0) {
3668 __ reinitialize_ptrue();
3669 }
3670 %}
3671
3672 enc_class aarch64_enc_call_epilog() %{
3673 if (VerifyStackAtCalls) {
3674 // Check that stack depth is unchanged: find majik cookie on stack
3675 __ call_Unimplemented();
3676 }
3677 %}
3678
3679 enc_class aarch64_enc_java_to_runtime(method meth) %{
3680 // some calls to generated routines (arraycopy code) are scheduled
3681 // by C2 as runtime calls. if so we can call them using a br (they
3682 // will be in a reachable segment) otherwise we have to use a blr
3683 // which loads the absolute address into a register.
3684 address entry = (address)$meth$$method;
3685 CodeBlob *cb = CodeCache::find_blob(entry);
3686 if (cb) {
3687 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3688 if (call == nullptr) {
3689 ciEnv::current()->record_failure("CodeCache is full");
3690 return;
3691 }
3692 __ post_call_nop();
3693 } else {
3694 Label retaddr;
3695 __ adr(rscratch2, retaddr);
3696 __ lea(rscratch1, RuntimeAddress(entry));
3697 // Leave a breadcrumb for JavaFrameAnchor::capture_last_Java_pc()
3698 __ stp(zr, rscratch2, Address(__ pre(sp, -2 * wordSize)));
3699 __ blr(rscratch1);
3700 __ bind(retaddr);
3701 __ post_call_nop();
3702 __ add(sp, sp, 2 * wordSize);
3703 }
3704 if (Compile::current()->max_vector_size() > 0) {
3705 __ reinitialize_ptrue();
3706 }
3707 %}
3708
3709 enc_class aarch64_enc_rethrow() %{
3710 __ far_jump(RuntimeAddress(OptoRuntime::rethrow_stub()));
3711 %}
3712
3713 enc_class aarch64_enc_ret() %{
3714 #ifdef ASSERT
3715 if (Compile::current()->max_vector_size() > 0) {
3716 __ verify_ptrue();
3717 }
3718 #endif
3719 __ ret(lr);
3720 %}
3721
3722 enc_class aarch64_enc_tail_call(iRegP jump_target) %{
3723 Register target_reg = as_Register($jump_target$$reg);
3724 __ br(target_reg);
3725 %}
3726
3727 enc_class aarch64_enc_tail_jmp(iRegP jump_target) %{
3728 Register target_reg = as_Register($jump_target$$reg);
3729 // exception oop should be in r0
3730 // ret addr has been popped into lr
3731 // callee expects it in r3
3732 __ mov(r3, lr);
3733 __ br(target_reg);
3734 %}
3735
3736 %}
3737
3738 //----------FRAME--------------------------------------------------------------
3739 // Definition of frame structure and management information.
3740 //
3741 // S T A C K L A Y O U T Allocators stack-slot number
3742 // | (to get allocators register number
3743 // G Owned by | | v add OptoReg::stack0())
3744 // r CALLER | |
3745 // o | +--------+ pad to even-align allocators stack-slot
3746 // w V | pad0 | numbers; owned by CALLER
3747 // t -----------+--------+----> Matcher::_in_arg_limit, unaligned
3748 // h ^ | in | 5
3749 // | | args | 4 Holes in incoming args owned by SELF
3750 // | | | | 3
3751 // | | +--------+
3752 // V | | old out| Empty on Intel, window on Sparc
3753 // | old |preserve| Must be even aligned.
3754 // | SP-+--------+----> Matcher::_old_SP, even aligned
3755 // | | in | 3 area for Intel ret address
3756 // Owned by |preserve| Empty on Sparc.
3757 // SELF +--------+
3758 // | | pad2 | 2 pad to align old SP
3759 // | +--------+ 1
3760 // | | locks | 0
3761 // | +--------+----> OptoReg::stack0(), even aligned
3762 // | | pad1 | 11 pad to align new SP
3763 // | +--------+
3764 // | | | 10
3765 // | | spills | 9 spills
3766 // V | | 8 (pad0 slot for callee)
3767 // -----------+--------+----> Matcher::_out_arg_limit, unaligned
3768 // ^ | out | 7
3769 // | | args | 6 Holes in outgoing args owned by CALLEE
3770 // Owned by +--------+
3771 // CALLEE | new out| 6 Empty on Intel, window on Sparc
3772 // | new |preserve| Must be even-aligned.
3773 // | SP-+--------+----> Matcher::_new_SP, even aligned
3774 // | | |
3775 //
3776 // Note 1: Only region 8-11 is determined by the allocator. Region 0-5 is
3777 // known from SELF's arguments and the Java calling convention.
3778 // Region 6-7 is determined per call site.
3779 // Note 2: If the calling convention leaves holes in the incoming argument
3780 // area, those holes are owned by SELF. Holes in the outgoing area
3781 // are owned by the CALLEE. Holes should not be necessary in the
3782 // incoming area, as the Java calling convention is completely under
3783 // the control of the AD file. Doubles can be sorted and packed to
3784 // avoid holes. Holes in the outgoing arguments may be necessary for
3785 // varargs C calling conventions.
3786 // Note 3: Region 0-3 is even aligned, with pad2 as needed. Region 3-5 is
3787 // even aligned with pad0 as needed.
3788 // Region 6 is even aligned. Region 6-7 is NOT even aligned;
3789 // (the latter is true on Intel but is it false on AArch64?)
3790 // region 6-11 is even aligned; it may be padded out more so that
3791 // the region from SP to FP meets the minimum stack alignment.
3792 // Note 4: For I2C adapters, the incoming FP may not meet the minimum stack
3793 // alignment. Region 11, pad1, may be dynamically extended so that
3794 // SP meets the minimum alignment.
3795
3796 frame %{
3797 // These three registers define part of the calling convention
3798 // between compiled code and the interpreter.
3799
3800 // Inline Cache Register or Method for I2C.
3801 inline_cache_reg(R12);
3802
3803 // Number of stack slots consumed by locking an object
3804 sync_stack_slots(2);
3805
3806 // Compiled code's Frame Pointer
3807 frame_pointer(R31);
3808
3809 // Interpreter stores its frame pointer in a register which is
3810 // stored to the stack by I2CAdaptors.
3811 // I2CAdaptors convert from interpreted java to compiled java.
3812 interpreter_frame_pointer(R29);
3813
3814 // Stack alignment requirement
3815 stack_alignment(StackAlignmentInBytes); // Alignment size in bytes (128-bit -> 16 bytes)
3816
3817 // Number of outgoing stack slots killed above the out_preserve_stack_slots
3818 // for calls to C. Supports the var-args backing area for register parms.
3819 varargs_C_out_slots_killed(frame::arg_reg_save_area_bytes/BytesPerInt);
3820
3821 // The after-PROLOG location of the return address. Location of
3822 // return address specifies a type (REG or STACK) and a number
3823 // representing the register number (i.e. - use a register name) or
3824 // stack slot.
3825 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
3826 // Otherwise, it is above the locks and verification slot and alignment word
3827 // TODO this may well be correct but need to check why that - 2 is there
3828 // ppc port uses 0 but we definitely need to allow for fixed_slots
3829 // which folds in the space used for monitors
3830 return_addr(STACK - 2 +
3831 align_up((Compile::current()->in_preserve_stack_slots() +
3832 Compile::current()->fixed_slots()),
3833 stack_alignment_in_slots()));
3834
3835 // Location of compiled Java return values. Same as C for now.
3836 return_value
3837 %{
3838 // TODO do we allow ideal_reg == Op_RegN???
3839 assert(ideal_reg >= Op_RegI && ideal_reg <= Op_RegL,
3840 "only return normal values");
3841
3842 static const int lo[Op_RegL + 1] = { // enum name
3843 0, // Op_Node
3844 0, // Op_Set
3845 R0_num, // Op_RegN
3846 R0_num, // Op_RegI
3847 R0_num, // Op_RegP
3848 V0_num, // Op_RegF
3849 V0_num, // Op_RegD
3850 R0_num // Op_RegL
3851 };
3852
3853 static const int hi[Op_RegL + 1] = { // enum name
3854 0, // Op_Node
3855 0, // Op_Set
3856 OptoReg::Bad, // Op_RegN
3857 OptoReg::Bad, // Op_RegI
3858 R0_H_num, // Op_RegP
3859 OptoReg::Bad, // Op_RegF
3860 V0_H_num, // Op_RegD
3861 R0_H_num // Op_RegL
3862 };
3863
3864 return OptoRegPair(hi[ideal_reg], lo[ideal_reg]);
3865 %}
3866 %}
3867
3868 //----------ATTRIBUTES---------------------------------------------------------
3869 //----------Operand Attributes-------------------------------------------------
3870 op_attrib op_cost(1); // Required cost attribute
3871
3872 //----------Instruction Attributes---------------------------------------------
3873 ins_attrib ins_cost(INSN_COST); // Required cost attribute
3874 ins_attrib ins_size(32); // Required size attribute (in bits)
3875 ins_attrib ins_short_branch(0); // Required flag: is this instruction
3876 // a non-matching short branch variant
3877 // of some long branch?
3878 ins_attrib ins_alignment(4); // Required alignment attribute (must
3879 // be a power of 2) specifies the
3880 // alignment that some part of the
3881 // instruction (not necessarily the
3882 // start) requires. If > 1, a
3883 // compute_padding() function must be
3884 // provided for the instruction
3885
3886 //----------OPERANDS-----------------------------------------------------------
3887 // Operand definitions must precede instruction definitions for correct parsing
3888 // in the ADLC because operands constitute user defined types which are used in
3889 // instruction definitions.
3890
3891 //----------Simple Operands----------------------------------------------------
3892
3893 // Integer operands 32 bit
3894 // 32 bit immediate
3895 operand immI()
3896 %{
3897 match(ConI);
3898
3899 op_cost(0);
3900 format %{ %}
3901 interface(CONST_INTER);
3902 %}
3903
3904 // 32 bit zero
3905 operand immI0()
3906 %{
3907 predicate(n->get_int() == 0);
3908 match(ConI);
3909
3910 op_cost(0);
3911 format %{ %}
3912 interface(CONST_INTER);
3913 %}
3914
3915 // 32 bit unit increment
3916 operand immI_1()
3917 %{
3918 predicate(n->get_int() == 1);
3919 match(ConI);
3920
3921 op_cost(0);
3922 format %{ %}
3923 interface(CONST_INTER);
3924 %}
3925
3926 // 32 bit unit decrement
3927 operand immI_M1()
3928 %{
3929 predicate(n->get_int() == -1);
3930 match(ConI);
3931
3932 op_cost(0);
3933 format %{ %}
3934 interface(CONST_INTER);
3935 %}
3936
3937 // Shift values for add/sub extension shift
3938 operand immIExt()
3939 %{
3940 predicate(0 <= n->get_int() && (n->get_int() <= 4));
3941 match(ConI);
3942
3943 op_cost(0);
3944 format %{ %}
3945 interface(CONST_INTER);
3946 %}
3947
3948 operand immI_gt_1()
3949 %{
3950 predicate(n->get_int() > 1);
3951 match(ConI);
3952
3953 op_cost(0);
3954 format %{ %}
3955 interface(CONST_INTER);
3956 %}
3957
3958 operand immI_le_4()
3959 %{
3960 predicate(n->get_int() <= 4);
3961 match(ConI);
3962
3963 op_cost(0);
3964 format %{ %}
3965 interface(CONST_INTER);
3966 %}
3967
3968 operand immI_16()
3969 %{
3970 predicate(n->get_int() == 16);
3971 match(ConI);
3972
3973 op_cost(0);
3974 format %{ %}
3975 interface(CONST_INTER);
3976 %}
3977
3978 operand immI_24()
3979 %{
3980 predicate(n->get_int() == 24);
3981 match(ConI);
3982
3983 op_cost(0);
3984 format %{ %}
3985 interface(CONST_INTER);
3986 %}
3987
3988 operand immI_32()
3989 %{
3990 predicate(n->get_int() == 32);
3991 match(ConI);
3992
3993 op_cost(0);
3994 format %{ %}
3995 interface(CONST_INTER);
3996 %}
3997
3998 operand immI_48()
3999 %{
4000 predicate(n->get_int() == 48);
4001 match(ConI);
4002
4003 op_cost(0);
4004 format %{ %}
4005 interface(CONST_INTER);
4006 %}
4007
4008 operand immI_56()
4009 %{
4010 predicate(n->get_int() == 56);
4011 match(ConI);
4012
4013 op_cost(0);
4014 format %{ %}
4015 interface(CONST_INTER);
4016 %}
4017
4018 operand immI_255()
4019 %{
4020 predicate(n->get_int() == 255);
4021 match(ConI);
4022
4023 op_cost(0);
4024 format %{ %}
4025 interface(CONST_INTER);
4026 %}
4027
4028 operand immI_65535()
4029 %{
4030 predicate(n->get_int() == 65535);
4031 match(ConI);
4032
4033 op_cost(0);
4034 format %{ %}
4035 interface(CONST_INTER);
4036 %}
4037
4038 operand immI_positive()
4039 %{
4040 predicate(n->get_int() > 0);
4041 match(ConI);
4042
4043 op_cost(0);
4044 format %{ %}
4045 interface(CONST_INTER);
4046 %}
4047
4048 // BoolTest condition for signed compare
4049 operand immI_cmp_cond()
4050 %{
4051 predicate(!Matcher::is_unsigned_booltest_pred(n->get_int()));
4052 match(ConI);
4053
4054 op_cost(0);
4055 format %{ %}
4056 interface(CONST_INTER);
4057 %}
4058
4059 // BoolTest condition for unsigned compare
4060 operand immI_cmpU_cond()
4061 %{
4062 predicate(Matcher::is_unsigned_booltest_pred(n->get_int()));
4063 match(ConI);
4064
4065 op_cost(0);
4066 format %{ %}
4067 interface(CONST_INTER);
4068 %}
4069
4070 operand immL_255()
4071 %{
4072 predicate(n->get_long() == 255L);
4073 match(ConL);
4074
4075 op_cost(0);
4076 format %{ %}
4077 interface(CONST_INTER);
4078 %}
4079
4080 operand immL_65535()
4081 %{
4082 predicate(n->get_long() == 65535L);
4083 match(ConL);
4084
4085 op_cost(0);
4086 format %{ %}
4087 interface(CONST_INTER);
4088 %}
4089
4090 operand immL_4294967295()
4091 %{
4092 predicate(n->get_long() == 4294967295L);
4093 match(ConL);
4094
4095 op_cost(0);
4096 format %{ %}
4097 interface(CONST_INTER);
4098 %}
4099
4100 operand immL_bitmask()
4101 %{
4102 predicate((n->get_long() != 0)
4103 && ((n->get_long() & 0xc000000000000000l) == 0)
4104 && is_power_of_2(n->get_long() + 1));
4105 match(ConL);
4106
4107 op_cost(0);
4108 format %{ %}
4109 interface(CONST_INTER);
4110 %}
4111
4112 operand immI_bitmask()
4113 %{
4114 predicate((n->get_int() != 0)
4115 && ((n->get_int() & 0xc0000000) == 0)
4116 && is_power_of_2(n->get_int() + 1));
4117 match(ConI);
4118
4119 op_cost(0);
4120 format %{ %}
4121 interface(CONST_INTER);
4122 %}
4123
4124 operand immL_positive_bitmaskI()
4125 %{
4126 predicate((n->get_long() != 0)
4127 && ((julong)n->get_long() < 0x80000000ULL)
4128 && is_power_of_2(n->get_long() + 1));
4129 match(ConL);
4130
4131 op_cost(0);
4132 format %{ %}
4133 interface(CONST_INTER);
4134 %}
4135
4136 // Scale values for scaled offset addressing modes (up to long but not quad)
4137 operand immIScale()
4138 %{
4139 predicate(0 <= n->get_int() && (n->get_int() <= 3));
4140 match(ConI);
4141
4142 op_cost(0);
4143 format %{ %}
4144 interface(CONST_INTER);
4145 %}
4146
4147 // 5 bit signed integer
4148 operand immI5()
4149 %{
4150 predicate(Assembler::is_simm(n->get_int(), 5));
4151 match(ConI);
4152
4153 op_cost(0);
4154 format %{ %}
4155 interface(CONST_INTER);
4156 %}
4157
4158 // 7 bit unsigned integer
4159 operand immIU7()
4160 %{
4161 predicate(Assembler::is_uimm(n->get_int(), 7));
4162 match(ConI);
4163
4164 op_cost(0);
4165 format %{ %}
4166 interface(CONST_INTER);
4167 %}
4168
4169 // Offset for scaled or unscaled immediate loads and stores
4170 operand immIOffset()
4171 %{
4172 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4173 match(ConI);
4174
4175 op_cost(0);
4176 format %{ %}
4177 interface(CONST_INTER);
4178 %}
4179
4180 operand immIOffset1()
4181 %{
4182 predicate(Address::offset_ok_for_immed(n->get_int(), 0));
4183 match(ConI);
4184
4185 op_cost(0);
4186 format %{ %}
4187 interface(CONST_INTER);
4188 %}
4189
4190 operand immIOffset2()
4191 %{
4192 predicate(Address::offset_ok_for_immed(n->get_int(), 1));
4193 match(ConI);
4194
4195 op_cost(0);
4196 format %{ %}
4197 interface(CONST_INTER);
4198 %}
4199
4200 operand immIOffset4()
4201 %{
4202 predicate(Address::offset_ok_for_immed(n->get_int(), 2));
4203 match(ConI);
4204
4205 op_cost(0);
4206 format %{ %}
4207 interface(CONST_INTER);
4208 %}
4209
4210 operand immIOffset8()
4211 %{
4212 predicate(Address::offset_ok_for_immed(n->get_int(), 3));
4213 match(ConI);
4214
4215 op_cost(0);
4216 format %{ %}
4217 interface(CONST_INTER);
4218 %}
4219
4220 operand immIOffset16()
4221 %{
4222 predicate(Address::offset_ok_for_immed(n->get_int(), 4));
4223 match(ConI);
4224
4225 op_cost(0);
4226 format %{ %}
4227 interface(CONST_INTER);
4228 %}
4229
4230 operand immLOffset()
4231 %{
4232 predicate(n->get_long() >= -256 && n->get_long() <= 65520);
4233 match(ConL);
4234
4235 op_cost(0);
4236 format %{ %}
4237 interface(CONST_INTER);
4238 %}
4239
4240 operand immLoffset1()
4241 %{
4242 predicate(Address::offset_ok_for_immed(n->get_long(), 0));
4243 match(ConL);
4244
4245 op_cost(0);
4246 format %{ %}
4247 interface(CONST_INTER);
4248 %}
4249
4250 operand immLoffset2()
4251 %{
4252 predicate(Address::offset_ok_for_immed(n->get_long(), 1));
4253 match(ConL);
4254
4255 op_cost(0);
4256 format %{ %}
4257 interface(CONST_INTER);
4258 %}
4259
4260 operand immLoffset4()
4261 %{
4262 predicate(Address::offset_ok_for_immed(n->get_long(), 2));
4263 match(ConL);
4264
4265 op_cost(0);
4266 format %{ %}
4267 interface(CONST_INTER);
4268 %}
4269
4270 operand immLoffset8()
4271 %{
4272 predicate(Address::offset_ok_for_immed(n->get_long(), 3));
4273 match(ConL);
4274
4275 op_cost(0);
4276 format %{ %}
4277 interface(CONST_INTER);
4278 %}
4279
4280 operand immLoffset16()
4281 %{
4282 predicate(Address::offset_ok_for_immed(n->get_long(), 4));
4283 match(ConL);
4284
4285 op_cost(0);
4286 format %{ %}
4287 interface(CONST_INTER);
4288 %}
4289
4290 // 5 bit signed long integer
4291 operand immL5()
4292 %{
4293 predicate(Assembler::is_simm(n->get_long(), 5));
4294 match(ConL);
4295
4296 op_cost(0);
4297 format %{ %}
4298 interface(CONST_INTER);
4299 %}
4300
4301 // 7 bit unsigned long integer
4302 operand immLU7()
4303 %{
4304 predicate(Assembler::is_uimm(n->get_long(), 7));
4305 match(ConL);
4306
4307 op_cost(0);
4308 format %{ %}
4309 interface(CONST_INTER);
4310 %}
4311
4312 // 8 bit signed value.
4313 operand immI8()
4314 %{
4315 predicate(n->get_int() <= 127 && n->get_int() >= -128);
4316 match(ConI);
4317
4318 op_cost(0);
4319 format %{ %}
4320 interface(CONST_INTER);
4321 %}
4322
4323 // 8 bit signed value (simm8), or #simm8 LSL 8.
4324 operand immI8_shift8()
4325 %{
4326 predicate((n->get_int() <= 127 && n->get_int() >= -128) ||
4327 (n->get_int() <= 32512 && n->get_int() >= -32768 && (n->get_int() & 0xff) == 0));
4328 match(ConI);
4329
4330 op_cost(0);
4331 format %{ %}
4332 interface(CONST_INTER);
4333 %}
4334
4335 // 8 bit signed value (simm8), or #simm8 LSL 8.
4336 operand immL8_shift8()
4337 %{
4338 predicate((n->get_long() <= 127 && n->get_long() >= -128) ||
4339 (n->get_long() <= 32512 && n->get_long() >= -32768 && (n->get_long() & 0xff) == 0));
4340 match(ConL);
4341
4342 op_cost(0);
4343 format %{ %}
4344 interface(CONST_INTER);
4345 %}
4346
4347 // 8 bit integer valid for vector add sub immediate
4348 operand immBAddSubV()
4349 %{
4350 predicate(n->get_int() <= 255 && n->get_int() >= -255);
4351 match(ConI);
4352
4353 op_cost(0);
4354 format %{ %}
4355 interface(CONST_INTER);
4356 %}
4357
4358 // 32 bit integer valid for add sub immediate
4359 operand immIAddSub()
4360 %{
4361 predicate(Assembler::operand_valid_for_add_sub_immediate((int64_t)n->get_int()));
4362 match(ConI);
4363 op_cost(0);
4364 format %{ %}
4365 interface(CONST_INTER);
4366 %}
4367
4368 // 32 bit integer valid for vector add sub immediate
4369 operand immIAddSubV()
4370 %{
4371 predicate(Assembler::operand_valid_for_sve_add_sub_immediate((int64_t)n->get_int()));
4372 match(ConI);
4373
4374 op_cost(0);
4375 format %{ %}
4376 interface(CONST_INTER);
4377 %}
4378
4379 // 32 bit unsigned integer valid for logical immediate
4380
4381 operand immBLog()
4382 %{
4383 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerByte, (uint64_t)n->get_int()));
4384 match(ConI);
4385
4386 op_cost(0);
4387 format %{ %}
4388 interface(CONST_INTER);
4389 %}
4390
4391 operand immSLog()
4392 %{
4393 predicate(Assembler::operand_valid_for_sve_logical_immediate(BitsPerShort, (uint64_t)n->get_int()));
4394 match(ConI);
4395
4396 op_cost(0);
4397 format %{ %}
4398 interface(CONST_INTER);
4399 %}
4400
4401 operand immILog()
4402 %{
4403 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/true, (uint64_t)n->get_int()));
4404 match(ConI);
4405
4406 op_cost(0);
4407 format %{ %}
4408 interface(CONST_INTER);
4409 %}
4410
4411 // Integer operands 64 bit
4412 // 64 bit immediate
4413 operand immL()
4414 %{
4415 match(ConL);
4416
4417 op_cost(0);
4418 format %{ %}
4419 interface(CONST_INTER);
4420 %}
4421
4422 // 64 bit zero
4423 operand immL0()
4424 %{
4425 predicate(n->get_long() == 0);
4426 match(ConL);
4427
4428 op_cost(0);
4429 format %{ %}
4430 interface(CONST_INTER);
4431 %}
4432
4433 // 64 bit unit decrement
4434 operand immL_M1()
4435 %{
4436 predicate(n->get_long() == -1);
4437 match(ConL);
4438
4439 op_cost(0);
4440 format %{ %}
4441 interface(CONST_INTER);
4442 %}
4443
4444 // 64 bit integer valid for add sub immediate
4445 operand immLAddSub()
4446 %{
4447 predicate(Assembler::operand_valid_for_add_sub_immediate(n->get_long()));
4448 match(ConL);
4449 op_cost(0);
4450 format %{ %}
4451 interface(CONST_INTER);
4452 %}
4453
4454 // 64 bit integer valid for addv subv immediate
4455 operand immLAddSubV()
4456 %{
4457 predicate(Assembler::operand_valid_for_sve_add_sub_immediate(n->get_long()));
4458 match(ConL);
4459
4460 op_cost(0);
4461 format %{ %}
4462 interface(CONST_INTER);
4463 %}
4464
4465 // 64 bit integer valid for logical immediate
4466 operand immLLog()
4467 %{
4468 predicate(Assembler::operand_valid_for_logical_immediate(/*is32*/false, (uint64_t)n->get_long()));
4469 match(ConL);
4470 op_cost(0);
4471 format %{ %}
4472 interface(CONST_INTER);
4473 %}
4474
4475 // Long Immediate: low 32-bit mask
4476 operand immL_32bits()
4477 %{
4478 predicate(n->get_long() == 0xFFFFFFFFL);
4479 match(ConL);
4480 op_cost(0);
4481 format %{ %}
4482 interface(CONST_INTER);
4483 %}
4484
4485 // Pointer operands
4486 // Pointer Immediate
4487 operand immP()
4488 %{
4489 match(ConP);
4490
4491 op_cost(0);
4492 format %{ %}
4493 interface(CONST_INTER);
4494 %}
4495
4496 // nullptr Pointer Immediate
4497 operand immP0()
4498 %{
4499 predicate(n->get_ptr() == 0);
4500 match(ConP);
4501
4502 op_cost(0);
4503 format %{ %}
4504 interface(CONST_INTER);
4505 %}
4506
4507 // Pointer Immediate One
4508 // this is used in object initialization (initial object header)
4509 operand immP_1()
4510 %{
4511 predicate(n->get_ptr() == 1);
4512 match(ConP);
4513
4514 op_cost(0);
4515 format %{ %}
4516 interface(CONST_INTER);
4517 %}
4518
4519 // Card Table Byte Map Base
4520 operand immByteMapBase()
4521 %{
4522 // Get base of card map
4523 predicate(BarrierSet::barrier_set()->is_a(BarrierSet::CardTableBarrierSet) &&
4524 (CardTable::CardValue*)n->get_ptr() == ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base());
4525 match(ConP);
4526
4527 op_cost(0);
4528 format %{ %}
4529 interface(CONST_INTER);
4530 %}
4531
4532 // Float and Double operands
4533 // Double Immediate
4534 operand immD()
4535 %{
4536 match(ConD);
4537 op_cost(0);
4538 format %{ %}
4539 interface(CONST_INTER);
4540 %}
4541
4542 // Double Immediate: +0.0d
4543 operand immD0()
4544 %{
4545 predicate(jlong_cast(n->getd()) == 0);
4546 match(ConD);
4547
4548 op_cost(0);
4549 format %{ %}
4550 interface(CONST_INTER);
4551 %}
4552
4553 // constant 'double +0.0'.
4554 operand immDPacked()
4555 %{
4556 predicate(Assembler::operand_valid_for_float_immediate(n->getd()));
4557 match(ConD);
4558 op_cost(0);
4559 format %{ %}
4560 interface(CONST_INTER);
4561 %}
4562
4563 // Float Immediate
4564 operand immF()
4565 %{
4566 match(ConF);
4567 op_cost(0);
4568 format %{ %}
4569 interface(CONST_INTER);
4570 %}
4571
4572 // Float Immediate: +0.0f.
4573 operand immF0()
4574 %{
4575 predicate(jint_cast(n->getf()) == 0);
4576 match(ConF);
4577
4578 op_cost(0);
4579 format %{ %}
4580 interface(CONST_INTER);
4581 %}
4582
4583 //
4584 operand immFPacked()
4585 %{
4586 predicate(Assembler::operand_valid_for_float_immediate((double)n->getf()));
4587 match(ConF);
4588 op_cost(0);
4589 format %{ %}
4590 interface(CONST_INTER);
4591 %}
4592
4593 // Narrow pointer operands
4594 // Narrow Pointer Immediate
4595 operand immN()
4596 %{
4597 match(ConN);
4598
4599 op_cost(0);
4600 format %{ %}
4601 interface(CONST_INTER);
4602 %}
4603
4604 // Narrow nullptr Pointer Immediate
4605 operand immN0()
4606 %{
4607 predicate(n->get_narrowcon() == 0);
4608 match(ConN);
4609
4610 op_cost(0);
4611 format %{ %}
4612 interface(CONST_INTER);
4613 %}
4614
4615 operand immNKlass()
4616 %{
4617 match(ConNKlass);
4618
4619 op_cost(0);
4620 format %{ %}
4621 interface(CONST_INTER);
4622 %}
4623
4624 // Integer 32 bit Register Operands
4625 // Integer 32 bitRegister (excludes SP)
4626 operand iRegI()
4627 %{
4628 constraint(ALLOC_IN_RC(any_reg32));
4629 match(RegI);
4630 match(iRegINoSp);
4631 op_cost(0);
4632 format %{ %}
4633 interface(REG_INTER);
4634 %}
4635
4636 // Integer 32 bit Register not Special
4637 operand iRegINoSp()
4638 %{
4639 constraint(ALLOC_IN_RC(no_special_reg32));
4640 match(RegI);
4641 op_cost(0);
4642 format %{ %}
4643 interface(REG_INTER);
4644 %}
4645
4646 // Integer 64 bit Register Operands
4647 // Integer 64 bit Register (includes SP)
4648 operand iRegL()
4649 %{
4650 constraint(ALLOC_IN_RC(any_reg));
4651 match(RegL);
4652 match(iRegLNoSp);
4653 op_cost(0);
4654 format %{ %}
4655 interface(REG_INTER);
4656 %}
4657
4658 // Integer 64 bit Register not Special
4659 operand iRegLNoSp()
4660 %{
4661 constraint(ALLOC_IN_RC(no_special_reg));
4662 match(RegL);
4663 match(iRegL_R0);
4664 format %{ %}
4665 interface(REG_INTER);
4666 %}
4667
4668 // Pointer Register Operands
4669 // Pointer Register
4670 operand iRegP()
4671 %{
4672 constraint(ALLOC_IN_RC(ptr_reg));
4673 match(RegP);
4674 match(iRegPNoSp);
4675 match(iRegP_R0);
4676 //match(iRegP_R2);
4677 //match(iRegP_R4);
4678 match(iRegP_R5);
4679 match(thread_RegP);
4680 op_cost(0);
4681 format %{ %}
4682 interface(REG_INTER);
4683 %}
4684
4685 // Pointer 64 bit Register not Special
4686 operand iRegPNoSp()
4687 %{
4688 constraint(ALLOC_IN_RC(no_special_ptr_reg));
4689 match(RegP);
4690 // match(iRegP);
4691 // match(iRegP_R0);
4692 // match(iRegP_R2);
4693 // match(iRegP_R4);
4694 // match(iRegP_R5);
4695 // match(thread_RegP);
4696 op_cost(0);
4697 format %{ %}
4698 interface(REG_INTER);
4699 %}
4700
4701 // This operand is not allowed to use rfp even if
4702 // rfp is not used to hold the frame pointer.
4703 operand iRegPNoSpNoRfp()
4704 %{
4705 constraint(ALLOC_IN_RC(no_special_no_rfp_ptr_reg));
4706 match(RegP);
4707 match(iRegPNoSp);
4708 op_cost(0);
4709 format %{ %}
4710 interface(REG_INTER);
4711 %}
4712
4713 // Pointer 64 bit Register R0 only
4714 operand iRegP_R0()
4715 %{
4716 constraint(ALLOC_IN_RC(r0_reg));
4717 match(RegP);
4718 // match(iRegP);
4719 match(iRegPNoSp);
4720 op_cost(0);
4721 format %{ %}
4722 interface(REG_INTER);
4723 %}
4724
4725 // Pointer 64 bit Register R1 only
4726 operand iRegP_R1()
4727 %{
4728 constraint(ALLOC_IN_RC(r1_reg));
4729 match(RegP);
4730 // match(iRegP);
4731 match(iRegPNoSp);
4732 op_cost(0);
4733 format %{ %}
4734 interface(REG_INTER);
4735 %}
4736
4737 // Pointer 64 bit Register R2 only
4738 operand iRegP_R2()
4739 %{
4740 constraint(ALLOC_IN_RC(r2_reg));
4741 match(RegP);
4742 // match(iRegP);
4743 match(iRegPNoSp);
4744 op_cost(0);
4745 format %{ %}
4746 interface(REG_INTER);
4747 %}
4748
4749 // Pointer 64 bit Register R3 only
4750 operand iRegP_R3()
4751 %{
4752 constraint(ALLOC_IN_RC(r3_reg));
4753 match(RegP);
4754 // match(iRegP);
4755 match(iRegPNoSp);
4756 op_cost(0);
4757 format %{ %}
4758 interface(REG_INTER);
4759 %}
4760
4761 // Pointer 64 bit Register R4 only
4762 operand iRegP_R4()
4763 %{
4764 constraint(ALLOC_IN_RC(r4_reg));
4765 match(RegP);
4766 // match(iRegP);
4767 match(iRegPNoSp);
4768 op_cost(0);
4769 format %{ %}
4770 interface(REG_INTER);
4771 %}
4772
4773 // Pointer 64 bit Register R5 only
4774 operand iRegP_R5()
4775 %{
4776 constraint(ALLOC_IN_RC(r5_reg));
4777 match(RegP);
4778 // match(iRegP);
4779 match(iRegPNoSp);
4780 op_cost(0);
4781 format %{ %}
4782 interface(REG_INTER);
4783 %}
4784
4785 // Pointer 64 bit Register R10 only
4786 operand iRegP_R10()
4787 %{
4788 constraint(ALLOC_IN_RC(r10_reg));
4789 match(RegP);
4790 // match(iRegP);
4791 match(iRegPNoSp);
4792 op_cost(0);
4793 format %{ %}
4794 interface(REG_INTER);
4795 %}
4796
4797 // Long 64 bit Register R0 only
4798 operand iRegL_R0()
4799 %{
4800 constraint(ALLOC_IN_RC(r0_reg));
4801 match(RegL);
4802 match(iRegLNoSp);
4803 op_cost(0);
4804 format %{ %}
4805 interface(REG_INTER);
4806 %}
4807
4808 // Long 64 bit Register R11 only
4809 operand iRegL_R11()
4810 %{
4811 constraint(ALLOC_IN_RC(r11_reg));
4812 match(RegL);
4813 match(iRegLNoSp);
4814 op_cost(0);
4815 format %{ %}
4816 interface(REG_INTER);
4817 %}
4818
4819 // Register R0 only
4820 operand iRegI_R0()
4821 %{
4822 constraint(ALLOC_IN_RC(int_r0_reg));
4823 match(RegI);
4824 match(iRegINoSp);
4825 op_cost(0);
4826 format %{ %}
4827 interface(REG_INTER);
4828 %}
4829
4830 // Register R2 only
4831 operand iRegI_R2()
4832 %{
4833 constraint(ALLOC_IN_RC(int_r2_reg));
4834 match(RegI);
4835 match(iRegINoSp);
4836 op_cost(0);
4837 format %{ %}
4838 interface(REG_INTER);
4839 %}
4840
4841 // Register R3 only
4842 operand iRegI_R3()
4843 %{
4844 constraint(ALLOC_IN_RC(int_r3_reg));
4845 match(RegI);
4846 match(iRegINoSp);
4847 op_cost(0);
4848 format %{ %}
4849 interface(REG_INTER);
4850 %}
4851
4852
4853 // Register R4 only
4854 operand iRegI_R4()
4855 %{
4856 constraint(ALLOC_IN_RC(int_r4_reg));
4857 match(RegI);
4858 match(iRegINoSp);
4859 op_cost(0);
4860 format %{ %}
4861 interface(REG_INTER);
4862 %}
4863
4864
4865 // Pointer Register Operands
4866 // Narrow Pointer Register
4867 operand iRegN()
4868 %{
4869 constraint(ALLOC_IN_RC(any_reg32));
4870 match(RegN);
4871 match(iRegNNoSp);
4872 op_cost(0);
4873 format %{ %}
4874 interface(REG_INTER);
4875 %}
4876
4877 // Integer 64 bit Register not Special
4878 operand iRegNNoSp()
4879 %{
4880 constraint(ALLOC_IN_RC(no_special_reg32));
4881 match(RegN);
4882 op_cost(0);
4883 format %{ %}
4884 interface(REG_INTER);
4885 %}
4886
4887 // Float Register
4888 // Float register operands
4889 operand vRegF()
4890 %{
4891 constraint(ALLOC_IN_RC(float_reg));
4892 match(RegF);
4893
4894 op_cost(0);
4895 format %{ %}
4896 interface(REG_INTER);
4897 %}
4898
4899 // Double Register
4900 // Double register operands
4901 operand vRegD()
4902 %{
4903 constraint(ALLOC_IN_RC(double_reg));
4904 match(RegD);
4905
4906 op_cost(0);
4907 format %{ %}
4908 interface(REG_INTER);
4909 %}
4910
4911 // Generic vector class. This will be used for
4912 // all vector operands, including NEON and SVE.
4913 operand vReg()
4914 %{
4915 constraint(ALLOC_IN_RC(dynamic));
4916 match(VecA);
4917 match(VecD);
4918 match(VecX);
4919
4920 op_cost(0);
4921 format %{ %}
4922 interface(REG_INTER);
4923 %}
4924
4925 operand vecA()
4926 %{
4927 constraint(ALLOC_IN_RC(vectora_reg));
4928 match(VecA);
4929
4930 op_cost(0);
4931 format %{ %}
4932 interface(REG_INTER);
4933 %}
4934
4935 operand vecD()
4936 %{
4937 constraint(ALLOC_IN_RC(vectord_reg));
4938 match(VecD);
4939
4940 op_cost(0);
4941 format %{ %}
4942 interface(REG_INTER);
4943 %}
4944
4945 operand vecX()
4946 %{
4947 constraint(ALLOC_IN_RC(vectorx_reg));
4948 match(VecX);
4949
4950 op_cost(0);
4951 format %{ %}
4952 interface(REG_INTER);
4953 %}
4954
4955 operand vRegD_V0()
4956 %{
4957 constraint(ALLOC_IN_RC(v0_reg));
4958 match(RegD);
4959 op_cost(0);
4960 format %{ %}
4961 interface(REG_INTER);
4962 %}
4963
4964 operand vRegD_V1()
4965 %{
4966 constraint(ALLOC_IN_RC(v1_reg));
4967 match(RegD);
4968 op_cost(0);
4969 format %{ %}
4970 interface(REG_INTER);
4971 %}
4972
4973 operand vRegD_V2()
4974 %{
4975 constraint(ALLOC_IN_RC(v2_reg));
4976 match(RegD);
4977 op_cost(0);
4978 format %{ %}
4979 interface(REG_INTER);
4980 %}
4981
4982 operand vRegD_V3()
4983 %{
4984 constraint(ALLOC_IN_RC(v3_reg));
4985 match(RegD);
4986 op_cost(0);
4987 format %{ %}
4988 interface(REG_INTER);
4989 %}
4990
4991 operand vRegD_V4()
4992 %{
4993 constraint(ALLOC_IN_RC(v4_reg));
4994 match(RegD);
4995 op_cost(0);
4996 format %{ %}
4997 interface(REG_INTER);
4998 %}
4999
5000 operand vRegD_V5()
5001 %{
5002 constraint(ALLOC_IN_RC(v5_reg));
5003 match(RegD);
5004 op_cost(0);
5005 format %{ %}
5006 interface(REG_INTER);
5007 %}
5008
5009 operand vRegD_V6()
5010 %{
5011 constraint(ALLOC_IN_RC(v6_reg));
5012 match(RegD);
5013 op_cost(0);
5014 format %{ %}
5015 interface(REG_INTER);
5016 %}
5017
5018 operand vRegD_V7()
5019 %{
5020 constraint(ALLOC_IN_RC(v7_reg));
5021 match(RegD);
5022 op_cost(0);
5023 format %{ %}
5024 interface(REG_INTER);
5025 %}
5026
5027 operand pReg()
5028 %{
5029 constraint(ALLOC_IN_RC(pr_reg));
5030 match(RegVectMask);
5031 match(pRegGov);
5032 op_cost(0);
5033 format %{ %}
5034 interface(REG_INTER);
5035 %}
5036
5037 operand pRegGov()
5038 %{
5039 constraint(ALLOC_IN_RC(gov_pr));
5040 match(RegVectMask);
5041 match(pReg);
5042 op_cost(0);
5043 format %{ %}
5044 interface(REG_INTER);
5045 %}
5046
5047 operand pRegGov_P0()
5048 %{
5049 constraint(ALLOC_IN_RC(p0_reg));
5050 match(RegVectMask);
5051 op_cost(0);
5052 format %{ %}
5053 interface(REG_INTER);
5054 %}
5055
5056 operand pRegGov_P1()
5057 %{
5058 constraint(ALLOC_IN_RC(p1_reg));
5059 match(RegVectMask);
5060 op_cost(0);
5061 format %{ %}
5062 interface(REG_INTER);
5063 %}
5064
5065 // Flags register, used as output of signed compare instructions
5066
5067 // note that on AArch64 we also use this register as the output for
5068 // for floating point compare instructions (CmpF CmpD). this ensures
5069 // that ordered inequality tests use GT, GE, LT or LE none of which
5070 // pass through cases where the result is unordered i.e. one or both
5071 // inputs to the compare is a NaN. this means that the ideal code can
5072 // replace e.g. a GT with an LE and not end up capturing the NaN case
5073 // (where the comparison should always fail). EQ and NE tests are
5074 // always generated in ideal code so that unordered folds into the NE
5075 // case, matching the behaviour of AArch64 NE.
5076 //
5077 // This differs from x86 where the outputs of FP compares use a
5078 // special FP flags registers and where compares based on this
5079 // register are distinguished into ordered inequalities (cmpOpUCF) and
5080 // EQ/NEQ tests (cmpOpUCF2). x86 has to special case the latter tests
5081 // to explicitly handle the unordered case in branches. x86 also has
5082 // to include extra CMoveX rules to accept a cmpOpUCF input.
5083
5084 operand rFlagsReg()
5085 %{
5086 constraint(ALLOC_IN_RC(int_flags));
5087 match(RegFlags);
5088
5089 op_cost(0);
5090 format %{ "RFLAGS" %}
5091 interface(REG_INTER);
5092 %}
5093
5094 // Flags register, used as output of unsigned compare instructions
5095 operand rFlagsRegU()
5096 %{
5097 constraint(ALLOC_IN_RC(int_flags));
5098 match(RegFlags);
5099
5100 op_cost(0);
5101 format %{ "RFLAGSU" %}
5102 interface(REG_INTER);
5103 %}
5104
5105 // Special Registers
5106
5107 // Method Register
5108 operand inline_cache_RegP(iRegP reg)
5109 %{
5110 constraint(ALLOC_IN_RC(method_reg)); // inline_cache_reg
5111 match(reg);
5112 match(iRegPNoSp);
5113 op_cost(0);
5114 format %{ %}
5115 interface(REG_INTER);
5116 %}
5117
5118 // Thread Register
5119 operand thread_RegP(iRegP reg)
5120 %{
5121 constraint(ALLOC_IN_RC(thread_reg)); // link_reg
5122 match(reg);
5123 op_cost(0);
5124 format %{ %}
5125 interface(REG_INTER);
5126 %}
5127
5128 //----------Memory Operands----------------------------------------------------
5129
5130 operand indirect(iRegP reg)
5131 %{
5132 constraint(ALLOC_IN_RC(ptr_reg));
5133 match(reg);
5134 op_cost(0);
5135 format %{ "[$reg]" %}
5136 interface(MEMORY_INTER) %{
5137 base($reg);
5138 index(0xffffffff);
5139 scale(0x0);
5140 disp(0x0);
5141 %}
5142 %}
5143
5144 operand indIndexScaledI2L(iRegP reg, iRegI ireg, immIScale scale)
5145 %{
5146 constraint(ALLOC_IN_RC(ptr_reg));
5147 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5148 match(AddP reg (LShiftL (ConvI2L ireg) scale));
5149 op_cost(0);
5150 format %{ "$reg, $ireg sxtw($scale), 0, I2L" %}
5151 interface(MEMORY_INTER) %{
5152 base($reg);
5153 index($ireg);
5154 scale($scale);
5155 disp(0x0);
5156 %}
5157 %}
5158
5159 operand indIndexScaled(iRegP reg, iRegL lreg, immIScale scale)
5160 %{
5161 constraint(ALLOC_IN_RC(ptr_reg));
5162 predicate(size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5163 match(AddP reg (LShiftL lreg scale));
5164 op_cost(0);
5165 format %{ "$reg, $lreg lsl($scale)" %}
5166 interface(MEMORY_INTER) %{
5167 base($reg);
5168 index($lreg);
5169 scale($scale);
5170 disp(0x0);
5171 %}
5172 %}
5173
5174 operand indIndexI2L(iRegP reg, iRegI ireg)
5175 %{
5176 constraint(ALLOC_IN_RC(ptr_reg));
5177 match(AddP reg (ConvI2L ireg));
5178 op_cost(0);
5179 format %{ "$reg, $ireg, 0, I2L" %}
5180 interface(MEMORY_INTER) %{
5181 base($reg);
5182 index($ireg);
5183 scale(0x0);
5184 disp(0x0);
5185 %}
5186 %}
5187
5188 operand indIndex(iRegP reg, iRegL lreg)
5189 %{
5190 constraint(ALLOC_IN_RC(ptr_reg));
5191 match(AddP reg lreg);
5192 op_cost(0);
5193 format %{ "$reg, $lreg" %}
5194 interface(MEMORY_INTER) %{
5195 base($reg);
5196 index($lreg);
5197 scale(0x0);
5198 disp(0x0);
5199 %}
5200 %}
5201
5202 operand indOffI1(iRegP reg, immIOffset1 off)
5203 %{
5204 constraint(ALLOC_IN_RC(ptr_reg));
5205 match(AddP reg off);
5206 op_cost(0);
5207 format %{ "[$reg, $off]" %}
5208 interface(MEMORY_INTER) %{
5209 base($reg);
5210 index(0xffffffff);
5211 scale(0x0);
5212 disp($off);
5213 %}
5214 %}
5215
5216 operand indOffI2(iRegP reg, immIOffset2 off)
5217 %{
5218 constraint(ALLOC_IN_RC(ptr_reg));
5219 match(AddP reg off);
5220 op_cost(0);
5221 format %{ "[$reg, $off]" %}
5222 interface(MEMORY_INTER) %{
5223 base($reg);
5224 index(0xffffffff);
5225 scale(0x0);
5226 disp($off);
5227 %}
5228 %}
5229
5230 operand indOffI4(iRegP reg, immIOffset4 off)
5231 %{
5232 constraint(ALLOC_IN_RC(ptr_reg));
5233 match(AddP reg off);
5234 op_cost(0);
5235 format %{ "[$reg, $off]" %}
5236 interface(MEMORY_INTER) %{
5237 base($reg);
5238 index(0xffffffff);
5239 scale(0x0);
5240 disp($off);
5241 %}
5242 %}
5243
5244 operand indOffI8(iRegP reg, immIOffset8 off)
5245 %{
5246 constraint(ALLOC_IN_RC(ptr_reg));
5247 match(AddP reg off);
5248 op_cost(0);
5249 format %{ "[$reg, $off]" %}
5250 interface(MEMORY_INTER) %{
5251 base($reg);
5252 index(0xffffffff);
5253 scale(0x0);
5254 disp($off);
5255 %}
5256 %}
5257
5258 operand indOffI16(iRegP reg, immIOffset16 off)
5259 %{
5260 constraint(ALLOC_IN_RC(ptr_reg));
5261 match(AddP reg off);
5262 op_cost(0);
5263 format %{ "[$reg, $off]" %}
5264 interface(MEMORY_INTER) %{
5265 base($reg);
5266 index(0xffffffff);
5267 scale(0x0);
5268 disp($off);
5269 %}
5270 %}
5271
5272 operand indOffL1(iRegP reg, immLoffset1 off)
5273 %{
5274 constraint(ALLOC_IN_RC(ptr_reg));
5275 match(AddP reg off);
5276 op_cost(0);
5277 format %{ "[$reg, $off]" %}
5278 interface(MEMORY_INTER) %{
5279 base($reg);
5280 index(0xffffffff);
5281 scale(0x0);
5282 disp($off);
5283 %}
5284 %}
5285
5286 operand indOffL2(iRegP reg, immLoffset2 off)
5287 %{
5288 constraint(ALLOC_IN_RC(ptr_reg));
5289 match(AddP reg off);
5290 op_cost(0);
5291 format %{ "[$reg, $off]" %}
5292 interface(MEMORY_INTER) %{
5293 base($reg);
5294 index(0xffffffff);
5295 scale(0x0);
5296 disp($off);
5297 %}
5298 %}
5299
5300 operand indOffL4(iRegP reg, immLoffset4 off)
5301 %{
5302 constraint(ALLOC_IN_RC(ptr_reg));
5303 match(AddP reg off);
5304 op_cost(0);
5305 format %{ "[$reg, $off]" %}
5306 interface(MEMORY_INTER) %{
5307 base($reg);
5308 index(0xffffffff);
5309 scale(0x0);
5310 disp($off);
5311 %}
5312 %}
5313
5314 operand indOffL8(iRegP reg, immLoffset8 off)
5315 %{
5316 constraint(ALLOC_IN_RC(ptr_reg));
5317 match(AddP reg off);
5318 op_cost(0);
5319 format %{ "[$reg, $off]" %}
5320 interface(MEMORY_INTER) %{
5321 base($reg);
5322 index(0xffffffff);
5323 scale(0x0);
5324 disp($off);
5325 %}
5326 %}
5327
5328 operand indOffL16(iRegP reg, immLoffset16 off)
5329 %{
5330 constraint(ALLOC_IN_RC(ptr_reg));
5331 match(AddP reg off);
5332 op_cost(0);
5333 format %{ "[$reg, $off]" %}
5334 interface(MEMORY_INTER) %{
5335 base($reg);
5336 index(0xffffffff);
5337 scale(0x0);
5338 disp($off);
5339 %}
5340 %}
5341
5342 operand indirectX2P(iRegL reg)
5343 %{
5344 constraint(ALLOC_IN_RC(ptr_reg));
5345 match(CastX2P reg);
5346 op_cost(0);
5347 format %{ "[$reg]\t# long -> ptr" %}
5348 interface(MEMORY_INTER) %{
5349 base($reg);
5350 index(0xffffffff);
5351 scale(0x0);
5352 disp(0x0);
5353 %}
5354 %}
5355
5356 operand indOffX2P(iRegL reg, immLOffset off)
5357 %{
5358 constraint(ALLOC_IN_RC(ptr_reg));
5359 match(AddP (CastX2P reg) off);
5360 op_cost(0);
5361 format %{ "[$reg, $off]\t# long -> ptr" %}
5362 interface(MEMORY_INTER) %{
5363 base($reg);
5364 index(0xffffffff);
5365 scale(0x0);
5366 disp($off);
5367 %}
5368 %}
5369
5370 operand indirectN(iRegN reg)
5371 %{
5372 predicate(CompressedOops::shift() == 0);
5373 constraint(ALLOC_IN_RC(ptr_reg));
5374 match(DecodeN reg);
5375 op_cost(0);
5376 format %{ "[$reg]\t# narrow" %}
5377 interface(MEMORY_INTER) %{
5378 base($reg);
5379 index(0xffffffff);
5380 scale(0x0);
5381 disp(0x0);
5382 %}
5383 %}
5384
5385 operand indIndexScaledI2LN(iRegN reg, iRegI ireg, immIScale scale)
5386 %{
5387 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5388 constraint(ALLOC_IN_RC(ptr_reg));
5389 match(AddP (DecodeN reg) (LShiftL (ConvI2L ireg) scale));
5390 op_cost(0);
5391 format %{ "$reg, $ireg sxtw($scale), 0, I2L\t# narrow" %}
5392 interface(MEMORY_INTER) %{
5393 base($reg);
5394 index($ireg);
5395 scale($scale);
5396 disp(0x0);
5397 %}
5398 %}
5399
5400 operand indIndexScaledN(iRegN reg, iRegL lreg, immIScale scale)
5401 %{
5402 predicate(CompressedOops::shift() == 0 && size_fits_all_mem_uses(n->as_AddP(), n->in(AddPNode::Offset)->in(2)->get_int()));
5403 constraint(ALLOC_IN_RC(ptr_reg));
5404 match(AddP (DecodeN reg) (LShiftL lreg scale));
5405 op_cost(0);
5406 format %{ "$reg, $lreg lsl($scale)\t# narrow" %}
5407 interface(MEMORY_INTER) %{
5408 base($reg);
5409 index($lreg);
5410 scale($scale);
5411 disp(0x0);
5412 %}
5413 %}
5414
5415 operand indIndexI2LN(iRegN reg, iRegI ireg)
5416 %{
5417 predicate(CompressedOops::shift() == 0);
5418 constraint(ALLOC_IN_RC(ptr_reg));
5419 match(AddP (DecodeN reg) (ConvI2L ireg));
5420 op_cost(0);
5421 format %{ "$reg, $ireg, 0, I2L\t# narrow" %}
5422 interface(MEMORY_INTER) %{
5423 base($reg);
5424 index($ireg);
5425 scale(0x0);
5426 disp(0x0);
5427 %}
5428 %}
5429
5430 operand indIndexN(iRegN reg, iRegL lreg)
5431 %{
5432 predicate(CompressedOops::shift() == 0);
5433 constraint(ALLOC_IN_RC(ptr_reg));
5434 match(AddP (DecodeN reg) lreg);
5435 op_cost(0);
5436 format %{ "$reg, $lreg\t# narrow" %}
5437 interface(MEMORY_INTER) %{
5438 base($reg);
5439 index($lreg);
5440 scale(0x0);
5441 disp(0x0);
5442 %}
5443 %}
5444
5445 operand indOffIN(iRegN reg, immIOffset off)
5446 %{
5447 predicate(CompressedOops::shift() == 0);
5448 constraint(ALLOC_IN_RC(ptr_reg));
5449 match(AddP (DecodeN reg) off);
5450 op_cost(0);
5451 format %{ "[$reg, $off]\t# narrow" %}
5452 interface(MEMORY_INTER) %{
5453 base($reg);
5454 index(0xffffffff);
5455 scale(0x0);
5456 disp($off);
5457 %}
5458 %}
5459
5460 operand indOffLN(iRegN reg, immLOffset off)
5461 %{
5462 predicate(CompressedOops::shift() == 0);
5463 constraint(ALLOC_IN_RC(ptr_reg));
5464 match(AddP (DecodeN reg) off);
5465 op_cost(0);
5466 format %{ "[$reg, $off]\t# narrow" %}
5467 interface(MEMORY_INTER) %{
5468 base($reg);
5469 index(0xffffffff);
5470 scale(0x0);
5471 disp($off);
5472 %}
5473 %}
5474
5475
5476 //----------Special Memory Operands--------------------------------------------
5477 // Stack Slot Operand - This operand is used for loading and storing temporary
5478 // values on the stack where a match requires a value to
5479 // flow through memory.
5480 operand stackSlotP(sRegP reg)
5481 %{
5482 constraint(ALLOC_IN_RC(stack_slots));
5483 op_cost(100);
5484 // No match rule because this operand is only generated in matching
5485 // match(RegP);
5486 format %{ "[$reg]" %}
5487 interface(MEMORY_INTER) %{
5488 base(0x1e); // RSP
5489 index(0x0); // No Index
5490 scale(0x0); // No Scale
5491 disp($reg); // Stack Offset
5492 %}
5493 %}
5494
5495 operand stackSlotI(sRegI reg)
5496 %{
5497 constraint(ALLOC_IN_RC(stack_slots));
5498 // No match rule because this operand is only generated in matching
5499 // match(RegI);
5500 format %{ "[$reg]" %}
5501 interface(MEMORY_INTER) %{
5502 base(0x1e); // RSP
5503 index(0x0); // No Index
5504 scale(0x0); // No Scale
5505 disp($reg); // Stack Offset
5506 %}
5507 %}
5508
5509 operand stackSlotF(sRegF reg)
5510 %{
5511 constraint(ALLOC_IN_RC(stack_slots));
5512 // No match rule because this operand is only generated in matching
5513 // match(RegF);
5514 format %{ "[$reg]" %}
5515 interface(MEMORY_INTER) %{
5516 base(0x1e); // RSP
5517 index(0x0); // No Index
5518 scale(0x0); // No Scale
5519 disp($reg); // Stack Offset
5520 %}
5521 %}
5522
5523 operand stackSlotD(sRegD reg)
5524 %{
5525 constraint(ALLOC_IN_RC(stack_slots));
5526 // No match rule because this operand is only generated in matching
5527 // match(RegD);
5528 format %{ "[$reg]" %}
5529 interface(MEMORY_INTER) %{
5530 base(0x1e); // RSP
5531 index(0x0); // No Index
5532 scale(0x0); // No Scale
5533 disp($reg); // Stack Offset
5534 %}
5535 %}
5536
5537 operand stackSlotL(sRegL reg)
5538 %{
5539 constraint(ALLOC_IN_RC(stack_slots));
5540 // No match rule because this operand is only generated in matching
5541 // match(RegL);
5542 format %{ "[$reg]" %}
5543 interface(MEMORY_INTER) %{
5544 base(0x1e); // RSP
5545 index(0x0); // No Index
5546 scale(0x0); // No Scale
5547 disp($reg); // Stack Offset
5548 %}
5549 %}
5550
5551 // Operands for expressing Control Flow
5552 // NOTE: Label is a predefined operand which should not be redefined in
5553 // the AD file. It is generically handled within the ADLC.
5554
5555 //----------Conditional Branch Operands----------------------------------------
5556 // Comparison Op - This is the operation of the comparison, and is limited to
5557 // the following set of codes:
5558 // L (<), LE (<=), G (>), GE (>=), E (==), NE (!=)
5559 //
5560 // Other attributes of the comparison, such as unsignedness, are specified
5561 // by the comparison instruction that sets a condition code flags register.
5562 // That result is represented by a flags operand whose subtype is appropriate
5563 // to the unsignedness (etc.) of the comparison.
5564 //
5565 // Later, the instruction which matches both the Comparison Op (a Bool) and
5566 // the flags (produced by the Cmp) specifies the coding of the comparison op
5567 // by matching a specific subtype of Bool operand below, such as cmpOpU.
5568
5569 // used for signed integral comparisons and fp comparisons
5570
5571 operand cmpOp()
5572 %{
5573 match(Bool);
5574
5575 format %{ "" %}
5576 interface(COND_INTER) %{
5577 equal(0x0, "eq");
5578 not_equal(0x1, "ne");
5579 less(0xb, "lt");
5580 greater_equal(0xa, "ge");
5581 less_equal(0xd, "le");
5582 greater(0xc, "gt");
5583 overflow(0x6, "vs");
5584 no_overflow(0x7, "vc");
5585 %}
5586 %}
5587
5588 // used for unsigned integral comparisons
5589
5590 operand cmpOpU()
5591 %{
5592 match(Bool);
5593
5594 format %{ "" %}
5595 interface(COND_INTER) %{
5596 equal(0x0, "eq");
5597 not_equal(0x1, "ne");
5598 less(0x3, "lo");
5599 greater_equal(0x2, "hs");
5600 less_equal(0x9, "ls");
5601 greater(0x8, "hi");
5602 overflow(0x6, "vs");
5603 no_overflow(0x7, "vc");
5604 %}
5605 %}
5606
5607 // used for certain integral comparisons which can be
5608 // converted to cbxx or tbxx instructions
5609
5610 operand cmpOpEqNe()
5611 %{
5612 match(Bool);
5613 op_cost(0);
5614 predicate(n->as_Bool()->_test._test == BoolTest::ne
5615 || n->as_Bool()->_test._test == BoolTest::eq);
5616
5617 format %{ "" %}
5618 interface(COND_INTER) %{
5619 equal(0x0, "eq");
5620 not_equal(0x1, "ne");
5621 less(0xb, "lt");
5622 greater_equal(0xa, "ge");
5623 less_equal(0xd, "le");
5624 greater(0xc, "gt");
5625 overflow(0x6, "vs");
5626 no_overflow(0x7, "vc");
5627 %}
5628 %}
5629
5630 // used for certain integral comparisons which can be
5631 // converted to cbxx or tbxx instructions
5632
5633 operand cmpOpLtGe()
5634 %{
5635 match(Bool);
5636 op_cost(0);
5637
5638 predicate(n->as_Bool()->_test._test == BoolTest::lt
5639 || n->as_Bool()->_test._test == BoolTest::ge);
5640
5641 format %{ "" %}
5642 interface(COND_INTER) %{
5643 equal(0x0, "eq");
5644 not_equal(0x1, "ne");
5645 less(0xb, "lt");
5646 greater_equal(0xa, "ge");
5647 less_equal(0xd, "le");
5648 greater(0xc, "gt");
5649 overflow(0x6, "vs");
5650 no_overflow(0x7, "vc");
5651 %}
5652 %}
5653
5654 // used for certain unsigned integral comparisons which can be
5655 // converted to cbxx or tbxx instructions
5656
5657 operand cmpOpUEqNeLeGt()
5658 %{
5659 match(Bool);
5660 op_cost(0);
5661
5662 predicate(n->as_Bool()->_test._test == BoolTest::eq ||
5663 n->as_Bool()->_test._test == BoolTest::ne ||
5664 n->as_Bool()->_test._test == BoolTest::le ||
5665 n->as_Bool()->_test._test == BoolTest::gt);
5666
5667 format %{ "" %}
5668 interface(COND_INTER) %{
5669 equal(0x0, "eq");
5670 not_equal(0x1, "ne");
5671 less(0x3, "lo");
5672 greater_equal(0x2, "hs");
5673 less_equal(0x9, "ls");
5674 greater(0x8, "hi");
5675 overflow(0x6, "vs");
5676 no_overflow(0x7, "vc");
5677 %}
5678 %}
5679
5680 // Special operand allowing long args to int ops to be truncated for free
5681
5682 operand iRegL2I(iRegL reg) %{
5683
5684 op_cost(0);
5685
5686 match(ConvL2I reg);
5687
5688 format %{ "l2i($reg)" %}
5689
5690 interface(REG_INTER)
5691 %}
5692
5693 operand iRegL2P(iRegL reg) %{
5694
5695 op_cost(0);
5696
5697 match(CastX2P reg);
5698
5699 format %{ "l2p($reg)" %}
5700
5701 interface(REG_INTER)
5702 %}
5703
5704 opclass vmem2(indirect, indIndex, indOffI2, indOffL2);
5705 opclass vmem4(indirect, indIndex, indOffI4, indOffL4);
5706 opclass vmem8(indirect, indIndex, indOffI8, indOffL8);
5707 opclass vmem16(indirect, indIndex, indOffI16, indOffL16);
5708
5709 //----------OPERAND CLASSES----------------------------------------------------
5710 // Operand Classes are groups of operands that are used as to simplify
5711 // instruction definitions by not requiring the AD writer to specify
5712 // separate instructions for every form of operand when the
5713 // instruction accepts multiple operand types with the same basic
5714 // encoding and format. The classic case of this is memory operands.
5715
5716 // memory is used to define read/write location for load/store
5717 // instruction defs. we can turn a memory op into an Address
5718
5719 opclass memory1(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI1, indOffL1,
5720 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5721
5722 opclass memory2(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI2, indOffL2,
5723 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indirectX2P, indOffX2P);
5724
5725 opclass memory4(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI4, indOffL4,
5726 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5727
5728 opclass memory8(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex, indOffI8, indOffL8,
5729 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5730
5731 // All of the memory operands. For the pipeline description.
5732 opclass memory(indirect, indIndexScaled, indIndexScaledI2L, indIndexI2L, indIndex,
5733 indOffI1, indOffL1, indOffI2, indOffL2, indOffI4, indOffL4, indOffI8, indOffL8,
5734 indirectN, indIndexScaledN, indIndexScaledI2LN, indIndexI2LN, indIndexN, indOffIN, indOffLN, indirectX2P, indOffX2P);
5735
5736
5737 // iRegIorL2I is used for src inputs in rules for 32 bit int (I)
5738 // operations. it allows the src to be either an iRegI or a (ConvL2I
5739 // iRegL). in the latter case the l2i normally planted for a ConvL2I
5740 // can be elided because the 32-bit instruction will just employ the
5741 // lower 32 bits anyway.
5742 //
5743 // n.b. this does not elide all L2I conversions. if the truncated
5744 // value is consumed by more than one operation then the ConvL2I
5745 // cannot be bundled into the consuming nodes so an l2i gets planted
5746 // (actually a movw $dst $src) and the downstream instructions consume
5747 // the result of the l2i as an iRegI input. That's a shame since the
5748 // movw is actually redundant but its not too costly.
5749
5750 opclass iRegIorL2I(iRegI, iRegL2I);
5751 opclass iRegPorL2P(iRegP, iRegL2P);
5752
5753 //----------PIPELINE-----------------------------------------------------------
5754 // Rules which define the behavior of the target architectures pipeline.
5755
5756 // For specific pipelines, eg A53, define the stages of that pipeline
5757 //pipe_desc(ISS, EX1, EX2, WR);
5758 #define ISS S0
5759 #define EX1 S1
5760 #define EX2 S2
5761 #define WR S3
5762
5763 // Integer ALU reg operation
5764 pipeline %{
5765
5766 attributes %{
5767 // ARM instructions are of fixed length
5768 fixed_size_instructions; // Fixed size instructions TODO does
5769 max_instructions_per_bundle = 4; // A53 = 2, A57 = 4
5770 // ARM instructions come in 32-bit word units
5771 instruction_unit_size = 4; // An instruction is 4 bytes long
5772 instruction_fetch_unit_size = 64; // The processor fetches one line
5773 instruction_fetch_units = 1; // of 64 bytes
5774
5775 // List of nop instructions
5776 nops( MachNop );
5777 %}
5778
5779 // We don't use an actual pipeline model so don't care about resources
5780 // or description. we do use pipeline classes to introduce fixed
5781 // latencies
5782
5783 //----------RESOURCES----------------------------------------------------------
5784 // Resources are the functional units available to the machine
5785
5786 resources( INS0, INS1, INS01 = INS0 | INS1,
5787 ALU0, ALU1, ALU = ALU0 | ALU1,
5788 MAC,
5789 DIV,
5790 BRANCH,
5791 LDST,
5792 NEON_FP);
5793
5794 //----------PIPELINE DESCRIPTION-----------------------------------------------
5795 // Pipeline Description specifies the stages in the machine's pipeline
5796
5797 // Define the pipeline as a generic 6 stage pipeline
5798 pipe_desc(S0, S1, S2, S3, S4, S5);
5799
5800 //----------PIPELINE CLASSES---------------------------------------------------
5801 // Pipeline Classes describe the stages in which input and output are
5802 // referenced by the hardware pipeline.
5803
5804 pipe_class fp_dop_reg_reg_s(vRegF dst, vRegF src1, vRegF src2)
5805 %{
5806 single_instruction;
5807 src1 : S1(read);
5808 src2 : S2(read);
5809 dst : S5(write);
5810 INS01 : ISS;
5811 NEON_FP : S5;
5812 %}
5813
5814 pipe_class fp_dop_reg_reg_d(vRegD dst, vRegD src1, vRegD src2)
5815 %{
5816 single_instruction;
5817 src1 : S1(read);
5818 src2 : S2(read);
5819 dst : S5(write);
5820 INS01 : ISS;
5821 NEON_FP : S5;
5822 %}
5823
5824 pipe_class fp_uop_s(vRegF dst, vRegF src)
5825 %{
5826 single_instruction;
5827 src : S1(read);
5828 dst : S5(write);
5829 INS01 : ISS;
5830 NEON_FP : S5;
5831 %}
5832
5833 pipe_class fp_uop_d(vRegD dst, vRegD src)
5834 %{
5835 single_instruction;
5836 src : S1(read);
5837 dst : S5(write);
5838 INS01 : ISS;
5839 NEON_FP : S5;
5840 %}
5841
5842 pipe_class fp_d2f(vRegF dst, vRegD src)
5843 %{
5844 single_instruction;
5845 src : S1(read);
5846 dst : S5(write);
5847 INS01 : ISS;
5848 NEON_FP : S5;
5849 %}
5850
5851 pipe_class fp_f2d(vRegD dst, vRegF src)
5852 %{
5853 single_instruction;
5854 src : S1(read);
5855 dst : S5(write);
5856 INS01 : ISS;
5857 NEON_FP : S5;
5858 %}
5859
5860 pipe_class fp_f2i(iRegINoSp dst, vRegF src)
5861 %{
5862 single_instruction;
5863 src : S1(read);
5864 dst : S5(write);
5865 INS01 : ISS;
5866 NEON_FP : S5;
5867 %}
5868
5869 pipe_class fp_f2l(iRegLNoSp dst, vRegF src)
5870 %{
5871 single_instruction;
5872 src : S1(read);
5873 dst : S5(write);
5874 INS01 : ISS;
5875 NEON_FP : S5;
5876 %}
5877
5878 pipe_class fp_i2f(vRegF dst, iRegIorL2I src)
5879 %{
5880 single_instruction;
5881 src : S1(read);
5882 dst : S5(write);
5883 INS01 : ISS;
5884 NEON_FP : S5;
5885 %}
5886
5887 pipe_class fp_l2f(vRegF dst, iRegL src)
5888 %{
5889 single_instruction;
5890 src : S1(read);
5891 dst : S5(write);
5892 INS01 : ISS;
5893 NEON_FP : S5;
5894 %}
5895
5896 pipe_class fp_d2i(iRegINoSp dst, vRegD src)
5897 %{
5898 single_instruction;
5899 src : S1(read);
5900 dst : S5(write);
5901 INS01 : ISS;
5902 NEON_FP : S5;
5903 %}
5904
5905 pipe_class fp_d2l(iRegLNoSp dst, vRegD src)
5906 %{
5907 single_instruction;
5908 src : S1(read);
5909 dst : S5(write);
5910 INS01 : ISS;
5911 NEON_FP : S5;
5912 %}
5913
5914 pipe_class fp_i2d(vRegD dst, iRegIorL2I src)
5915 %{
5916 single_instruction;
5917 src : S1(read);
5918 dst : S5(write);
5919 INS01 : ISS;
5920 NEON_FP : S5;
5921 %}
5922
5923 pipe_class fp_l2d(vRegD dst, iRegIorL2I src)
5924 %{
5925 single_instruction;
5926 src : S1(read);
5927 dst : S5(write);
5928 INS01 : ISS;
5929 NEON_FP : S5;
5930 %}
5931
5932 pipe_class fp_div_s(vRegF dst, vRegF src1, vRegF src2)
5933 %{
5934 single_instruction;
5935 src1 : S1(read);
5936 src2 : S2(read);
5937 dst : S5(write);
5938 INS0 : ISS;
5939 NEON_FP : S5;
5940 %}
5941
5942 pipe_class fp_div_d(vRegD dst, vRegD src1, vRegD src2)
5943 %{
5944 single_instruction;
5945 src1 : S1(read);
5946 src2 : S2(read);
5947 dst : S5(write);
5948 INS0 : ISS;
5949 NEON_FP : S5;
5950 %}
5951
5952 pipe_class fp_cond_reg_reg_s(vRegF dst, vRegF src1, vRegF src2, rFlagsReg cr)
5953 %{
5954 single_instruction;
5955 cr : S1(read);
5956 src1 : S1(read);
5957 src2 : S1(read);
5958 dst : S3(write);
5959 INS01 : ISS;
5960 NEON_FP : S3;
5961 %}
5962
5963 pipe_class fp_cond_reg_reg_d(vRegD dst, vRegD src1, vRegD src2, rFlagsReg cr)
5964 %{
5965 single_instruction;
5966 cr : S1(read);
5967 src1 : S1(read);
5968 src2 : S1(read);
5969 dst : S3(write);
5970 INS01 : ISS;
5971 NEON_FP : S3;
5972 %}
5973
5974 pipe_class fp_imm_s(vRegF dst)
5975 %{
5976 single_instruction;
5977 dst : S3(write);
5978 INS01 : ISS;
5979 NEON_FP : S3;
5980 %}
5981
5982 pipe_class fp_imm_d(vRegD dst)
5983 %{
5984 single_instruction;
5985 dst : S3(write);
5986 INS01 : ISS;
5987 NEON_FP : S3;
5988 %}
5989
5990 pipe_class fp_load_constant_s(vRegF dst)
5991 %{
5992 single_instruction;
5993 dst : S4(write);
5994 INS01 : ISS;
5995 NEON_FP : S4;
5996 %}
5997
5998 pipe_class fp_load_constant_d(vRegD dst)
5999 %{
6000 single_instruction;
6001 dst : S4(write);
6002 INS01 : ISS;
6003 NEON_FP : S4;
6004 %}
6005
6006 //------- Integer ALU operations --------------------------
6007
6008 // Integer ALU reg-reg operation
6009 // Operands needed in EX1, result generated in EX2
6010 // Eg. ADD x0, x1, x2
6011 pipe_class ialu_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6012 %{
6013 single_instruction;
6014 dst : EX2(write);
6015 src1 : EX1(read);
6016 src2 : EX1(read);
6017 INS01 : ISS; // Dual issue as instruction 0 or 1
6018 ALU : EX2;
6019 %}
6020
6021 // Integer ALU reg-reg operation with constant shift
6022 // Shifted register must be available in LATE_ISS instead of EX1
6023 // Eg. ADD x0, x1, x2, LSL #2
6024 pipe_class ialu_reg_reg_shift(iRegI dst, iRegI src1, iRegI src2, immI shift)
6025 %{
6026 single_instruction;
6027 dst : EX2(write);
6028 src1 : EX1(read);
6029 src2 : ISS(read);
6030 INS01 : ISS;
6031 ALU : EX2;
6032 %}
6033
6034 // Integer ALU reg operation with constant shift
6035 // Eg. LSL x0, x1, #shift
6036 pipe_class ialu_reg_shift(iRegI dst, iRegI src1)
6037 %{
6038 single_instruction;
6039 dst : EX2(write);
6040 src1 : ISS(read);
6041 INS01 : ISS;
6042 ALU : EX2;
6043 %}
6044
6045 // Integer ALU reg-reg operation with variable shift
6046 // Both operands must be available in LATE_ISS instead of EX1
6047 // Result is available in EX1 instead of EX2
6048 // Eg. LSLV x0, x1, x2
6049 pipe_class ialu_reg_reg_vshift(iRegI dst, iRegI src1, iRegI src2)
6050 %{
6051 single_instruction;
6052 dst : EX1(write);
6053 src1 : ISS(read);
6054 src2 : ISS(read);
6055 INS01 : ISS;
6056 ALU : EX1;
6057 %}
6058
6059 // Integer ALU reg-reg operation with extract
6060 // As for _vshift above, but result generated in EX2
6061 // Eg. EXTR x0, x1, x2, #N
6062 pipe_class ialu_reg_reg_extr(iRegI dst, iRegI src1, iRegI src2)
6063 %{
6064 single_instruction;
6065 dst : EX2(write);
6066 src1 : ISS(read);
6067 src2 : ISS(read);
6068 INS1 : ISS; // Can only dual issue as Instruction 1
6069 ALU : EX1;
6070 %}
6071
6072 // Integer ALU reg operation
6073 // Eg. NEG x0, x1
6074 pipe_class ialu_reg(iRegI dst, iRegI src)
6075 %{
6076 single_instruction;
6077 dst : EX2(write);
6078 src : EX1(read);
6079 INS01 : ISS;
6080 ALU : EX2;
6081 %}
6082
6083 // Integer ALU reg mmediate operation
6084 // Eg. ADD x0, x1, #N
6085 pipe_class ialu_reg_imm(iRegI dst, iRegI src1)
6086 %{
6087 single_instruction;
6088 dst : EX2(write);
6089 src1 : EX1(read);
6090 INS01 : ISS;
6091 ALU : EX2;
6092 %}
6093
6094 // Integer ALU immediate operation (no source operands)
6095 // Eg. MOV x0, #N
6096 pipe_class ialu_imm(iRegI dst)
6097 %{
6098 single_instruction;
6099 dst : EX1(write);
6100 INS01 : ISS;
6101 ALU : EX1;
6102 %}
6103
6104 //------- Compare operation -------------------------------
6105
6106 // Compare reg-reg
6107 // Eg. CMP x0, x1
6108 pipe_class icmp_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
6109 %{
6110 single_instruction;
6111 // fixed_latency(16);
6112 cr : EX2(write);
6113 op1 : EX1(read);
6114 op2 : EX1(read);
6115 INS01 : ISS;
6116 ALU : EX2;
6117 %}
6118
6119 // Compare reg-reg
6120 // Eg. CMP x0, #N
6121 pipe_class icmp_reg_imm(rFlagsReg cr, iRegI op1)
6122 %{
6123 single_instruction;
6124 // fixed_latency(16);
6125 cr : EX2(write);
6126 op1 : EX1(read);
6127 INS01 : ISS;
6128 ALU : EX2;
6129 %}
6130
6131 //------- Conditional instructions ------------------------
6132
6133 // Conditional no operands
6134 // Eg. CSINC x0, zr, zr, <cond>
6135 pipe_class icond_none(iRegI dst, rFlagsReg cr)
6136 %{
6137 single_instruction;
6138 cr : EX1(read);
6139 dst : EX2(write);
6140 INS01 : ISS;
6141 ALU : EX2;
6142 %}
6143
6144 // Conditional 2 operand
6145 // EG. CSEL X0, X1, X2, <cond>
6146 pipe_class icond_reg_reg(iRegI dst, iRegI src1, iRegI src2, rFlagsReg cr)
6147 %{
6148 single_instruction;
6149 cr : EX1(read);
6150 src1 : EX1(read);
6151 src2 : EX1(read);
6152 dst : EX2(write);
6153 INS01 : ISS;
6154 ALU : EX2;
6155 %}
6156
6157 // Conditional 2 operand
6158 // EG. CSEL X0, X1, X2, <cond>
6159 pipe_class icond_reg(iRegI dst, iRegI src, rFlagsReg cr)
6160 %{
6161 single_instruction;
6162 cr : EX1(read);
6163 src : EX1(read);
6164 dst : EX2(write);
6165 INS01 : ISS;
6166 ALU : EX2;
6167 %}
6168
6169 //------- Multiply pipeline operations --------------------
6170
6171 // Multiply reg-reg
6172 // Eg. MUL w0, w1, w2
6173 pipe_class imul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6174 %{
6175 single_instruction;
6176 dst : WR(write);
6177 src1 : ISS(read);
6178 src2 : ISS(read);
6179 INS01 : ISS;
6180 MAC : WR;
6181 %}
6182
6183 // Multiply accumulate
6184 // Eg. MADD w0, w1, w2, w3
6185 pipe_class imac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6186 %{
6187 single_instruction;
6188 dst : WR(write);
6189 src1 : ISS(read);
6190 src2 : ISS(read);
6191 src3 : ISS(read);
6192 INS01 : ISS;
6193 MAC : WR;
6194 %}
6195
6196 // Eg. MUL w0, w1, w2
6197 pipe_class lmul_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6198 %{
6199 single_instruction;
6200 fixed_latency(3); // Maximum latency for 64 bit mul
6201 dst : WR(write);
6202 src1 : ISS(read);
6203 src2 : ISS(read);
6204 INS01 : ISS;
6205 MAC : WR;
6206 %}
6207
6208 // Multiply accumulate
6209 // Eg. MADD w0, w1, w2, w3
6210 pipe_class lmac_reg_reg(iRegI dst, iRegI src1, iRegI src2, iRegI src3)
6211 %{
6212 single_instruction;
6213 fixed_latency(3); // Maximum latency for 64 bit mul
6214 dst : WR(write);
6215 src1 : ISS(read);
6216 src2 : ISS(read);
6217 src3 : ISS(read);
6218 INS01 : ISS;
6219 MAC : WR;
6220 %}
6221
6222 //------- Divide pipeline operations --------------------
6223
6224 // Eg. SDIV w0, w1, w2
6225 pipe_class idiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6226 %{
6227 single_instruction;
6228 fixed_latency(8); // Maximum latency for 32 bit divide
6229 dst : WR(write);
6230 src1 : ISS(read);
6231 src2 : ISS(read);
6232 INS0 : ISS; // Can only dual issue as instruction 0
6233 DIV : WR;
6234 %}
6235
6236 // Eg. SDIV x0, x1, x2
6237 pipe_class ldiv_reg_reg(iRegI dst, iRegI src1, iRegI src2)
6238 %{
6239 single_instruction;
6240 fixed_latency(16); // Maximum latency for 64 bit divide
6241 dst : WR(write);
6242 src1 : ISS(read);
6243 src2 : ISS(read);
6244 INS0 : ISS; // Can only dual issue as instruction 0
6245 DIV : WR;
6246 %}
6247
6248 //------- Load pipeline operations ------------------------
6249
6250 // Load - prefetch
6251 // Eg. PFRM <mem>
6252 pipe_class iload_prefetch(memory mem)
6253 %{
6254 single_instruction;
6255 mem : ISS(read);
6256 INS01 : ISS;
6257 LDST : WR;
6258 %}
6259
6260 // Load - reg, mem
6261 // Eg. LDR x0, <mem>
6262 pipe_class iload_reg_mem(iRegI dst, memory mem)
6263 %{
6264 single_instruction;
6265 dst : WR(write);
6266 mem : ISS(read);
6267 INS01 : ISS;
6268 LDST : WR;
6269 %}
6270
6271 // Load - reg, reg
6272 // Eg. LDR x0, [sp, x1]
6273 pipe_class iload_reg_reg(iRegI dst, iRegI src)
6274 %{
6275 single_instruction;
6276 dst : WR(write);
6277 src : ISS(read);
6278 INS01 : ISS;
6279 LDST : WR;
6280 %}
6281
6282 //------- Store pipeline operations -----------------------
6283
6284 // Store - zr, mem
6285 // Eg. STR zr, <mem>
6286 pipe_class istore_mem(memory mem)
6287 %{
6288 single_instruction;
6289 mem : ISS(read);
6290 INS01 : ISS;
6291 LDST : WR;
6292 %}
6293
6294 // Store - reg, mem
6295 // Eg. STR x0, <mem>
6296 pipe_class istore_reg_mem(iRegI src, memory mem)
6297 %{
6298 single_instruction;
6299 mem : ISS(read);
6300 src : EX2(read);
6301 INS01 : ISS;
6302 LDST : WR;
6303 %}
6304
6305 // Store - reg, reg
6306 // Eg. STR x0, [sp, x1]
6307 pipe_class istore_reg_reg(iRegI dst, iRegI src)
6308 %{
6309 single_instruction;
6310 dst : ISS(read);
6311 src : EX2(read);
6312 INS01 : ISS;
6313 LDST : WR;
6314 %}
6315
6316 //------- Store pipeline operations -----------------------
6317
6318 // Branch
6319 pipe_class pipe_branch()
6320 %{
6321 single_instruction;
6322 INS01 : ISS;
6323 BRANCH : EX1;
6324 %}
6325
6326 // Conditional branch
6327 pipe_class pipe_branch_cond(rFlagsReg cr)
6328 %{
6329 single_instruction;
6330 cr : EX1(read);
6331 INS01 : ISS;
6332 BRANCH : EX1;
6333 %}
6334
6335 // Compare & Branch
6336 // EG. CBZ/CBNZ
6337 pipe_class pipe_cmp_branch(iRegI op1)
6338 %{
6339 single_instruction;
6340 op1 : EX1(read);
6341 INS01 : ISS;
6342 BRANCH : EX1;
6343 %}
6344
6345 //------- Synchronisation operations ----------------------
6346
6347 // Any operation requiring serialization.
6348 // EG. DMB/Atomic Ops/Load Acquire/Str Release
6349 pipe_class pipe_serial()
6350 %{
6351 single_instruction;
6352 force_serialization;
6353 fixed_latency(16);
6354 INS01 : ISS(2); // Cannot dual issue with any other instruction
6355 LDST : WR;
6356 %}
6357
6358 // Generic big/slow expanded idiom - also serialized
6359 pipe_class pipe_slow()
6360 %{
6361 instruction_count(10);
6362 multiple_bundles;
6363 force_serialization;
6364 fixed_latency(16);
6365 INS01 : ISS(2); // Cannot dual issue with any other instruction
6366 LDST : WR;
6367 %}
6368
6369 // Empty pipeline class
6370 pipe_class pipe_class_empty()
6371 %{
6372 single_instruction;
6373 fixed_latency(0);
6374 %}
6375
6376 // Default pipeline class.
6377 pipe_class pipe_class_default()
6378 %{
6379 single_instruction;
6380 fixed_latency(2);
6381 %}
6382
6383 // Pipeline class for compares.
6384 pipe_class pipe_class_compare()
6385 %{
6386 single_instruction;
6387 fixed_latency(16);
6388 %}
6389
6390 // Pipeline class for memory operations.
6391 pipe_class pipe_class_memory()
6392 %{
6393 single_instruction;
6394 fixed_latency(16);
6395 %}
6396
6397 // Pipeline class for call.
6398 pipe_class pipe_class_call()
6399 %{
6400 single_instruction;
6401 fixed_latency(100);
6402 %}
6403
6404 // Define the class for the Nop node.
6405 define %{
6406 MachNop = pipe_class_empty;
6407 %}
6408
6409 %}
6410 //----------INSTRUCTIONS-------------------------------------------------------
6411 //
6412 // match -- States which machine-independent subtree may be replaced
6413 // by this instruction.
6414 // ins_cost -- The estimated cost of this instruction is used by instruction
6415 // selection to identify a minimum cost tree of machine
6416 // instructions that matches a tree of machine-independent
6417 // instructions.
6418 // format -- A string providing the disassembly for this instruction.
6419 // The value of an instruction's operand may be inserted
6420 // by referring to it with a '$' prefix.
6421 // opcode -- Three instruction opcodes may be provided. These are referred
6422 // to within an encode class as $primary, $secondary, and $tertiary
6423 // rrspectively. The primary opcode is commonly used to
6424 // indicate the type of machine instruction, while secondary
6425 // and tertiary are often used for prefix options or addressing
6426 // modes.
6427 // ins_encode -- A list of encode classes with parameters. The encode class
6428 // name must have been defined in an 'enc_class' specification
6429 // in the encode section of the architecture description.
6430
6431 // ============================================================================
6432 // Memory (Load/Store) Instructions
6433
6434 // Load Instructions
6435
6436 // Load Byte (8 bit signed)
6437 instruct loadB(iRegINoSp dst, memory1 mem)
6438 %{
6439 match(Set dst (LoadB mem));
6440 predicate(!needs_acquiring_load(n));
6441
6442 ins_cost(4 * INSN_COST);
6443 format %{ "ldrsbw $dst, $mem\t# byte" %}
6444
6445 ins_encode(aarch64_enc_ldrsbw(dst, mem));
6446
6447 ins_pipe(iload_reg_mem);
6448 %}
6449
6450 // Load Byte (8 bit signed) into long
6451 instruct loadB2L(iRegLNoSp dst, memory1 mem)
6452 %{
6453 match(Set dst (ConvI2L (LoadB mem)));
6454 predicate(!needs_acquiring_load(n->in(1)));
6455
6456 ins_cost(4 * INSN_COST);
6457 format %{ "ldrsb $dst, $mem\t# byte" %}
6458
6459 ins_encode(aarch64_enc_ldrsb(dst, mem));
6460
6461 ins_pipe(iload_reg_mem);
6462 %}
6463
6464 // Load Byte (8 bit unsigned)
6465 instruct loadUB(iRegINoSp dst, memory1 mem)
6466 %{
6467 match(Set dst (LoadUB mem));
6468 predicate(!needs_acquiring_load(n));
6469
6470 ins_cost(4 * INSN_COST);
6471 format %{ "ldrbw $dst, $mem\t# byte" %}
6472
6473 ins_encode(aarch64_enc_ldrb(dst, mem));
6474
6475 ins_pipe(iload_reg_mem);
6476 %}
6477
6478 // Load Byte (8 bit unsigned) into long
6479 instruct loadUB2L(iRegLNoSp dst, memory1 mem)
6480 %{
6481 match(Set dst (ConvI2L (LoadUB mem)));
6482 predicate(!needs_acquiring_load(n->in(1)));
6483
6484 ins_cost(4 * INSN_COST);
6485 format %{ "ldrb $dst, $mem\t# byte" %}
6486
6487 ins_encode(aarch64_enc_ldrb(dst, mem));
6488
6489 ins_pipe(iload_reg_mem);
6490 %}
6491
6492 // Load Short (16 bit signed)
6493 instruct loadS(iRegINoSp dst, memory2 mem)
6494 %{
6495 match(Set dst (LoadS mem));
6496 predicate(!needs_acquiring_load(n));
6497
6498 ins_cost(4 * INSN_COST);
6499 format %{ "ldrshw $dst, $mem\t# short" %}
6500
6501 ins_encode(aarch64_enc_ldrshw(dst, mem));
6502
6503 ins_pipe(iload_reg_mem);
6504 %}
6505
6506 // Load Short (16 bit signed) into long
6507 instruct loadS2L(iRegLNoSp dst, memory2 mem)
6508 %{
6509 match(Set dst (ConvI2L (LoadS mem)));
6510 predicate(!needs_acquiring_load(n->in(1)));
6511
6512 ins_cost(4 * INSN_COST);
6513 format %{ "ldrsh $dst, $mem\t# short" %}
6514
6515 ins_encode(aarch64_enc_ldrsh(dst, mem));
6516
6517 ins_pipe(iload_reg_mem);
6518 %}
6519
6520 // Load Char (16 bit unsigned)
6521 instruct loadUS(iRegINoSp dst, memory2 mem)
6522 %{
6523 match(Set dst (LoadUS mem));
6524 predicate(!needs_acquiring_load(n));
6525
6526 ins_cost(4 * INSN_COST);
6527 format %{ "ldrh $dst, $mem\t# short" %}
6528
6529 ins_encode(aarch64_enc_ldrh(dst, mem));
6530
6531 ins_pipe(iload_reg_mem);
6532 %}
6533
6534 // Load Short/Char (16 bit unsigned) into long
6535 instruct loadUS2L(iRegLNoSp dst, memory2 mem)
6536 %{
6537 match(Set dst (ConvI2L (LoadUS mem)));
6538 predicate(!needs_acquiring_load(n->in(1)));
6539
6540 ins_cost(4 * INSN_COST);
6541 format %{ "ldrh $dst, $mem\t# short" %}
6542
6543 ins_encode(aarch64_enc_ldrh(dst, mem));
6544
6545 ins_pipe(iload_reg_mem);
6546 %}
6547
6548 // Load Integer (32 bit signed)
6549 instruct loadI(iRegINoSp dst, memory4 mem)
6550 %{
6551 match(Set dst (LoadI mem));
6552 predicate(!needs_acquiring_load(n));
6553
6554 ins_cost(4 * INSN_COST);
6555 format %{ "ldrw $dst, $mem\t# int" %}
6556
6557 ins_encode(aarch64_enc_ldrw(dst, mem));
6558
6559 ins_pipe(iload_reg_mem);
6560 %}
6561
6562 // Load Integer (32 bit signed) into long
6563 instruct loadI2L(iRegLNoSp dst, memory4 mem)
6564 %{
6565 match(Set dst (ConvI2L (LoadI mem)));
6566 predicate(!needs_acquiring_load(n->in(1)));
6567
6568 ins_cost(4 * INSN_COST);
6569 format %{ "ldrsw $dst, $mem\t# int" %}
6570
6571 ins_encode(aarch64_enc_ldrsw(dst, mem));
6572
6573 ins_pipe(iload_reg_mem);
6574 %}
6575
6576 // Load Integer (32 bit unsigned) into long
6577 instruct loadUI2L(iRegLNoSp dst, memory4 mem, immL_32bits mask)
6578 %{
6579 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
6580 predicate(!needs_acquiring_load(n->in(1)->in(1)->as_Load()));
6581
6582 ins_cost(4 * INSN_COST);
6583 format %{ "ldrw $dst, $mem\t# int" %}
6584
6585 ins_encode(aarch64_enc_ldrw(dst, mem));
6586
6587 ins_pipe(iload_reg_mem);
6588 %}
6589
6590 // Load Long (64 bit signed)
6591 instruct loadL(iRegLNoSp dst, memory8 mem)
6592 %{
6593 match(Set dst (LoadL mem));
6594 predicate(!needs_acquiring_load(n));
6595
6596 ins_cost(4 * INSN_COST);
6597 format %{ "ldr $dst, $mem\t# int" %}
6598
6599 ins_encode(aarch64_enc_ldr(dst, mem));
6600
6601 ins_pipe(iload_reg_mem);
6602 %}
6603
6604 // Load Range
6605 instruct loadRange(iRegINoSp dst, memory4 mem)
6606 %{
6607 match(Set dst (LoadRange mem));
6608
6609 ins_cost(4 * INSN_COST);
6610 format %{ "ldrw $dst, $mem\t# range" %}
6611
6612 ins_encode(aarch64_enc_ldrw(dst, mem));
6613
6614 ins_pipe(iload_reg_mem);
6615 %}
6616
6617 // Load Pointer
6618 instruct loadP(iRegPNoSp dst, memory8 mem)
6619 %{
6620 match(Set dst (LoadP mem));
6621 predicate(!needs_acquiring_load(n) && (n->as_Load()->barrier_data() == 0));
6622
6623 ins_cost(4 * INSN_COST);
6624 format %{ "ldr $dst, $mem\t# ptr" %}
6625
6626 ins_encode(aarch64_enc_ldr(dst, mem));
6627
6628 ins_pipe(iload_reg_mem);
6629 %}
6630
6631 // Load Compressed Pointer
6632 instruct loadN(iRegNNoSp dst, memory4 mem)
6633 %{
6634 match(Set dst (LoadN mem));
6635 predicate(!needs_acquiring_load(n) && n->as_Load()->barrier_data() == 0);
6636
6637 ins_cost(4 * INSN_COST);
6638 format %{ "ldrw $dst, $mem\t# compressed ptr" %}
6639
6640 ins_encode(aarch64_enc_ldrw(dst, mem));
6641
6642 ins_pipe(iload_reg_mem);
6643 %}
6644
6645 // Load Klass Pointer
6646 instruct loadKlass(iRegPNoSp dst, memory8 mem)
6647 %{
6648 match(Set dst (LoadKlass mem));
6649 predicate(!needs_acquiring_load(n));
6650
6651 ins_cost(4 * INSN_COST);
6652 format %{ "ldr $dst, $mem\t# class" %}
6653
6654 ins_encode(aarch64_enc_ldr(dst, mem));
6655
6656 ins_pipe(iload_reg_mem);
6657 %}
6658
6659 // Load Narrow Klass Pointer
6660 instruct loadNKlass(iRegNNoSp dst, memory4 mem)
6661 %{
6662 match(Set dst (LoadNKlass mem));
6663 predicate(!needs_acquiring_load(n));
6664
6665 ins_cost(4 * INSN_COST);
6666 format %{ "ldrw $dst, $mem\t# compressed class ptr" %}
6667
6668 ins_encode(aarch64_enc_ldrw(dst, mem));
6669
6670 ins_pipe(iload_reg_mem);
6671 %}
6672
6673 // Load Float
6674 instruct loadF(vRegF dst, memory4 mem)
6675 %{
6676 match(Set dst (LoadF mem));
6677 predicate(!needs_acquiring_load(n));
6678
6679 ins_cost(4 * INSN_COST);
6680 format %{ "ldrs $dst, $mem\t# float" %}
6681
6682 ins_encode( aarch64_enc_ldrs(dst, mem) );
6683
6684 ins_pipe(pipe_class_memory);
6685 %}
6686
6687 // Load Double
6688 instruct loadD(vRegD dst, memory8 mem)
6689 %{
6690 match(Set dst (LoadD mem));
6691 predicate(!needs_acquiring_load(n));
6692
6693 ins_cost(4 * INSN_COST);
6694 format %{ "ldrd $dst, $mem\t# double" %}
6695
6696 ins_encode( aarch64_enc_ldrd(dst, mem) );
6697
6698 ins_pipe(pipe_class_memory);
6699 %}
6700
6701
6702 // Load Int Constant
6703 instruct loadConI(iRegINoSp dst, immI src)
6704 %{
6705 match(Set dst src);
6706
6707 ins_cost(INSN_COST);
6708 format %{ "mov $dst, $src\t# int" %}
6709
6710 ins_encode( aarch64_enc_movw_imm(dst, src) );
6711
6712 ins_pipe(ialu_imm);
6713 %}
6714
6715 // Load Long Constant
6716 instruct loadConL(iRegLNoSp dst, immL src)
6717 %{
6718 match(Set dst src);
6719
6720 ins_cost(INSN_COST);
6721 format %{ "mov $dst, $src\t# long" %}
6722
6723 ins_encode( aarch64_enc_mov_imm(dst, src) );
6724
6725 ins_pipe(ialu_imm);
6726 %}
6727
6728 // Load Pointer Constant
6729
6730 instruct loadConP(iRegPNoSp dst, immP con)
6731 %{
6732 match(Set dst con);
6733
6734 ins_cost(INSN_COST * 4);
6735 format %{
6736 "mov $dst, $con\t# ptr\n\t"
6737 %}
6738
6739 ins_encode(aarch64_enc_mov_p(dst, con));
6740
6741 ins_pipe(ialu_imm);
6742 %}
6743
6744 // Load Null Pointer Constant
6745
6746 instruct loadConP0(iRegPNoSp dst, immP0 con)
6747 %{
6748 match(Set dst con);
6749
6750 ins_cost(INSN_COST);
6751 format %{ "mov $dst, $con\t# nullptr ptr" %}
6752
6753 ins_encode(aarch64_enc_mov_p0(dst, con));
6754
6755 ins_pipe(ialu_imm);
6756 %}
6757
6758 // Load Pointer Constant One
6759
6760 instruct loadConP1(iRegPNoSp dst, immP_1 con)
6761 %{
6762 match(Set dst con);
6763
6764 ins_cost(INSN_COST);
6765 format %{ "mov $dst, $con\t# nullptr ptr" %}
6766
6767 ins_encode(aarch64_enc_mov_p1(dst, con));
6768
6769 ins_pipe(ialu_imm);
6770 %}
6771
6772 // Load Byte Map Base Constant
6773
6774 instruct loadByteMapBase(iRegPNoSp dst, immByteMapBase con)
6775 %{
6776 match(Set dst con);
6777
6778 ins_cost(INSN_COST);
6779 format %{ "adr $dst, $con\t# Byte Map Base" %}
6780
6781 ins_encode(aarch64_enc_mov_byte_map_base(dst, con));
6782
6783 ins_pipe(ialu_imm);
6784 %}
6785
6786 // Load Narrow Pointer Constant
6787
6788 instruct loadConN(iRegNNoSp dst, immN con)
6789 %{
6790 match(Set dst con);
6791
6792 ins_cost(INSN_COST * 4);
6793 format %{ "mov $dst, $con\t# compressed ptr" %}
6794
6795 ins_encode(aarch64_enc_mov_n(dst, con));
6796
6797 ins_pipe(ialu_imm);
6798 %}
6799
6800 // Load Narrow Null Pointer Constant
6801
6802 instruct loadConN0(iRegNNoSp dst, immN0 con)
6803 %{
6804 match(Set dst con);
6805
6806 ins_cost(INSN_COST);
6807 format %{ "mov $dst, $con\t# compressed nullptr ptr" %}
6808
6809 ins_encode(aarch64_enc_mov_n0(dst, con));
6810
6811 ins_pipe(ialu_imm);
6812 %}
6813
6814 // Load Narrow Klass Constant
6815
6816 instruct loadConNKlass(iRegNNoSp dst, immNKlass con)
6817 %{
6818 match(Set dst con);
6819
6820 ins_cost(INSN_COST);
6821 format %{ "mov $dst, $con\t# compressed klass ptr" %}
6822
6823 ins_encode(aarch64_enc_mov_nk(dst, con));
6824
6825 ins_pipe(ialu_imm);
6826 %}
6827
6828 // Load Packed Float Constant
6829
6830 instruct loadConF_packed(vRegF dst, immFPacked con) %{
6831 match(Set dst con);
6832 ins_cost(INSN_COST * 4);
6833 format %{ "fmovs $dst, $con"%}
6834 ins_encode %{
6835 __ fmovs(as_FloatRegister($dst$$reg), (double)$con$$constant);
6836 %}
6837
6838 ins_pipe(fp_imm_s);
6839 %}
6840
6841 // Load Float Constant
6842
6843 instruct loadConF(vRegF dst, immF con) %{
6844 match(Set dst con);
6845
6846 ins_cost(INSN_COST * 4);
6847
6848 format %{
6849 "ldrs $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6850 %}
6851
6852 ins_encode %{
6853 __ ldrs(as_FloatRegister($dst$$reg), $constantaddress($con));
6854 %}
6855
6856 ins_pipe(fp_load_constant_s);
6857 %}
6858
6859 // Load Packed Double Constant
6860
6861 instruct loadConD_packed(vRegD dst, immDPacked con) %{
6862 match(Set dst con);
6863 ins_cost(INSN_COST);
6864 format %{ "fmovd $dst, $con"%}
6865 ins_encode %{
6866 __ fmovd(as_FloatRegister($dst$$reg), $con$$constant);
6867 %}
6868
6869 ins_pipe(fp_imm_d);
6870 %}
6871
6872 // Load Double Constant
6873
6874 instruct loadConD(vRegD dst, immD con) %{
6875 match(Set dst con);
6876
6877 ins_cost(INSN_COST * 5);
6878 format %{
6879 "ldrd $dst, [$constantaddress]\t# load from constant table: float=$con\n\t"
6880 %}
6881
6882 ins_encode %{
6883 __ ldrd(as_FloatRegister($dst$$reg), $constantaddress($con));
6884 %}
6885
6886 ins_pipe(fp_load_constant_d);
6887 %}
6888
6889 // Store Instructions
6890
6891 // Store Byte
6892 instruct storeB(iRegIorL2I src, memory1 mem)
6893 %{
6894 match(Set mem (StoreB mem src));
6895 predicate(!needs_releasing_store(n));
6896
6897 ins_cost(INSN_COST);
6898 format %{ "strb $src, $mem\t# byte" %}
6899
6900 ins_encode(aarch64_enc_strb(src, mem));
6901
6902 ins_pipe(istore_reg_mem);
6903 %}
6904
6905
6906 instruct storeimmB0(immI0 zero, memory1 mem)
6907 %{
6908 match(Set mem (StoreB mem zero));
6909 predicate(!needs_releasing_store(n));
6910
6911 ins_cost(INSN_COST);
6912 format %{ "strb rscractch2, $mem\t# byte" %}
6913
6914 ins_encode(aarch64_enc_strb0(mem));
6915
6916 ins_pipe(istore_mem);
6917 %}
6918
6919 // Store Char/Short
6920 instruct storeC(iRegIorL2I src, memory2 mem)
6921 %{
6922 match(Set mem (StoreC mem src));
6923 predicate(!needs_releasing_store(n));
6924
6925 ins_cost(INSN_COST);
6926 format %{ "strh $src, $mem\t# short" %}
6927
6928 ins_encode(aarch64_enc_strh(src, mem));
6929
6930 ins_pipe(istore_reg_mem);
6931 %}
6932
6933 instruct storeimmC0(immI0 zero, memory2 mem)
6934 %{
6935 match(Set mem (StoreC mem zero));
6936 predicate(!needs_releasing_store(n));
6937
6938 ins_cost(INSN_COST);
6939 format %{ "strh zr, $mem\t# short" %}
6940
6941 ins_encode(aarch64_enc_strh0(mem));
6942
6943 ins_pipe(istore_mem);
6944 %}
6945
6946 // Store Integer
6947
6948 instruct storeI(iRegIorL2I src, memory4 mem)
6949 %{
6950 match(Set mem(StoreI mem src));
6951 predicate(!needs_releasing_store(n));
6952
6953 ins_cost(INSN_COST);
6954 format %{ "strw $src, $mem\t# int" %}
6955
6956 ins_encode(aarch64_enc_strw(src, mem));
6957
6958 ins_pipe(istore_reg_mem);
6959 %}
6960
6961 instruct storeimmI0(immI0 zero, memory4 mem)
6962 %{
6963 match(Set mem(StoreI mem zero));
6964 predicate(!needs_releasing_store(n));
6965
6966 ins_cost(INSN_COST);
6967 format %{ "strw zr, $mem\t# int" %}
6968
6969 ins_encode(aarch64_enc_strw0(mem));
6970
6971 ins_pipe(istore_mem);
6972 %}
6973
6974 // Store Long (64 bit signed)
6975 instruct storeL(iRegL src, memory8 mem)
6976 %{
6977 match(Set mem (StoreL mem src));
6978 predicate(!needs_releasing_store(n));
6979
6980 ins_cost(INSN_COST);
6981 format %{ "str $src, $mem\t# int" %}
6982
6983 ins_encode(aarch64_enc_str(src, mem));
6984
6985 ins_pipe(istore_reg_mem);
6986 %}
6987
6988 // Store Long (64 bit signed)
6989 instruct storeimmL0(immL0 zero, memory8 mem)
6990 %{
6991 match(Set mem (StoreL mem zero));
6992 predicate(!needs_releasing_store(n));
6993
6994 ins_cost(INSN_COST);
6995 format %{ "str zr, $mem\t# int" %}
6996
6997 ins_encode(aarch64_enc_str0(mem));
6998
6999 ins_pipe(istore_mem);
7000 %}
7001
7002 // Store Pointer
7003 instruct storeP(iRegP src, memory8 mem)
7004 %{
7005 match(Set mem (StoreP mem src));
7006 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7007
7008 ins_cost(INSN_COST);
7009 format %{ "str $src, $mem\t# ptr" %}
7010
7011 ins_encode(aarch64_enc_str(src, mem));
7012
7013 ins_pipe(istore_reg_mem);
7014 %}
7015
7016 // Store Pointer
7017 instruct storeimmP0(immP0 zero, memory8 mem)
7018 %{
7019 match(Set mem (StoreP mem zero));
7020 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7021
7022 ins_cost(INSN_COST);
7023 format %{ "str zr, $mem\t# ptr" %}
7024
7025 ins_encode(aarch64_enc_str0(mem));
7026
7027 ins_pipe(istore_mem);
7028 %}
7029
7030 // Store Compressed Pointer
7031 instruct storeN(iRegN src, memory4 mem)
7032 %{
7033 match(Set mem (StoreN mem src));
7034 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7035
7036 ins_cost(INSN_COST);
7037 format %{ "strw $src, $mem\t# compressed ptr" %}
7038
7039 ins_encode(aarch64_enc_strw(src, mem));
7040
7041 ins_pipe(istore_reg_mem);
7042 %}
7043
7044 instruct storeImmN0(immN0 zero, memory4 mem)
7045 %{
7046 match(Set mem (StoreN mem zero));
7047 predicate(!needs_releasing_store(n) && n->as_Store()->barrier_data() == 0);
7048
7049 ins_cost(INSN_COST);
7050 format %{ "strw zr, $mem\t# compressed ptr" %}
7051
7052 ins_encode(aarch64_enc_strw0(mem));
7053
7054 ins_pipe(istore_mem);
7055 %}
7056
7057 // Store Float
7058 instruct storeF(vRegF src, memory4 mem)
7059 %{
7060 match(Set mem (StoreF mem src));
7061 predicate(!needs_releasing_store(n));
7062
7063 ins_cost(INSN_COST);
7064 format %{ "strs $src, $mem\t# float" %}
7065
7066 ins_encode( aarch64_enc_strs(src, mem) );
7067
7068 ins_pipe(pipe_class_memory);
7069 %}
7070
7071 // TODO
7072 // implement storeImmF0 and storeFImmPacked
7073
7074 // Store Double
7075 instruct storeD(vRegD src, memory8 mem)
7076 %{
7077 match(Set mem (StoreD mem src));
7078 predicate(!needs_releasing_store(n));
7079
7080 ins_cost(INSN_COST);
7081 format %{ "strd $src, $mem\t# double" %}
7082
7083 ins_encode( aarch64_enc_strd(src, mem) );
7084
7085 ins_pipe(pipe_class_memory);
7086 %}
7087
7088 // Store Compressed Klass Pointer
7089 instruct storeNKlass(iRegN src, memory4 mem)
7090 %{
7091 predicate(!needs_releasing_store(n));
7092 match(Set mem (StoreNKlass mem src));
7093
7094 ins_cost(INSN_COST);
7095 format %{ "strw $src, $mem\t# compressed klass ptr" %}
7096
7097 ins_encode(aarch64_enc_strw(src, mem));
7098
7099 ins_pipe(istore_reg_mem);
7100 %}
7101
7102 // TODO
7103 // implement storeImmD0 and storeDImmPacked
7104
7105 // prefetch instructions
7106 // Must be safe to execute with invalid address (cannot fault).
7107
7108 instruct prefetchalloc( memory8 mem ) %{
7109 match(PrefetchAllocation mem);
7110
7111 ins_cost(INSN_COST);
7112 format %{ "prfm $mem, PSTL1KEEP\t# Prefetch into level 1 cache write keep" %}
7113
7114 ins_encode( aarch64_enc_prefetchw(mem) );
7115
7116 ins_pipe(iload_prefetch);
7117 %}
7118
7119 // ---------------- volatile loads and stores ----------------
7120
7121 // Load Byte (8 bit signed)
7122 instruct loadB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7123 %{
7124 match(Set dst (LoadB mem));
7125
7126 ins_cost(VOLATILE_REF_COST);
7127 format %{ "ldarsb $dst, $mem\t# byte" %}
7128
7129 ins_encode(aarch64_enc_ldarsb(dst, mem));
7130
7131 ins_pipe(pipe_serial);
7132 %}
7133
7134 // Load Byte (8 bit signed) into long
7135 instruct loadB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7136 %{
7137 match(Set dst (ConvI2L (LoadB mem)));
7138
7139 ins_cost(VOLATILE_REF_COST);
7140 format %{ "ldarsb $dst, $mem\t# byte" %}
7141
7142 ins_encode(aarch64_enc_ldarsb(dst, mem));
7143
7144 ins_pipe(pipe_serial);
7145 %}
7146
7147 // Load Byte (8 bit unsigned)
7148 instruct loadUB_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7149 %{
7150 match(Set dst (LoadUB mem));
7151
7152 ins_cost(VOLATILE_REF_COST);
7153 format %{ "ldarb $dst, $mem\t# byte" %}
7154
7155 ins_encode(aarch64_enc_ldarb(dst, mem));
7156
7157 ins_pipe(pipe_serial);
7158 %}
7159
7160 // Load Byte (8 bit unsigned) into long
7161 instruct loadUB2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7162 %{
7163 match(Set dst (ConvI2L (LoadUB mem)));
7164
7165 ins_cost(VOLATILE_REF_COST);
7166 format %{ "ldarb $dst, $mem\t# byte" %}
7167
7168 ins_encode(aarch64_enc_ldarb(dst, mem));
7169
7170 ins_pipe(pipe_serial);
7171 %}
7172
7173 // Load Short (16 bit signed)
7174 instruct loadS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7175 %{
7176 match(Set dst (LoadS mem));
7177
7178 ins_cost(VOLATILE_REF_COST);
7179 format %{ "ldarshw $dst, $mem\t# short" %}
7180
7181 ins_encode(aarch64_enc_ldarshw(dst, mem));
7182
7183 ins_pipe(pipe_serial);
7184 %}
7185
7186 instruct loadUS_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7187 %{
7188 match(Set dst (LoadUS mem));
7189
7190 ins_cost(VOLATILE_REF_COST);
7191 format %{ "ldarhw $dst, $mem\t# short" %}
7192
7193 ins_encode(aarch64_enc_ldarhw(dst, mem));
7194
7195 ins_pipe(pipe_serial);
7196 %}
7197
7198 // Load Short/Char (16 bit unsigned) into long
7199 instruct loadUS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7200 %{
7201 match(Set dst (ConvI2L (LoadUS mem)));
7202
7203 ins_cost(VOLATILE_REF_COST);
7204 format %{ "ldarh $dst, $mem\t# short" %}
7205
7206 ins_encode(aarch64_enc_ldarh(dst, mem));
7207
7208 ins_pipe(pipe_serial);
7209 %}
7210
7211 // Load Short/Char (16 bit signed) into long
7212 instruct loadS2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7213 %{
7214 match(Set dst (ConvI2L (LoadS mem)));
7215
7216 ins_cost(VOLATILE_REF_COST);
7217 format %{ "ldarh $dst, $mem\t# short" %}
7218
7219 ins_encode(aarch64_enc_ldarsh(dst, mem));
7220
7221 ins_pipe(pipe_serial);
7222 %}
7223
7224 // Load Integer (32 bit signed)
7225 instruct loadI_volatile(iRegINoSp dst, /* sync_memory*/indirect mem)
7226 %{
7227 match(Set dst (LoadI mem));
7228
7229 ins_cost(VOLATILE_REF_COST);
7230 format %{ "ldarw $dst, $mem\t# int" %}
7231
7232 ins_encode(aarch64_enc_ldarw(dst, mem));
7233
7234 ins_pipe(pipe_serial);
7235 %}
7236
7237 // Load Integer (32 bit unsigned) into long
7238 instruct loadUI2L_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem, immL_32bits mask)
7239 %{
7240 match(Set dst (AndL (ConvI2L (LoadI mem)) mask));
7241
7242 ins_cost(VOLATILE_REF_COST);
7243 format %{ "ldarw $dst, $mem\t# int" %}
7244
7245 ins_encode(aarch64_enc_ldarw(dst, mem));
7246
7247 ins_pipe(pipe_serial);
7248 %}
7249
7250 // Load Long (64 bit signed)
7251 instruct loadL_volatile(iRegLNoSp dst, /* sync_memory*/indirect mem)
7252 %{
7253 match(Set dst (LoadL mem));
7254
7255 ins_cost(VOLATILE_REF_COST);
7256 format %{ "ldar $dst, $mem\t# int" %}
7257
7258 ins_encode(aarch64_enc_ldar(dst, mem));
7259
7260 ins_pipe(pipe_serial);
7261 %}
7262
7263 // Load Pointer
7264 instruct loadP_volatile(iRegPNoSp dst, /* sync_memory*/indirect mem)
7265 %{
7266 match(Set dst (LoadP mem));
7267 predicate(n->as_Load()->barrier_data() == 0);
7268
7269 ins_cost(VOLATILE_REF_COST);
7270 format %{ "ldar $dst, $mem\t# ptr" %}
7271
7272 ins_encode(aarch64_enc_ldar(dst, mem));
7273
7274 ins_pipe(pipe_serial);
7275 %}
7276
7277 // Load Compressed Pointer
7278 instruct loadN_volatile(iRegNNoSp dst, /* sync_memory*/indirect mem)
7279 %{
7280 match(Set dst (LoadN mem));
7281 predicate(n->as_Load()->barrier_data() == 0);
7282
7283 ins_cost(VOLATILE_REF_COST);
7284 format %{ "ldarw $dst, $mem\t# compressed ptr" %}
7285
7286 ins_encode(aarch64_enc_ldarw(dst, mem));
7287
7288 ins_pipe(pipe_serial);
7289 %}
7290
7291 // Load Float
7292 instruct loadF_volatile(vRegF dst, /* sync_memory*/indirect mem)
7293 %{
7294 match(Set dst (LoadF mem));
7295
7296 ins_cost(VOLATILE_REF_COST);
7297 format %{ "ldars $dst, $mem\t# float" %}
7298
7299 ins_encode( aarch64_enc_fldars(dst, mem) );
7300
7301 ins_pipe(pipe_serial);
7302 %}
7303
7304 // Load Double
7305 instruct loadD_volatile(vRegD dst, /* sync_memory*/indirect mem)
7306 %{
7307 match(Set dst (LoadD mem));
7308
7309 ins_cost(VOLATILE_REF_COST);
7310 format %{ "ldard $dst, $mem\t# double" %}
7311
7312 ins_encode( aarch64_enc_fldard(dst, mem) );
7313
7314 ins_pipe(pipe_serial);
7315 %}
7316
7317 // Store Byte
7318 instruct storeB_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7319 %{
7320 match(Set mem (StoreB mem src));
7321
7322 ins_cost(VOLATILE_REF_COST);
7323 format %{ "stlrb $src, $mem\t# byte" %}
7324
7325 ins_encode(aarch64_enc_stlrb(src, mem));
7326
7327 ins_pipe(pipe_class_memory);
7328 %}
7329
7330 instruct storeimmB0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7331 %{
7332 match(Set mem (StoreB mem zero));
7333
7334 ins_cost(VOLATILE_REF_COST);
7335 format %{ "stlrb zr, $mem\t# byte" %}
7336
7337 ins_encode(aarch64_enc_stlrb0(mem));
7338
7339 ins_pipe(pipe_class_memory);
7340 %}
7341
7342 // Store Char/Short
7343 instruct storeC_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7344 %{
7345 match(Set mem (StoreC mem src));
7346
7347 ins_cost(VOLATILE_REF_COST);
7348 format %{ "stlrh $src, $mem\t# short" %}
7349
7350 ins_encode(aarch64_enc_stlrh(src, mem));
7351
7352 ins_pipe(pipe_class_memory);
7353 %}
7354
7355 instruct storeimmC0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7356 %{
7357 match(Set mem (StoreC mem zero));
7358
7359 ins_cost(VOLATILE_REF_COST);
7360 format %{ "stlrh zr, $mem\t# short" %}
7361
7362 ins_encode(aarch64_enc_stlrh0(mem));
7363
7364 ins_pipe(pipe_class_memory);
7365 %}
7366
7367 // Store Integer
7368
7369 instruct storeI_volatile(iRegIorL2I src, /* sync_memory*/indirect mem)
7370 %{
7371 match(Set mem(StoreI mem src));
7372
7373 ins_cost(VOLATILE_REF_COST);
7374 format %{ "stlrw $src, $mem\t# int" %}
7375
7376 ins_encode(aarch64_enc_stlrw(src, mem));
7377
7378 ins_pipe(pipe_class_memory);
7379 %}
7380
7381 instruct storeimmI0_volatile(immI0 zero, /* sync_memory*/indirect mem)
7382 %{
7383 match(Set mem(StoreI mem zero));
7384
7385 ins_cost(VOLATILE_REF_COST);
7386 format %{ "stlrw zr, $mem\t# int" %}
7387
7388 ins_encode(aarch64_enc_stlrw0(mem));
7389
7390 ins_pipe(pipe_class_memory);
7391 %}
7392
7393 // Store Long (64 bit signed)
7394 instruct storeL_volatile(iRegL src, /* sync_memory*/indirect mem)
7395 %{
7396 match(Set mem (StoreL mem src));
7397
7398 ins_cost(VOLATILE_REF_COST);
7399 format %{ "stlr $src, $mem\t# int" %}
7400
7401 ins_encode(aarch64_enc_stlr(src, mem));
7402
7403 ins_pipe(pipe_class_memory);
7404 %}
7405
7406 instruct storeimmL0_volatile(immL0 zero, /* sync_memory*/indirect mem)
7407 %{
7408 match(Set mem (StoreL mem zero));
7409
7410 ins_cost(VOLATILE_REF_COST);
7411 format %{ "stlr zr, $mem\t# int" %}
7412
7413 ins_encode(aarch64_enc_stlr0(mem));
7414
7415 ins_pipe(pipe_class_memory);
7416 %}
7417
7418 // Store Pointer
7419 instruct storeP_volatile(iRegP src, /* sync_memory*/indirect mem)
7420 %{
7421 match(Set mem (StoreP mem src));
7422 predicate(n->as_Store()->barrier_data() == 0);
7423
7424 ins_cost(VOLATILE_REF_COST);
7425 format %{ "stlr $src, $mem\t# ptr" %}
7426
7427 ins_encode(aarch64_enc_stlr(src, mem));
7428
7429 ins_pipe(pipe_class_memory);
7430 %}
7431
7432 instruct storeimmP0_volatile(immP0 zero, /* sync_memory*/indirect mem)
7433 %{
7434 match(Set mem (StoreP mem zero));
7435 predicate(n->as_Store()->barrier_data() == 0);
7436
7437 ins_cost(VOLATILE_REF_COST);
7438 format %{ "stlr zr, $mem\t# ptr" %}
7439
7440 ins_encode(aarch64_enc_stlr0(mem));
7441
7442 ins_pipe(pipe_class_memory);
7443 %}
7444
7445 // Store Compressed Pointer
7446 instruct storeN_volatile(iRegN src, /* sync_memory*/indirect mem)
7447 %{
7448 match(Set mem (StoreN mem src));
7449 predicate(n->as_Store()->barrier_data() == 0);
7450
7451 ins_cost(VOLATILE_REF_COST);
7452 format %{ "stlrw $src, $mem\t# compressed ptr" %}
7453
7454 ins_encode(aarch64_enc_stlrw(src, mem));
7455
7456 ins_pipe(pipe_class_memory);
7457 %}
7458
7459 instruct storeimmN0_volatile(immN0 zero, /* sync_memory*/indirect mem)
7460 %{
7461 match(Set mem (StoreN mem zero));
7462 predicate(n->as_Store()->barrier_data() == 0);
7463
7464 ins_cost(VOLATILE_REF_COST);
7465 format %{ "stlrw zr, $mem\t# compressed ptr" %}
7466
7467 ins_encode(aarch64_enc_stlrw0(mem));
7468
7469 ins_pipe(pipe_class_memory);
7470 %}
7471
7472 // Store Float
7473 instruct storeF_volatile(vRegF src, /* sync_memory*/indirect mem)
7474 %{
7475 match(Set mem (StoreF mem src));
7476
7477 ins_cost(VOLATILE_REF_COST);
7478 format %{ "stlrs $src, $mem\t# float" %}
7479
7480 ins_encode( aarch64_enc_fstlrs(src, mem) );
7481
7482 ins_pipe(pipe_class_memory);
7483 %}
7484
7485 // TODO
7486 // implement storeImmF0 and storeFImmPacked
7487
7488 // Store Double
7489 instruct storeD_volatile(vRegD src, /* sync_memory*/indirect mem)
7490 %{
7491 match(Set mem (StoreD mem src));
7492
7493 ins_cost(VOLATILE_REF_COST);
7494 format %{ "stlrd $src, $mem\t# double" %}
7495
7496 ins_encode( aarch64_enc_fstlrd(src, mem) );
7497
7498 ins_pipe(pipe_class_memory);
7499 %}
7500
7501 // ---------------- end of volatile loads and stores ----------------
7502
7503 instruct cacheWB(indirect addr)
7504 %{
7505 predicate(VM_Version::supports_data_cache_line_flush());
7506 match(CacheWB addr);
7507
7508 ins_cost(100);
7509 format %{"cache wb $addr" %}
7510 ins_encode %{
7511 assert($addr->index_position() < 0, "should be");
7512 assert($addr$$disp == 0, "should be");
7513 __ cache_wb(Address($addr$$base$$Register, 0));
7514 %}
7515 ins_pipe(pipe_slow); // XXX
7516 %}
7517
7518 instruct cacheWBPreSync()
7519 %{
7520 predicate(VM_Version::supports_data_cache_line_flush());
7521 match(CacheWBPreSync);
7522
7523 ins_cost(100);
7524 format %{"cache wb presync" %}
7525 ins_encode %{
7526 __ cache_wbsync(true);
7527 %}
7528 ins_pipe(pipe_slow); // XXX
7529 %}
7530
7531 instruct cacheWBPostSync()
7532 %{
7533 predicate(VM_Version::supports_data_cache_line_flush());
7534 match(CacheWBPostSync);
7535
7536 ins_cost(100);
7537 format %{"cache wb postsync" %}
7538 ins_encode %{
7539 __ cache_wbsync(false);
7540 %}
7541 ins_pipe(pipe_slow); // XXX
7542 %}
7543
7544 // ============================================================================
7545 // BSWAP Instructions
7546
7547 instruct bytes_reverse_int(iRegINoSp dst, iRegIorL2I src) %{
7548 match(Set dst (ReverseBytesI src));
7549
7550 ins_cost(INSN_COST);
7551 format %{ "revw $dst, $src" %}
7552
7553 ins_encode %{
7554 __ revw(as_Register($dst$$reg), as_Register($src$$reg));
7555 %}
7556
7557 ins_pipe(ialu_reg);
7558 %}
7559
7560 instruct bytes_reverse_long(iRegLNoSp dst, iRegL src) %{
7561 match(Set dst (ReverseBytesL src));
7562
7563 ins_cost(INSN_COST);
7564 format %{ "rev $dst, $src" %}
7565
7566 ins_encode %{
7567 __ rev(as_Register($dst$$reg), as_Register($src$$reg));
7568 %}
7569
7570 ins_pipe(ialu_reg);
7571 %}
7572
7573 instruct bytes_reverse_unsigned_short(iRegINoSp dst, iRegIorL2I src) %{
7574 match(Set dst (ReverseBytesUS src));
7575
7576 ins_cost(INSN_COST);
7577 format %{ "rev16w $dst, $src" %}
7578
7579 ins_encode %{
7580 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7581 %}
7582
7583 ins_pipe(ialu_reg);
7584 %}
7585
7586 instruct bytes_reverse_short(iRegINoSp dst, iRegIorL2I src) %{
7587 match(Set dst (ReverseBytesS src));
7588
7589 ins_cost(INSN_COST);
7590 format %{ "rev16w $dst, $src\n\t"
7591 "sbfmw $dst, $dst, #0, #15" %}
7592
7593 ins_encode %{
7594 __ rev16w(as_Register($dst$$reg), as_Register($src$$reg));
7595 __ sbfmw(as_Register($dst$$reg), as_Register($dst$$reg), 0U, 15U);
7596 %}
7597
7598 ins_pipe(ialu_reg);
7599 %}
7600
7601 // ============================================================================
7602 // Zero Count Instructions
7603
7604 instruct countLeadingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7605 match(Set dst (CountLeadingZerosI src));
7606
7607 ins_cost(INSN_COST);
7608 format %{ "clzw $dst, $src" %}
7609 ins_encode %{
7610 __ clzw(as_Register($dst$$reg), as_Register($src$$reg));
7611 %}
7612
7613 ins_pipe(ialu_reg);
7614 %}
7615
7616 instruct countLeadingZerosL(iRegINoSp dst, iRegL src) %{
7617 match(Set dst (CountLeadingZerosL src));
7618
7619 ins_cost(INSN_COST);
7620 format %{ "clz $dst, $src" %}
7621 ins_encode %{
7622 __ clz(as_Register($dst$$reg), as_Register($src$$reg));
7623 %}
7624
7625 ins_pipe(ialu_reg);
7626 %}
7627
7628 instruct countTrailingZerosI(iRegINoSp dst, iRegIorL2I src) %{
7629 match(Set dst (CountTrailingZerosI src));
7630
7631 ins_cost(INSN_COST * 2);
7632 format %{ "rbitw $dst, $src\n\t"
7633 "clzw $dst, $dst" %}
7634 ins_encode %{
7635 __ rbitw(as_Register($dst$$reg), as_Register($src$$reg));
7636 __ clzw(as_Register($dst$$reg), as_Register($dst$$reg));
7637 %}
7638
7639 ins_pipe(ialu_reg);
7640 %}
7641
7642 instruct countTrailingZerosL(iRegINoSp dst, iRegL src) %{
7643 match(Set dst (CountTrailingZerosL src));
7644
7645 ins_cost(INSN_COST * 2);
7646 format %{ "rbit $dst, $src\n\t"
7647 "clz $dst, $dst" %}
7648 ins_encode %{
7649 __ rbit(as_Register($dst$$reg), as_Register($src$$reg));
7650 __ clz(as_Register($dst$$reg), as_Register($dst$$reg));
7651 %}
7652
7653 ins_pipe(ialu_reg);
7654 %}
7655
7656 //---------- Population Count Instructions -------------------------------------
7657 //
7658
7659 instruct popCountI(iRegINoSp dst, iRegIorL2I src, vRegF tmp) %{
7660 match(Set dst (PopCountI src));
7661 effect(TEMP tmp);
7662 ins_cost(INSN_COST * 13);
7663
7664 format %{ "movw $src, $src\n\t"
7665 "mov $tmp, $src\t# vector (1D)\n\t"
7666 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7667 "addv $tmp, $tmp\t# vector (8B)\n\t"
7668 "mov $dst, $tmp\t# vector (1D)" %}
7669 ins_encode %{
7670 __ movw($src$$Register, $src$$Register); // ensure top 32 bits 0
7671 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7672 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7673 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7674 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7675 %}
7676
7677 ins_pipe(pipe_class_default);
7678 %}
7679
7680 instruct popCountI_mem(iRegINoSp dst, memory4 mem, vRegF tmp) %{
7681 match(Set dst (PopCountI (LoadI mem)));
7682 effect(TEMP tmp);
7683 ins_cost(INSN_COST * 13);
7684
7685 format %{ "ldrs $tmp, $mem\n\t"
7686 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7687 "addv $tmp, $tmp\t# vector (8B)\n\t"
7688 "mov $dst, $tmp\t# vector (1D)" %}
7689 ins_encode %{
7690 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7691 loadStore(masm, &MacroAssembler::ldrs, tmp_reg, $mem->opcode(),
7692 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
7693 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7694 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7695 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7696 %}
7697
7698 ins_pipe(pipe_class_default);
7699 %}
7700
7701 // Note: Long.bitCount(long) returns an int.
7702 instruct popCountL(iRegINoSp dst, iRegL src, vRegD tmp) %{
7703 match(Set dst (PopCountL src));
7704 effect(TEMP tmp);
7705 ins_cost(INSN_COST * 13);
7706
7707 format %{ "mov $tmp, $src\t# vector (1D)\n\t"
7708 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7709 "addv $tmp, $tmp\t# vector (8B)\n\t"
7710 "mov $dst, $tmp\t# vector (1D)" %}
7711 ins_encode %{
7712 __ mov($tmp$$FloatRegister, __ D, 0, $src$$Register);
7713 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7714 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7715 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7716 %}
7717
7718 ins_pipe(pipe_class_default);
7719 %}
7720
7721 instruct popCountL_mem(iRegINoSp dst, memory8 mem, vRegD tmp) %{
7722 match(Set dst (PopCountL (LoadL mem)));
7723 effect(TEMP tmp);
7724 ins_cost(INSN_COST * 13);
7725
7726 format %{ "ldrd $tmp, $mem\n\t"
7727 "cnt $tmp, $tmp\t# vector (8B)\n\t"
7728 "addv $tmp, $tmp\t# vector (8B)\n\t"
7729 "mov $dst, $tmp\t# vector (1D)" %}
7730 ins_encode %{
7731 FloatRegister tmp_reg = as_FloatRegister($tmp$$reg);
7732 loadStore(masm, &MacroAssembler::ldrd, tmp_reg, $mem->opcode(),
7733 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
7734 __ cnt($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7735 __ addv($tmp$$FloatRegister, __ T8B, $tmp$$FloatRegister);
7736 __ mov($dst$$Register, $tmp$$FloatRegister, __ D, 0);
7737 %}
7738
7739 ins_pipe(pipe_class_default);
7740 %}
7741
7742 // ============================================================================
7743 // VerifyVectorAlignment Instruction
7744
7745 instruct verify_vector_alignment(iRegP addr, immL_positive_bitmaskI mask, rFlagsReg cr) %{
7746 match(Set addr (VerifyVectorAlignment addr mask));
7747 effect(KILL cr);
7748 format %{ "verify_vector_alignment $addr $mask \t! verify alignment" %}
7749 ins_encode %{
7750 Label Lskip;
7751 // check if masked bits of addr are zero
7752 __ tst($addr$$Register, $mask$$constant);
7753 __ br(Assembler::EQ, Lskip);
7754 __ stop("verify_vector_alignment found a misaligned vector memory access");
7755 __ bind(Lskip);
7756 %}
7757 ins_pipe(pipe_slow);
7758 %}
7759
7760 // ============================================================================
7761 // MemBar Instruction
7762
7763 instruct load_fence() %{
7764 match(LoadFence);
7765 ins_cost(VOLATILE_REF_COST);
7766
7767 format %{ "load_fence" %}
7768
7769 ins_encode %{
7770 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7771 %}
7772 ins_pipe(pipe_serial);
7773 %}
7774
7775 instruct unnecessary_membar_acquire() %{
7776 predicate(unnecessary_acquire(n));
7777 match(MemBarAcquire);
7778 ins_cost(0);
7779
7780 format %{ "membar_acquire (elided)" %}
7781
7782 ins_encode %{
7783 __ block_comment("membar_acquire (elided)");
7784 %}
7785
7786 ins_pipe(pipe_class_empty);
7787 %}
7788
7789 instruct membar_acquire() %{
7790 match(MemBarAcquire);
7791 ins_cost(VOLATILE_REF_COST);
7792
7793 format %{ "membar_acquire\n\t"
7794 "dmb ishld" %}
7795
7796 ins_encode %{
7797 __ block_comment("membar_acquire");
7798 __ membar(Assembler::LoadLoad|Assembler::LoadStore);
7799 %}
7800
7801 ins_pipe(pipe_serial);
7802 %}
7803
7804
7805 instruct membar_acquire_lock() %{
7806 match(MemBarAcquireLock);
7807 ins_cost(VOLATILE_REF_COST);
7808
7809 format %{ "membar_acquire_lock (elided)" %}
7810
7811 ins_encode %{
7812 __ block_comment("membar_acquire_lock (elided)");
7813 %}
7814
7815 ins_pipe(pipe_serial);
7816 %}
7817
7818 instruct store_fence() %{
7819 match(StoreFence);
7820 ins_cost(VOLATILE_REF_COST);
7821
7822 format %{ "store_fence" %}
7823
7824 ins_encode %{
7825 __ membar(Assembler::LoadStore|Assembler::StoreStore);
7826 %}
7827 ins_pipe(pipe_serial);
7828 %}
7829
7830 instruct unnecessary_membar_release() %{
7831 predicate(unnecessary_release(n));
7832 match(MemBarRelease);
7833 ins_cost(0);
7834
7835 format %{ "membar_release (elided)" %}
7836
7837 ins_encode %{
7838 __ block_comment("membar_release (elided)");
7839 %}
7840 ins_pipe(pipe_serial);
7841 %}
7842
7843 instruct membar_release() %{
7844 match(MemBarRelease);
7845 ins_cost(VOLATILE_REF_COST);
7846
7847 format %{ "membar_release\n\t"
7848 "dmb ishst\n\tdmb ishld" %}
7849
7850 ins_encode %{
7851 __ block_comment("membar_release");
7852 // These will be merged if AlwaysMergeDMB is enabled.
7853 __ membar(Assembler::StoreStore);
7854 __ membar(Assembler::LoadStore);
7855 %}
7856 ins_pipe(pipe_serial);
7857 %}
7858
7859 instruct membar_storestore() %{
7860 match(MemBarStoreStore);
7861 match(StoreStoreFence);
7862 ins_cost(VOLATILE_REF_COST);
7863
7864 format %{ "MEMBAR-store-store" %}
7865
7866 ins_encode %{
7867 __ membar(Assembler::StoreStore);
7868 %}
7869 ins_pipe(pipe_serial);
7870 %}
7871
7872 instruct membar_release_lock() %{
7873 match(MemBarReleaseLock);
7874 ins_cost(VOLATILE_REF_COST);
7875
7876 format %{ "membar_release_lock (elided)" %}
7877
7878 ins_encode %{
7879 __ block_comment("membar_release_lock (elided)");
7880 %}
7881
7882 ins_pipe(pipe_serial);
7883 %}
7884
7885 instruct unnecessary_membar_volatile() %{
7886 predicate(unnecessary_volatile(n));
7887 match(MemBarVolatile);
7888 ins_cost(0);
7889
7890 format %{ "membar_volatile (elided)" %}
7891
7892 ins_encode %{
7893 __ block_comment("membar_volatile (elided)");
7894 %}
7895
7896 ins_pipe(pipe_serial);
7897 %}
7898
7899 instruct membar_volatile() %{
7900 match(MemBarVolatile);
7901 ins_cost(VOLATILE_REF_COST*100);
7902
7903 format %{ "membar_volatile\n\t"
7904 "dmb ish"%}
7905
7906 ins_encode %{
7907 __ block_comment("membar_volatile");
7908 __ membar(Assembler::StoreLoad);
7909 %}
7910
7911 ins_pipe(pipe_serial);
7912 %}
7913
7914 // ============================================================================
7915 // Cast/Convert Instructions
7916
7917 instruct castX2P(iRegPNoSp dst, iRegL src) %{
7918 match(Set dst (CastX2P src));
7919
7920 ins_cost(INSN_COST);
7921 format %{ "mov $dst, $src\t# long -> ptr" %}
7922
7923 ins_encode %{
7924 if ($dst$$reg != $src$$reg) {
7925 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7926 }
7927 %}
7928
7929 ins_pipe(ialu_reg);
7930 %}
7931
7932 instruct castP2X(iRegLNoSp dst, iRegP src) %{
7933 match(Set dst (CastP2X src));
7934
7935 ins_cost(INSN_COST);
7936 format %{ "mov $dst, $src\t# ptr -> long" %}
7937
7938 ins_encode %{
7939 if ($dst$$reg != $src$$reg) {
7940 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
7941 }
7942 %}
7943
7944 ins_pipe(ialu_reg);
7945 %}
7946
7947 // Convert oop into int for vectors alignment masking
7948 instruct convP2I(iRegINoSp dst, iRegP src) %{
7949 match(Set dst (ConvL2I (CastP2X src)));
7950
7951 ins_cost(INSN_COST);
7952 format %{ "movw $dst, $src\t# ptr -> int" %}
7953 ins_encode %{
7954 __ movw($dst$$Register, $src$$Register);
7955 %}
7956
7957 ins_pipe(ialu_reg);
7958 %}
7959
7960 // Convert compressed oop into int for vectors alignment masking
7961 // in case of 32bit oops (heap < 4Gb).
7962 instruct convN2I(iRegINoSp dst, iRegN src)
7963 %{
7964 predicate(CompressedOops::shift() == 0);
7965 match(Set dst (ConvL2I (CastP2X (DecodeN src))));
7966
7967 ins_cost(INSN_COST);
7968 format %{ "mov dst, $src\t# compressed ptr -> int" %}
7969 ins_encode %{
7970 __ movw($dst$$Register, $src$$Register);
7971 %}
7972
7973 ins_pipe(ialu_reg);
7974 %}
7975
7976
7977 // Convert oop pointer into compressed form
7978 instruct encodeHeapOop(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7979 predicate(n->bottom_type()->make_ptr()->ptr() != TypePtr::NotNull);
7980 match(Set dst (EncodeP src));
7981 effect(KILL cr);
7982 ins_cost(INSN_COST * 3);
7983 format %{ "encode_heap_oop $dst, $src" %}
7984 ins_encode %{
7985 Register s = $src$$Register;
7986 Register d = $dst$$Register;
7987 __ encode_heap_oop(d, s);
7988 %}
7989 ins_pipe(ialu_reg);
7990 %}
7991
7992 instruct encodeHeapOop_not_null(iRegNNoSp dst, iRegP src, rFlagsReg cr) %{
7993 predicate(n->bottom_type()->make_ptr()->ptr() == TypePtr::NotNull);
7994 match(Set dst (EncodeP src));
7995 ins_cost(INSN_COST * 3);
7996 format %{ "encode_heap_oop_not_null $dst, $src" %}
7997 ins_encode %{
7998 __ encode_heap_oop_not_null($dst$$Register, $src$$Register);
7999 %}
8000 ins_pipe(ialu_reg);
8001 %}
8002
8003 instruct decodeHeapOop(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8004 predicate(n->bottom_type()->is_ptr()->ptr() != TypePtr::NotNull &&
8005 n->bottom_type()->is_ptr()->ptr() != TypePtr::Constant);
8006 match(Set dst (DecodeN src));
8007 ins_cost(INSN_COST * 3);
8008 format %{ "decode_heap_oop $dst, $src" %}
8009 ins_encode %{
8010 Register s = $src$$Register;
8011 Register d = $dst$$Register;
8012 __ decode_heap_oop(d, s);
8013 %}
8014 ins_pipe(ialu_reg);
8015 %}
8016
8017 instruct decodeHeapOop_not_null(iRegPNoSp dst, iRegN src, rFlagsReg cr) %{
8018 predicate(n->bottom_type()->is_ptr()->ptr() == TypePtr::NotNull ||
8019 n->bottom_type()->is_ptr()->ptr() == TypePtr::Constant);
8020 match(Set dst (DecodeN src));
8021 ins_cost(INSN_COST * 3);
8022 format %{ "decode_heap_oop_not_null $dst, $src" %}
8023 ins_encode %{
8024 Register s = $src$$Register;
8025 Register d = $dst$$Register;
8026 __ decode_heap_oop_not_null(d, s);
8027 %}
8028 ins_pipe(ialu_reg);
8029 %}
8030
8031 // n.b. AArch64 implementations of encode_klass_not_null and
8032 // decode_klass_not_null do not modify the flags register so, unlike
8033 // Intel, we don't kill CR as a side effect here
8034
8035 instruct encodeKlass_not_null(iRegNNoSp dst, iRegP src) %{
8036 match(Set dst (EncodePKlass src));
8037
8038 ins_cost(INSN_COST * 3);
8039 format %{ "encode_klass_not_null $dst,$src" %}
8040
8041 ins_encode %{
8042 Register src_reg = as_Register($src$$reg);
8043 Register dst_reg = as_Register($dst$$reg);
8044 __ encode_klass_not_null(dst_reg, src_reg);
8045 %}
8046
8047 ins_pipe(ialu_reg);
8048 %}
8049
8050 instruct decodeKlass_not_null(iRegPNoSp dst, iRegN src) %{
8051 match(Set dst (DecodeNKlass src));
8052
8053 ins_cost(INSN_COST * 3);
8054 format %{ "decode_klass_not_null $dst,$src" %}
8055
8056 ins_encode %{
8057 Register src_reg = as_Register($src$$reg);
8058 Register dst_reg = as_Register($dst$$reg);
8059 if (dst_reg != src_reg) {
8060 __ decode_klass_not_null(dst_reg, src_reg);
8061 } else {
8062 __ decode_klass_not_null(dst_reg);
8063 }
8064 %}
8065
8066 ins_pipe(ialu_reg);
8067 %}
8068
8069 instruct checkCastPP(iRegPNoSp dst)
8070 %{
8071 match(Set dst (CheckCastPP dst));
8072
8073 size(0);
8074 format %{ "# checkcastPP of $dst" %}
8075 ins_encode(/* empty encoding */);
8076 ins_pipe(pipe_class_empty);
8077 %}
8078
8079 instruct castPP(iRegPNoSp dst)
8080 %{
8081 match(Set dst (CastPP dst));
8082
8083 size(0);
8084 format %{ "# castPP of $dst" %}
8085 ins_encode(/* empty encoding */);
8086 ins_pipe(pipe_class_empty);
8087 %}
8088
8089 instruct castII(iRegI dst)
8090 %{
8091 match(Set dst (CastII dst));
8092
8093 size(0);
8094 format %{ "# castII of $dst" %}
8095 ins_encode(/* empty encoding */);
8096 ins_cost(0);
8097 ins_pipe(pipe_class_empty);
8098 %}
8099
8100 instruct castLL(iRegL dst)
8101 %{
8102 match(Set dst (CastLL dst));
8103
8104 size(0);
8105 format %{ "# castLL of $dst" %}
8106 ins_encode(/* empty encoding */);
8107 ins_cost(0);
8108 ins_pipe(pipe_class_empty);
8109 %}
8110
8111 instruct castFF(vRegF dst)
8112 %{
8113 match(Set dst (CastFF dst));
8114
8115 size(0);
8116 format %{ "# castFF of $dst" %}
8117 ins_encode(/* empty encoding */);
8118 ins_cost(0);
8119 ins_pipe(pipe_class_empty);
8120 %}
8121
8122 instruct castDD(vRegD dst)
8123 %{
8124 match(Set dst (CastDD dst));
8125
8126 size(0);
8127 format %{ "# castDD of $dst" %}
8128 ins_encode(/* empty encoding */);
8129 ins_cost(0);
8130 ins_pipe(pipe_class_empty);
8131 %}
8132
8133 instruct castVV(vReg dst)
8134 %{
8135 match(Set dst (CastVV dst));
8136
8137 size(0);
8138 format %{ "# castVV of $dst" %}
8139 ins_encode(/* empty encoding */);
8140 ins_cost(0);
8141 ins_pipe(pipe_class_empty);
8142 %}
8143
8144 instruct castVVMask(pRegGov dst)
8145 %{
8146 match(Set dst (CastVV dst));
8147
8148 size(0);
8149 format %{ "# castVV of $dst" %}
8150 ins_encode(/* empty encoding */);
8151 ins_cost(0);
8152 ins_pipe(pipe_class_empty);
8153 %}
8154
8155 // ============================================================================
8156 // Atomic operation instructions
8157 //
8158
8159 // standard CompareAndSwapX when we are using barriers
8160 // these have higher priority than the rules selected by a predicate
8161
8162 // XXX No flag versions for CompareAndSwap{I,L,P,N} because matcher
8163 // can't match them
8164
8165 instruct compareAndSwapB(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8166
8167 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8168 ins_cost(2 * VOLATILE_REF_COST);
8169
8170 effect(KILL cr);
8171
8172 format %{
8173 "cmpxchgb $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8174 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8175 %}
8176
8177 ins_encode(aarch64_enc_cmpxchgb(mem, oldval, newval),
8178 aarch64_enc_cset_eq(res));
8179
8180 ins_pipe(pipe_slow);
8181 %}
8182
8183 instruct compareAndSwapS(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8184
8185 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8186 ins_cost(2 * VOLATILE_REF_COST);
8187
8188 effect(KILL cr);
8189
8190 format %{
8191 "cmpxchgs $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8192 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8193 %}
8194
8195 ins_encode(aarch64_enc_cmpxchgs(mem, oldval, newval),
8196 aarch64_enc_cset_eq(res));
8197
8198 ins_pipe(pipe_slow);
8199 %}
8200
8201 instruct compareAndSwapI(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8202
8203 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8204 ins_cost(2 * VOLATILE_REF_COST);
8205
8206 effect(KILL cr);
8207
8208 format %{
8209 "cmpxchgw $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8210 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8211 %}
8212
8213 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8214 aarch64_enc_cset_eq(res));
8215
8216 ins_pipe(pipe_slow);
8217 %}
8218
8219 instruct compareAndSwapL(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8220
8221 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8222 ins_cost(2 * VOLATILE_REF_COST);
8223
8224 effect(KILL cr);
8225
8226 format %{
8227 "cmpxchg $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8228 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8229 %}
8230
8231 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8232 aarch64_enc_cset_eq(res));
8233
8234 ins_pipe(pipe_slow);
8235 %}
8236
8237 instruct compareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8238
8239 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8240 predicate(n->as_LoadStore()->barrier_data() == 0);
8241 ins_cost(2 * VOLATILE_REF_COST);
8242
8243 effect(KILL cr);
8244
8245 format %{
8246 "cmpxchg $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8247 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8248 %}
8249
8250 ins_encode(aarch64_enc_cmpxchg(mem, oldval, newval),
8251 aarch64_enc_cset_eq(res));
8252
8253 ins_pipe(pipe_slow);
8254 %}
8255
8256 instruct compareAndSwapN(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8257
8258 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8259 predicate(n->as_LoadStore()->barrier_data() == 0);
8260 ins_cost(2 * VOLATILE_REF_COST);
8261
8262 effect(KILL cr);
8263
8264 format %{
8265 "cmpxchgw $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8266 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8267 %}
8268
8269 ins_encode(aarch64_enc_cmpxchgw(mem, oldval, newval),
8270 aarch64_enc_cset_eq(res));
8271
8272 ins_pipe(pipe_slow);
8273 %}
8274
8275 // alternative CompareAndSwapX when we are eliding barriers
8276
8277 instruct compareAndSwapBAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8278
8279 predicate(needs_acquiring_load_exclusive(n));
8280 match(Set res (CompareAndSwapB mem (Binary oldval newval)));
8281 ins_cost(VOLATILE_REF_COST);
8282
8283 effect(KILL cr);
8284
8285 format %{
8286 "cmpxchgb_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8287 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8288 %}
8289
8290 ins_encode(aarch64_enc_cmpxchgb_acq(mem, oldval, newval),
8291 aarch64_enc_cset_eq(res));
8292
8293 ins_pipe(pipe_slow);
8294 %}
8295
8296 instruct compareAndSwapSAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8297
8298 predicate(needs_acquiring_load_exclusive(n));
8299 match(Set res (CompareAndSwapS mem (Binary oldval newval)));
8300 ins_cost(VOLATILE_REF_COST);
8301
8302 effect(KILL cr);
8303
8304 format %{
8305 "cmpxchgs_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8306 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8307 %}
8308
8309 ins_encode(aarch64_enc_cmpxchgs_acq(mem, oldval, newval),
8310 aarch64_enc_cset_eq(res));
8311
8312 ins_pipe(pipe_slow);
8313 %}
8314
8315 instruct compareAndSwapIAcq(iRegINoSp res, indirect mem, iRegINoSp oldval, iRegINoSp newval, rFlagsReg cr) %{
8316
8317 predicate(needs_acquiring_load_exclusive(n));
8318 match(Set res (CompareAndSwapI mem (Binary oldval newval)));
8319 ins_cost(VOLATILE_REF_COST);
8320
8321 effect(KILL cr);
8322
8323 format %{
8324 "cmpxchgw_acq $mem, $oldval, $newval\t# (int) if $mem == $oldval then $mem <-- $newval"
8325 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8326 %}
8327
8328 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8329 aarch64_enc_cset_eq(res));
8330
8331 ins_pipe(pipe_slow);
8332 %}
8333
8334 instruct compareAndSwapLAcq(iRegINoSp res, indirect mem, iRegLNoSp oldval, iRegLNoSp newval, rFlagsReg cr) %{
8335
8336 predicate(needs_acquiring_load_exclusive(n));
8337 match(Set res (CompareAndSwapL mem (Binary oldval newval)));
8338 ins_cost(VOLATILE_REF_COST);
8339
8340 effect(KILL cr);
8341
8342 format %{
8343 "cmpxchg_acq $mem, $oldval, $newval\t# (long) if $mem == $oldval then $mem <-- $newval"
8344 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8345 %}
8346
8347 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8348 aarch64_enc_cset_eq(res));
8349
8350 ins_pipe(pipe_slow);
8351 %}
8352
8353 instruct compareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8354
8355 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8356 match(Set res (CompareAndSwapP mem (Binary oldval newval)));
8357 ins_cost(VOLATILE_REF_COST);
8358
8359 effect(KILL cr);
8360
8361 format %{
8362 "cmpxchg_acq $mem, $oldval, $newval\t# (ptr) if $mem == $oldval then $mem <-- $newval"
8363 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8364 %}
8365
8366 ins_encode(aarch64_enc_cmpxchg_acq(mem, oldval, newval),
8367 aarch64_enc_cset_eq(res));
8368
8369 ins_pipe(pipe_slow);
8370 %}
8371
8372 instruct compareAndSwapNAcq(iRegINoSp res, indirect mem, iRegNNoSp oldval, iRegNNoSp newval, rFlagsReg cr) %{
8373
8374 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8375 match(Set res (CompareAndSwapN mem (Binary oldval newval)));
8376 ins_cost(VOLATILE_REF_COST);
8377
8378 effect(KILL cr);
8379
8380 format %{
8381 "cmpxchgw_acq $mem, $oldval, $newval\t# (narrow oop) if $mem == $oldval then $mem <-- $newval"
8382 "cset $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8383 %}
8384
8385 ins_encode(aarch64_enc_cmpxchgw_acq(mem, oldval, newval),
8386 aarch64_enc_cset_eq(res));
8387
8388 ins_pipe(pipe_slow);
8389 %}
8390
8391
8392 // ---------------------------------------------------------------------
8393
8394 // BEGIN This section of the file is automatically generated. Do not edit --------------
8395
8396 // Sundry CAS operations. Note that release is always true,
8397 // regardless of the memory ordering of the CAS. This is because we
8398 // need the volatile case to be sequentially consistent but there is
8399 // no trailing StoreLoad barrier emitted by C2. Unfortunately we
8400 // can't check the type of memory ordering here, so we always emit a
8401 // STLXR.
8402
8403 // This section is generated from cas.m4
8404
8405
8406 // This pattern is generated automatically from cas.m4.
8407 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8408 instruct compareAndExchangeB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8409 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8410 ins_cost(2 * VOLATILE_REF_COST);
8411 effect(TEMP_DEF res, KILL cr);
8412 format %{
8413 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8414 %}
8415 ins_encode %{
8416 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8417 Assembler::byte, /*acquire*/ false, /*release*/ true,
8418 /*weak*/ false, $res$$Register);
8419 __ sxtbw($res$$Register, $res$$Register);
8420 %}
8421 ins_pipe(pipe_slow);
8422 %}
8423
8424 // This pattern is generated automatically from cas.m4.
8425 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8426 instruct compareAndExchangeS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8427 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8428 ins_cost(2 * VOLATILE_REF_COST);
8429 effect(TEMP_DEF res, KILL cr);
8430 format %{
8431 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8432 %}
8433 ins_encode %{
8434 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8435 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8436 /*weak*/ false, $res$$Register);
8437 __ sxthw($res$$Register, $res$$Register);
8438 %}
8439 ins_pipe(pipe_slow);
8440 %}
8441
8442 // This pattern is generated automatically from cas.m4.
8443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8444 instruct compareAndExchangeI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8445 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8446 ins_cost(2 * VOLATILE_REF_COST);
8447 effect(TEMP_DEF res, KILL cr);
8448 format %{
8449 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8450 %}
8451 ins_encode %{
8452 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8453 Assembler::word, /*acquire*/ false, /*release*/ true,
8454 /*weak*/ false, $res$$Register);
8455 %}
8456 ins_pipe(pipe_slow);
8457 %}
8458
8459 // This pattern is generated automatically from cas.m4.
8460 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8461 instruct compareAndExchangeL(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8462 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8463 ins_cost(2 * VOLATILE_REF_COST);
8464 effect(TEMP_DEF res, KILL cr);
8465 format %{
8466 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8467 %}
8468 ins_encode %{
8469 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8470 Assembler::xword, /*acquire*/ false, /*release*/ true,
8471 /*weak*/ false, $res$$Register);
8472 %}
8473 ins_pipe(pipe_slow);
8474 %}
8475
8476 // This pattern is generated automatically from cas.m4.
8477 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8478 instruct compareAndExchangeN(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8479 predicate(n->as_LoadStore()->barrier_data() == 0);
8480 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8481 ins_cost(2 * VOLATILE_REF_COST);
8482 effect(TEMP_DEF res, KILL cr);
8483 format %{
8484 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8485 %}
8486 ins_encode %{
8487 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8488 Assembler::word, /*acquire*/ false, /*release*/ true,
8489 /*weak*/ false, $res$$Register);
8490 %}
8491 ins_pipe(pipe_slow);
8492 %}
8493
8494 // This pattern is generated automatically from cas.m4.
8495 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8496 instruct compareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8497 predicate(n->as_LoadStore()->barrier_data() == 0);
8498 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8499 ins_cost(2 * VOLATILE_REF_COST);
8500 effect(TEMP_DEF res, KILL cr);
8501 format %{
8502 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8503 %}
8504 ins_encode %{
8505 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8506 Assembler::xword, /*acquire*/ false, /*release*/ true,
8507 /*weak*/ false, $res$$Register);
8508 %}
8509 ins_pipe(pipe_slow);
8510 %}
8511
8512 // This pattern is generated automatically from cas.m4.
8513 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8514 instruct compareAndExchangeBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8515 predicate(needs_acquiring_load_exclusive(n));
8516 match(Set res (CompareAndExchangeB mem (Binary oldval newval)));
8517 ins_cost(VOLATILE_REF_COST);
8518 effect(TEMP_DEF res, KILL cr);
8519 format %{
8520 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8521 %}
8522 ins_encode %{
8523 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8524 Assembler::byte, /*acquire*/ true, /*release*/ true,
8525 /*weak*/ false, $res$$Register);
8526 __ sxtbw($res$$Register, $res$$Register);
8527 %}
8528 ins_pipe(pipe_slow);
8529 %}
8530
8531 // This pattern is generated automatically from cas.m4.
8532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8533 instruct compareAndExchangeSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8534 predicate(needs_acquiring_load_exclusive(n));
8535 match(Set res (CompareAndExchangeS mem (Binary oldval newval)));
8536 ins_cost(VOLATILE_REF_COST);
8537 effect(TEMP_DEF res, KILL cr);
8538 format %{
8539 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8540 %}
8541 ins_encode %{
8542 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8543 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8544 /*weak*/ false, $res$$Register);
8545 __ sxthw($res$$Register, $res$$Register);
8546 %}
8547 ins_pipe(pipe_slow);
8548 %}
8549
8550 // This pattern is generated automatically from cas.m4.
8551 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8552 instruct compareAndExchangeIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8553 predicate(needs_acquiring_load_exclusive(n));
8554 match(Set res (CompareAndExchangeI mem (Binary oldval newval)));
8555 ins_cost(VOLATILE_REF_COST);
8556 effect(TEMP_DEF res, KILL cr);
8557 format %{
8558 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8559 %}
8560 ins_encode %{
8561 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8562 Assembler::word, /*acquire*/ true, /*release*/ true,
8563 /*weak*/ false, $res$$Register);
8564 %}
8565 ins_pipe(pipe_slow);
8566 %}
8567
8568 // This pattern is generated automatically from cas.m4.
8569 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8570 instruct compareAndExchangeLAcq(iRegLNoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8571 predicate(needs_acquiring_load_exclusive(n));
8572 match(Set res (CompareAndExchangeL mem (Binary oldval newval)));
8573 ins_cost(VOLATILE_REF_COST);
8574 effect(TEMP_DEF res, KILL cr);
8575 format %{
8576 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8577 %}
8578 ins_encode %{
8579 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8580 Assembler::xword, /*acquire*/ true, /*release*/ true,
8581 /*weak*/ false, $res$$Register);
8582 %}
8583 ins_pipe(pipe_slow);
8584 %}
8585
8586 // This pattern is generated automatically from cas.m4.
8587 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8588 instruct compareAndExchangeNAcq(iRegNNoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8589 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8590 match(Set res (CompareAndExchangeN mem (Binary oldval newval)));
8591 ins_cost(VOLATILE_REF_COST);
8592 effect(TEMP_DEF res, KILL cr);
8593 format %{
8594 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8595 %}
8596 ins_encode %{
8597 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8598 Assembler::word, /*acquire*/ true, /*release*/ true,
8599 /*weak*/ false, $res$$Register);
8600 %}
8601 ins_pipe(pipe_slow);
8602 %}
8603
8604 // This pattern is generated automatically from cas.m4.
8605 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8606 instruct compareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8607 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8608 match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
8609 ins_cost(VOLATILE_REF_COST);
8610 effect(TEMP_DEF res, KILL cr);
8611 format %{
8612 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8613 %}
8614 ins_encode %{
8615 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8616 Assembler::xword, /*acquire*/ true, /*release*/ true,
8617 /*weak*/ false, $res$$Register);
8618 %}
8619 ins_pipe(pipe_slow);
8620 %}
8621
8622 // This pattern is generated automatically from cas.m4.
8623 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8624 instruct weakCompareAndSwapB(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8625 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8626 ins_cost(2 * VOLATILE_REF_COST);
8627 effect(KILL cr);
8628 format %{
8629 "cmpxchgb $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8630 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8631 %}
8632 ins_encode %{
8633 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8634 Assembler::byte, /*acquire*/ false, /*release*/ true,
8635 /*weak*/ true, noreg);
8636 __ csetw($res$$Register, Assembler::EQ);
8637 %}
8638 ins_pipe(pipe_slow);
8639 %}
8640
8641 // This pattern is generated automatically from cas.m4.
8642 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8643 instruct weakCompareAndSwapS(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8644 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8645 ins_cost(2 * VOLATILE_REF_COST);
8646 effect(KILL cr);
8647 format %{
8648 "cmpxchgs $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8649 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8650 %}
8651 ins_encode %{
8652 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8653 Assembler::halfword, /*acquire*/ false, /*release*/ true,
8654 /*weak*/ true, noreg);
8655 __ csetw($res$$Register, Assembler::EQ);
8656 %}
8657 ins_pipe(pipe_slow);
8658 %}
8659
8660 // This pattern is generated automatically from cas.m4.
8661 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8662 instruct weakCompareAndSwapI(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8663 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8664 ins_cost(2 * VOLATILE_REF_COST);
8665 effect(KILL cr);
8666 format %{
8667 "cmpxchgw $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8668 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8669 %}
8670 ins_encode %{
8671 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8672 Assembler::word, /*acquire*/ false, /*release*/ true,
8673 /*weak*/ true, noreg);
8674 __ csetw($res$$Register, Assembler::EQ);
8675 %}
8676 ins_pipe(pipe_slow);
8677 %}
8678
8679 // This pattern is generated automatically from cas.m4.
8680 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8681 instruct weakCompareAndSwapL(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8682 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8683 ins_cost(2 * VOLATILE_REF_COST);
8684 effect(KILL cr);
8685 format %{
8686 "cmpxchg $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8687 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8688 %}
8689 ins_encode %{
8690 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8691 Assembler::xword, /*acquire*/ false, /*release*/ true,
8692 /*weak*/ true, noreg);
8693 __ csetw($res$$Register, Assembler::EQ);
8694 %}
8695 ins_pipe(pipe_slow);
8696 %}
8697
8698 // This pattern is generated automatically from cas.m4.
8699 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8700 instruct weakCompareAndSwapN(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8701 predicate(n->as_LoadStore()->barrier_data() == 0);
8702 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8703 ins_cost(2 * VOLATILE_REF_COST);
8704 effect(KILL cr);
8705 format %{
8706 "cmpxchgw $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8707 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8708 %}
8709 ins_encode %{
8710 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8711 Assembler::word, /*acquire*/ false, /*release*/ true,
8712 /*weak*/ true, noreg);
8713 __ csetw($res$$Register, Assembler::EQ);
8714 %}
8715 ins_pipe(pipe_slow);
8716 %}
8717
8718 // This pattern is generated automatically from cas.m4.
8719 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8720 instruct weakCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8721 predicate(n->as_LoadStore()->barrier_data() == 0);
8722 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8723 ins_cost(2 * VOLATILE_REF_COST);
8724 effect(KILL cr);
8725 format %{
8726 "cmpxchg $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8727 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8728 %}
8729 ins_encode %{
8730 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8731 Assembler::xword, /*acquire*/ false, /*release*/ true,
8732 /*weak*/ true, noreg);
8733 __ csetw($res$$Register, Assembler::EQ);
8734 %}
8735 ins_pipe(pipe_slow);
8736 %}
8737
8738 // This pattern is generated automatically from cas.m4.
8739 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8740 instruct weakCompareAndSwapBAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8741 predicate(needs_acquiring_load_exclusive(n));
8742 match(Set res (WeakCompareAndSwapB mem (Binary oldval newval)));
8743 ins_cost(VOLATILE_REF_COST);
8744 effect(KILL cr);
8745 format %{
8746 "cmpxchgb_acq $res = $mem, $oldval, $newval\t# (byte, weak) if $mem == $oldval then $mem <-- $newval"
8747 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8748 %}
8749 ins_encode %{
8750 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8751 Assembler::byte, /*acquire*/ true, /*release*/ true,
8752 /*weak*/ true, noreg);
8753 __ csetw($res$$Register, Assembler::EQ);
8754 %}
8755 ins_pipe(pipe_slow);
8756 %}
8757
8758 // This pattern is generated automatically from cas.m4.
8759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8760 instruct weakCompareAndSwapSAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8761 predicate(needs_acquiring_load_exclusive(n));
8762 match(Set res (WeakCompareAndSwapS mem (Binary oldval newval)));
8763 ins_cost(VOLATILE_REF_COST);
8764 effect(KILL cr);
8765 format %{
8766 "cmpxchgs_acq $res = $mem, $oldval, $newval\t# (short, weak) if $mem == $oldval then $mem <-- $newval"
8767 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8768 %}
8769 ins_encode %{
8770 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8771 Assembler::halfword, /*acquire*/ true, /*release*/ true,
8772 /*weak*/ true, noreg);
8773 __ csetw($res$$Register, Assembler::EQ);
8774 %}
8775 ins_pipe(pipe_slow);
8776 %}
8777
8778 // This pattern is generated automatically from cas.m4.
8779 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8780 instruct weakCompareAndSwapIAcq(iRegINoSp res, indirect mem, iRegI oldval, iRegI newval, rFlagsReg cr) %{
8781 predicate(needs_acquiring_load_exclusive(n));
8782 match(Set res (WeakCompareAndSwapI mem (Binary oldval newval)));
8783 ins_cost(VOLATILE_REF_COST);
8784 effect(KILL cr);
8785 format %{
8786 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (int, weak) if $mem == $oldval then $mem <-- $newval"
8787 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8788 %}
8789 ins_encode %{
8790 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8791 Assembler::word, /*acquire*/ true, /*release*/ true,
8792 /*weak*/ true, noreg);
8793 __ csetw($res$$Register, Assembler::EQ);
8794 %}
8795 ins_pipe(pipe_slow);
8796 %}
8797
8798 // This pattern is generated automatically from cas.m4.
8799 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8800 instruct weakCompareAndSwapLAcq(iRegINoSp res, indirect mem, iRegL oldval, iRegL newval, rFlagsReg cr) %{
8801 predicate(needs_acquiring_load_exclusive(n));
8802 match(Set res (WeakCompareAndSwapL mem (Binary oldval newval)));
8803 ins_cost(VOLATILE_REF_COST);
8804 effect(KILL cr);
8805 format %{
8806 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (long, weak) if $mem == $oldval then $mem <-- $newval"
8807 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8808 %}
8809 ins_encode %{
8810 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8811 Assembler::xword, /*acquire*/ true, /*release*/ true,
8812 /*weak*/ true, noreg);
8813 __ csetw($res$$Register, Assembler::EQ);
8814 %}
8815 ins_pipe(pipe_slow);
8816 %}
8817
8818 // This pattern is generated automatically from cas.m4.
8819 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8820 instruct weakCompareAndSwapNAcq(iRegINoSp res, indirect mem, iRegN oldval, iRegN newval, rFlagsReg cr) %{
8821 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8822 match(Set res (WeakCompareAndSwapN mem (Binary oldval newval)));
8823 ins_cost(VOLATILE_REF_COST);
8824 effect(KILL cr);
8825 format %{
8826 "cmpxchgw_acq $res = $mem, $oldval, $newval\t# (narrow oop, weak) if $mem == $oldval then $mem <-- $newval"
8827 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8828 %}
8829 ins_encode %{
8830 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8831 Assembler::word, /*acquire*/ true, /*release*/ true,
8832 /*weak*/ true, noreg);
8833 __ csetw($res$$Register, Assembler::EQ);
8834 %}
8835 ins_pipe(pipe_slow);
8836 %}
8837
8838 // This pattern is generated automatically from cas.m4.
8839 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
8840 instruct weakCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
8841 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8842 match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
8843 ins_cost(VOLATILE_REF_COST);
8844 effect(KILL cr);
8845 format %{
8846 "cmpxchg_acq $res = $mem, $oldval, $newval\t# (ptr, weak) if $mem == $oldval then $mem <-- $newval"
8847 "csetw $res, EQ\t# $res <-- (EQ ? 1 : 0)"
8848 %}
8849 ins_encode %{
8850 __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register,
8851 Assembler::xword, /*acquire*/ true, /*release*/ true,
8852 /*weak*/ true, noreg);
8853 __ csetw($res$$Register, Assembler::EQ);
8854 %}
8855 ins_pipe(pipe_slow);
8856 %}
8857
8858 // END This section of the file is automatically generated. Do not edit --------------
8859 // ---------------------------------------------------------------------
8860
8861 instruct get_and_setI(indirect mem, iRegI newv, iRegINoSp prev) %{
8862 match(Set prev (GetAndSetI mem newv));
8863 ins_cost(2 * VOLATILE_REF_COST);
8864 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8865 ins_encode %{
8866 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8867 %}
8868 ins_pipe(pipe_serial);
8869 %}
8870
8871 instruct get_and_setL(indirect mem, iRegL newv, iRegLNoSp prev) %{
8872 match(Set prev (GetAndSetL mem newv));
8873 ins_cost(2 * VOLATILE_REF_COST);
8874 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8875 ins_encode %{
8876 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8877 %}
8878 ins_pipe(pipe_serial);
8879 %}
8880
8881 instruct get_and_setN(indirect mem, iRegN newv, iRegINoSp prev) %{
8882 predicate(n->as_LoadStore()->barrier_data() == 0);
8883 match(Set prev (GetAndSetN mem newv));
8884 ins_cost(2 * VOLATILE_REF_COST);
8885 format %{ "atomic_xchgw $prev, $newv, [$mem]" %}
8886 ins_encode %{
8887 __ atomic_xchgw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8888 %}
8889 ins_pipe(pipe_serial);
8890 %}
8891
8892 instruct get_and_setP(indirect mem, iRegP newv, iRegPNoSp prev) %{
8893 predicate(n->as_LoadStore()->barrier_data() == 0);
8894 match(Set prev (GetAndSetP mem newv));
8895 ins_cost(2 * VOLATILE_REF_COST);
8896 format %{ "atomic_xchg $prev, $newv, [$mem]" %}
8897 ins_encode %{
8898 __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
8899 %}
8900 ins_pipe(pipe_serial);
8901 %}
8902
8903 instruct get_and_setIAcq(indirect mem, iRegI newv, iRegINoSp prev) %{
8904 predicate(needs_acquiring_load_exclusive(n));
8905 match(Set prev (GetAndSetI mem newv));
8906 ins_cost(VOLATILE_REF_COST);
8907 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8908 ins_encode %{
8909 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8910 %}
8911 ins_pipe(pipe_serial);
8912 %}
8913
8914 instruct get_and_setLAcq(indirect mem, iRegL newv, iRegLNoSp prev) %{
8915 predicate(needs_acquiring_load_exclusive(n));
8916 match(Set prev (GetAndSetL mem newv));
8917 ins_cost(VOLATILE_REF_COST);
8918 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8919 ins_encode %{
8920 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8921 %}
8922 ins_pipe(pipe_serial);
8923 %}
8924
8925 instruct get_and_setNAcq(indirect mem, iRegN newv, iRegINoSp prev) %{
8926 predicate(needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == 0);
8927 match(Set prev (GetAndSetN mem newv));
8928 ins_cost(VOLATILE_REF_COST);
8929 format %{ "atomic_xchgw_acq $prev, $newv, [$mem]" %}
8930 ins_encode %{
8931 __ atomic_xchgalw($prev$$Register, $newv$$Register, as_Register($mem$$base));
8932 %}
8933 ins_pipe(pipe_serial);
8934 %}
8935
8936 instruct get_and_setPAcq(indirect mem, iRegP newv, iRegPNoSp prev) %{
8937 predicate(needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == 0));
8938 match(Set prev (GetAndSetP mem newv));
8939 ins_cost(VOLATILE_REF_COST);
8940 format %{ "atomic_xchg_acq $prev, $newv, [$mem]" %}
8941 ins_encode %{
8942 __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
8943 %}
8944 ins_pipe(pipe_serial);
8945 %}
8946
8947
8948 instruct get_and_addL(indirect mem, iRegLNoSp newval, iRegL incr) %{
8949 match(Set newval (GetAndAddL mem incr));
8950 ins_cost(2 * VOLATILE_REF_COST + 1);
8951 format %{ "get_and_addL $newval, [$mem], $incr" %}
8952 ins_encode %{
8953 __ atomic_add($newval$$Register, $incr$$Register, as_Register($mem$$base));
8954 %}
8955 ins_pipe(pipe_serial);
8956 %}
8957
8958 instruct get_and_addL_no_res(indirect mem, Universe dummy, iRegL incr) %{
8959 predicate(n->as_LoadStore()->result_not_used());
8960 match(Set dummy (GetAndAddL mem incr));
8961 ins_cost(2 * VOLATILE_REF_COST);
8962 format %{ "get_and_addL [$mem], $incr" %}
8963 ins_encode %{
8964 __ atomic_add(noreg, $incr$$Register, as_Register($mem$$base));
8965 %}
8966 ins_pipe(pipe_serial);
8967 %}
8968
8969 instruct get_and_addLi(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
8970 match(Set newval (GetAndAddL mem incr));
8971 ins_cost(2 * VOLATILE_REF_COST + 1);
8972 format %{ "get_and_addL $newval, [$mem], $incr" %}
8973 ins_encode %{
8974 __ atomic_add($newval$$Register, $incr$$constant, as_Register($mem$$base));
8975 %}
8976 ins_pipe(pipe_serial);
8977 %}
8978
8979 instruct get_and_addLi_no_res(indirect mem, Universe dummy, immLAddSub incr) %{
8980 predicate(n->as_LoadStore()->result_not_used());
8981 match(Set dummy (GetAndAddL mem incr));
8982 ins_cost(2 * VOLATILE_REF_COST);
8983 format %{ "get_and_addL [$mem], $incr" %}
8984 ins_encode %{
8985 __ atomic_add(noreg, $incr$$constant, as_Register($mem$$base));
8986 %}
8987 ins_pipe(pipe_serial);
8988 %}
8989
8990 instruct get_and_addI(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
8991 match(Set newval (GetAndAddI mem incr));
8992 ins_cost(2 * VOLATILE_REF_COST + 1);
8993 format %{ "get_and_addI $newval, [$mem], $incr" %}
8994 ins_encode %{
8995 __ atomic_addw($newval$$Register, $incr$$Register, as_Register($mem$$base));
8996 %}
8997 ins_pipe(pipe_serial);
8998 %}
8999
9000 instruct get_and_addI_no_res(indirect mem, Universe dummy, iRegIorL2I incr) %{
9001 predicate(n->as_LoadStore()->result_not_used());
9002 match(Set dummy (GetAndAddI mem incr));
9003 ins_cost(2 * VOLATILE_REF_COST);
9004 format %{ "get_and_addI [$mem], $incr" %}
9005 ins_encode %{
9006 __ atomic_addw(noreg, $incr$$Register, as_Register($mem$$base));
9007 %}
9008 ins_pipe(pipe_serial);
9009 %}
9010
9011 instruct get_and_addIi(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9012 match(Set newval (GetAndAddI mem incr));
9013 ins_cost(2 * VOLATILE_REF_COST + 1);
9014 format %{ "get_and_addI $newval, [$mem], $incr" %}
9015 ins_encode %{
9016 __ atomic_addw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9017 %}
9018 ins_pipe(pipe_serial);
9019 %}
9020
9021 instruct get_and_addIi_no_res(indirect mem, Universe dummy, immIAddSub incr) %{
9022 predicate(n->as_LoadStore()->result_not_used());
9023 match(Set dummy (GetAndAddI mem incr));
9024 ins_cost(2 * VOLATILE_REF_COST);
9025 format %{ "get_and_addI [$mem], $incr" %}
9026 ins_encode %{
9027 __ atomic_addw(noreg, $incr$$constant, as_Register($mem$$base));
9028 %}
9029 ins_pipe(pipe_serial);
9030 %}
9031
9032 instruct get_and_addLAcq(indirect mem, iRegLNoSp newval, iRegL incr) %{
9033 predicate(needs_acquiring_load_exclusive(n));
9034 match(Set newval (GetAndAddL mem incr));
9035 ins_cost(VOLATILE_REF_COST + 1);
9036 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9037 ins_encode %{
9038 __ atomic_addal($newval$$Register, $incr$$Register, as_Register($mem$$base));
9039 %}
9040 ins_pipe(pipe_serial);
9041 %}
9042
9043 instruct get_and_addL_no_resAcq(indirect mem, Universe dummy, iRegL incr) %{
9044 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9045 match(Set dummy (GetAndAddL mem incr));
9046 ins_cost(VOLATILE_REF_COST);
9047 format %{ "get_and_addL_acq [$mem], $incr" %}
9048 ins_encode %{
9049 __ atomic_addal(noreg, $incr$$Register, as_Register($mem$$base));
9050 %}
9051 ins_pipe(pipe_serial);
9052 %}
9053
9054 instruct get_and_addLiAcq(indirect mem, iRegLNoSp newval, immLAddSub incr) %{
9055 predicate(needs_acquiring_load_exclusive(n));
9056 match(Set newval (GetAndAddL mem incr));
9057 ins_cost(VOLATILE_REF_COST + 1);
9058 format %{ "get_and_addL_acq $newval, [$mem], $incr" %}
9059 ins_encode %{
9060 __ atomic_addal($newval$$Register, $incr$$constant, as_Register($mem$$base));
9061 %}
9062 ins_pipe(pipe_serial);
9063 %}
9064
9065 instruct get_and_addLi_no_resAcq(indirect mem, Universe dummy, immLAddSub incr) %{
9066 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9067 match(Set dummy (GetAndAddL mem incr));
9068 ins_cost(VOLATILE_REF_COST);
9069 format %{ "get_and_addL_acq [$mem], $incr" %}
9070 ins_encode %{
9071 __ atomic_addal(noreg, $incr$$constant, as_Register($mem$$base));
9072 %}
9073 ins_pipe(pipe_serial);
9074 %}
9075
9076 instruct get_and_addIAcq(indirect mem, iRegINoSp newval, iRegIorL2I incr) %{
9077 predicate(needs_acquiring_load_exclusive(n));
9078 match(Set newval (GetAndAddI mem incr));
9079 ins_cost(VOLATILE_REF_COST + 1);
9080 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9081 ins_encode %{
9082 __ atomic_addalw($newval$$Register, $incr$$Register, as_Register($mem$$base));
9083 %}
9084 ins_pipe(pipe_serial);
9085 %}
9086
9087 instruct get_and_addI_no_resAcq(indirect mem, Universe dummy, iRegIorL2I incr) %{
9088 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9089 match(Set dummy (GetAndAddI mem incr));
9090 ins_cost(VOLATILE_REF_COST);
9091 format %{ "get_and_addI_acq [$mem], $incr" %}
9092 ins_encode %{
9093 __ atomic_addalw(noreg, $incr$$Register, as_Register($mem$$base));
9094 %}
9095 ins_pipe(pipe_serial);
9096 %}
9097
9098 instruct get_and_addIiAcq(indirect mem, iRegINoSp newval, immIAddSub incr) %{
9099 predicate(needs_acquiring_load_exclusive(n));
9100 match(Set newval (GetAndAddI mem incr));
9101 ins_cost(VOLATILE_REF_COST + 1);
9102 format %{ "get_and_addI_acq $newval, [$mem], $incr" %}
9103 ins_encode %{
9104 __ atomic_addalw($newval$$Register, $incr$$constant, as_Register($mem$$base));
9105 %}
9106 ins_pipe(pipe_serial);
9107 %}
9108
9109 instruct get_and_addIi_no_resAcq(indirect mem, Universe dummy, immIAddSub incr) %{
9110 predicate(n->as_LoadStore()->result_not_used() && needs_acquiring_load_exclusive(n));
9111 match(Set dummy (GetAndAddI mem incr));
9112 ins_cost(VOLATILE_REF_COST);
9113 format %{ "get_and_addI_acq [$mem], $incr" %}
9114 ins_encode %{
9115 __ atomic_addalw(noreg, $incr$$constant, as_Register($mem$$base));
9116 %}
9117 ins_pipe(pipe_serial);
9118 %}
9119
9120 // Manifest a CmpU result in an integer register.
9121 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9122 instruct cmpU3_reg_reg(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg flags)
9123 %{
9124 match(Set dst (CmpU3 src1 src2));
9125 effect(KILL flags);
9126
9127 ins_cost(INSN_COST * 3);
9128 format %{
9129 "cmpw $src1, $src2\n\t"
9130 "csetw $dst, ne\n\t"
9131 "cnegw $dst, lo\t# CmpU3(reg)"
9132 %}
9133 ins_encode %{
9134 __ cmpw($src1$$Register, $src2$$Register);
9135 __ csetw($dst$$Register, Assembler::NE);
9136 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9137 %}
9138
9139 ins_pipe(pipe_class_default);
9140 %}
9141
9142 instruct cmpU3_reg_imm(iRegINoSp dst, iRegI src1, immIAddSub src2, rFlagsReg flags)
9143 %{
9144 match(Set dst (CmpU3 src1 src2));
9145 effect(KILL flags);
9146
9147 ins_cost(INSN_COST * 3);
9148 format %{
9149 "subsw zr, $src1, $src2\n\t"
9150 "csetw $dst, ne\n\t"
9151 "cnegw $dst, lo\t# CmpU3(imm)"
9152 %}
9153 ins_encode %{
9154 __ subsw(zr, $src1$$Register, (int32_t)$src2$$constant);
9155 __ csetw($dst$$Register, Assembler::NE);
9156 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9157 %}
9158
9159 ins_pipe(pipe_class_default);
9160 %}
9161
9162 // Manifest a CmpUL result in an integer register.
9163 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9164 instruct cmpUL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9165 %{
9166 match(Set dst (CmpUL3 src1 src2));
9167 effect(KILL flags);
9168
9169 ins_cost(INSN_COST * 3);
9170 format %{
9171 "cmp $src1, $src2\n\t"
9172 "csetw $dst, ne\n\t"
9173 "cnegw $dst, lo\t# CmpUL3(reg)"
9174 %}
9175 ins_encode %{
9176 __ cmp($src1$$Register, $src2$$Register);
9177 __ csetw($dst$$Register, Assembler::NE);
9178 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9179 %}
9180
9181 ins_pipe(pipe_class_default);
9182 %}
9183
9184 instruct cmpUL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9185 %{
9186 match(Set dst (CmpUL3 src1 src2));
9187 effect(KILL flags);
9188
9189 ins_cost(INSN_COST * 3);
9190 format %{
9191 "subs zr, $src1, $src2\n\t"
9192 "csetw $dst, ne\n\t"
9193 "cnegw $dst, lo\t# CmpUL3(imm)"
9194 %}
9195 ins_encode %{
9196 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9197 __ csetw($dst$$Register, Assembler::NE);
9198 __ cnegw($dst$$Register, $dst$$Register, Assembler::LO);
9199 %}
9200
9201 ins_pipe(pipe_class_default);
9202 %}
9203
9204 // Manifest a CmpL result in an integer register.
9205 // (src1 < src2) ? -1 : ((src1 > src2) ? 1 : 0)
9206 instruct cmpL3_reg_reg(iRegINoSp dst, iRegL src1, iRegL src2, rFlagsReg flags)
9207 %{
9208 match(Set dst (CmpL3 src1 src2));
9209 effect(KILL flags);
9210
9211 ins_cost(INSN_COST * 3);
9212 format %{
9213 "cmp $src1, $src2\n\t"
9214 "csetw $dst, ne\n\t"
9215 "cnegw $dst, lt\t# CmpL3(reg)"
9216 %}
9217 ins_encode %{
9218 __ cmp($src1$$Register, $src2$$Register);
9219 __ csetw($dst$$Register, Assembler::NE);
9220 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9221 %}
9222
9223 ins_pipe(pipe_class_default);
9224 %}
9225
9226 instruct cmpL3_reg_imm(iRegINoSp dst, iRegL src1, immLAddSub src2, rFlagsReg flags)
9227 %{
9228 match(Set dst (CmpL3 src1 src2));
9229 effect(KILL flags);
9230
9231 ins_cost(INSN_COST * 3);
9232 format %{
9233 "subs zr, $src1, $src2\n\t"
9234 "csetw $dst, ne\n\t"
9235 "cnegw $dst, lt\t# CmpL3(imm)"
9236 %}
9237 ins_encode %{
9238 __ subs(zr, $src1$$Register, (int32_t)$src2$$constant);
9239 __ csetw($dst$$Register, Assembler::NE);
9240 __ cnegw($dst$$Register, $dst$$Register, Assembler::LT);
9241 %}
9242
9243 ins_pipe(pipe_class_default);
9244 %}
9245
9246 // ============================================================================
9247 // Conditional Move Instructions
9248
9249 // n.b. we have identical rules for both a signed compare op (cmpOp)
9250 // and an unsigned compare op (cmpOpU). it would be nice if we could
9251 // define an op class which merged both inputs and use it to type the
9252 // argument to a single rule. unfortunatelyt his fails because the
9253 // opclass does not live up to the COND_INTER interface of its
9254 // component operands. When the generic code tries to negate the
9255 // operand it ends up running the generci Machoper::negate method
9256 // which throws a ShouldNotHappen. So, we have to provide two flavours
9257 // of each rule, one for a cmpOp and a second for a cmpOpU (sigh).
9258
9259 instruct cmovI_reg_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9260 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9261
9262 ins_cost(INSN_COST * 2);
9263 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, int" %}
9264
9265 ins_encode %{
9266 __ cselw(as_Register($dst$$reg),
9267 as_Register($src2$$reg),
9268 as_Register($src1$$reg),
9269 (Assembler::Condition)$cmp$$cmpcode);
9270 %}
9271
9272 ins_pipe(icond_reg_reg);
9273 %}
9274
9275 instruct cmovUI_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9276 match(Set dst (CMoveI (Binary cmp cr) (Binary src1 src2)));
9277
9278 ins_cost(INSN_COST * 2);
9279 format %{ "cselw $dst, $src2, $src1 $cmp\t# unsigned, int" %}
9280
9281 ins_encode %{
9282 __ cselw(as_Register($dst$$reg),
9283 as_Register($src2$$reg),
9284 as_Register($src1$$reg),
9285 (Assembler::Condition)$cmp$$cmpcode);
9286 %}
9287
9288 ins_pipe(icond_reg_reg);
9289 %}
9290
9291 // special cases where one arg is zero
9292
9293 // n.b. this is selected in preference to the rule above because it
9294 // avoids loading constant 0 into a source register
9295
9296 // TODO
9297 // we ought only to be able to cull one of these variants as the ideal
9298 // transforms ought always to order the zero consistently (to left/right?)
9299
9300 instruct cmovI_zero_reg(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9301 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9302
9303 ins_cost(INSN_COST * 2);
9304 format %{ "cselw $dst, $src, zr $cmp\t# signed, int" %}
9305
9306 ins_encode %{
9307 __ cselw(as_Register($dst$$reg),
9308 as_Register($src$$reg),
9309 zr,
9310 (Assembler::Condition)$cmp$$cmpcode);
9311 %}
9312
9313 ins_pipe(icond_reg);
9314 %}
9315
9316 instruct cmovUI_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, iRegIorL2I src) %{
9317 match(Set dst (CMoveI (Binary cmp cr) (Binary zero src)));
9318
9319 ins_cost(INSN_COST * 2);
9320 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, int" %}
9321
9322 ins_encode %{
9323 __ cselw(as_Register($dst$$reg),
9324 as_Register($src$$reg),
9325 zr,
9326 (Assembler::Condition)$cmp$$cmpcode);
9327 %}
9328
9329 ins_pipe(icond_reg);
9330 %}
9331
9332 instruct cmovI_reg_zero(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9333 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9334
9335 ins_cost(INSN_COST * 2);
9336 format %{ "cselw $dst, zr, $src $cmp\t# signed, int" %}
9337
9338 ins_encode %{
9339 __ cselw(as_Register($dst$$reg),
9340 zr,
9341 as_Register($src$$reg),
9342 (Assembler::Condition)$cmp$$cmpcode);
9343 %}
9344
9345 ins_pipe(icond_reg);
9346 %}
9347
9348 instruct cmovUI_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, iRegIorL2I src, immI0 zero) %{
9349 match(Set dst (CMoveI (Binary cmp cr) (Binary src zero)));
9350
9351 ins_cost(INSN_COST * 2);
9352 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, int" %}
9353
9354 ins_encode %{
9355 __ cselw(as_Register($dst$$reg),
9356 zr,
9357 as_Register($src$$reg),
9358 (Assembler::Condition)$cmp$$cmpcode);
9359 %}
9360
9361 ins_pipe(icond_reg);
9362 %}
9363
9364 // special case for creating a boolean 0 or 1
9365
9366 // n.b. this is selected in preference to the rule above because it
9367 // avoids loading constants 0 and 1 into a source register
9368
9369 instruct cmovI_reg_zero_one(cmpOp cmp, rFlagsReg cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9370 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9371
9372 ins_cost(INSN_COST * 2);
9373 format %{ "csincw $dst, zr, zr $cmp\t# signed, int" %}
9374
9375 ins_encode %{
9376 // equivalently
9377 // cset(as_Register($dst$$reg),
9378 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9379 __ csincw(as_Register($dst$$reg),
9380 zr,
9381 zr,
9382 (Assembler::Condition)$cmp$$cmpcode);
9383 %}
9384
9385 ins_pipe(icond_none);
9386 %}
9387
9388 instruct cmovUI_reg_zero_one(cmpOpU cmp, rFlagsRegU cr, iRegINoSp dst, immI0 zero, immI_1 one) %{
9389 match(Set dst (CMoveI (Binary cmp cr) (Binary one zero)));
9390
9391 ins_cost(INSN_COST * 2);
9392 format %{ "csincw $dst, zr, zr $cmp\t# unsigned, int" %}
9393
9394 ins_encode %{
9395 // equivalently
9396 // cset(as_Register($dst$$reg),
9397 // negate_condition((Assembler::Condition)$cmp$$cmpcode));
9398 __ csincw(as_Register($dst$$reg),
9399 zr,
9400 zr,
9401 (Assembler::Condition)$cmp$$cmpcode);
9402 %}
9403
9404 ins_pipe(icond_none);
9405 %}
9406
9407 instruct cmovL_reg_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9408 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9409
9410 ins_cost(INSN_COST * 2);
9411 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, long" %}
9412
9413 ins_encode %{
9414 __ csel(as_Register($dst$$reg),
9415 as_Register($src2$$reg),
9416 as_Register($src1$$reg),
9417 (Assembler::Condition)$cmp$$cmpcode);
9418 %}
9419
9420 ins_pipe(icond_reg_reg);
9421 %}
9422
9423 instruct cmovUL_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src1, iRegL src2) %{
9424 match(Set dst (CMoveL (Binary cmp cr) (Binary src1 src2)));
9425
9426 ins_cost(INSN_COST * 2);
9427 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, long" %}
9428
9429 ins_encode %{
9430 __ csel(as_Register($dst$$reg),
9431 as_Register($src2$$reg),
9432 as_Register($src1$$reg),
9433 (Assembler::Condition)$cmp$$cmpcode);
9434 %}
9435
9436 ins_pipe(icond_reg_reg);
9437 %}
9438
9439 // special cases where one arg is zero
9440
9441 instruct cmovL_reg_zero(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9442 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9443
9444 ins_cost(INSN_COST * 2);
9445 format %{ "csel $dst, zr, $src $cmp\t# signed, long" %}
9446
9447 ins_encode %{
9448 __ csel(as_Register($dst$$reg),
9449 zr,
9450 as_Register($src$$reg),
9451 (Assembler::Condition)$cmp$$cmpcode);
9452 %}
9453
9454 ins_pipe(icond_reg);
9455 %}
9456
9457 instruct cmovUL_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, iRegL src, immL0 zero) %{
9458 match(Set dst (CMoveL (Binary cmp cr) (Binary src zero)));
9459
9460 ins_cost(INSN_COST * 2);
9461 format %{ "csel $dst, zr, $src $cmp\t# unsigned, long" %}
9462
9463 ins_encode %{
9464 __ csel(as_Register($dst$$reg),
9465 zr,
9466 as_Register($src$$reg),
9467 (Assembler::Condition)$cmp$$cmpcode);
9468 %}
9469
9470 ins_pipe(icond_reg);
9471 %}
9472
9473 instruct cmovL_zero_reg(cmpOp cmp, rFlagsReg cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9474 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9475
9476 ins_cost(INSN_COST * 2);
9477 format %{ "csel $dst, $src, zr $cmp\t# signed, long" %}
9478
9479 ins_encode %{
9480 __ csel(as_Register($dst$$reg),
9481 as_Register($src$$reg),
9482 zr,
9483 (Assembler::Condition)$cmp$$cmpcode);
9484 %}
9485
9486 ins_pipe(icond_reg);
9487 %}
9488
9489 instruct cmovUL_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegLNoSp dst, immL0 zero, iRegL src) %{
9490 match(Set dst (CMoveL (Binary cmp cr) (Binary zero src)));
9491
9492 ins_cost(INSN_COST * 2);
9493 format %{ "csel $dst, $src, zr $cmp\t# unsigned, long" %}
9494
9495 ins_encode %{
9496 __ csel(as_Register($dst$$reg),
9497 as_Register($src$$reg),
9498 zr,
9499 (Assembler::Condition)$cmp$$cmpcode);
9500 %}
9501
9502 ins_pipe(icond_reg);
9503 %}
9504
9505 instruct cmovP_reg_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9506 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9507
9508 ins_cost(INSN_COST * 2);
9509 format %{ "csel $dst, $src2, $src1 $cmp\t# signed, ptr" %}
9510
9511 ins_encode %{
9512 __ csel(as_Register($dst$$reg),
9513 as_Register($src2$$reg),
9514 as_Register($src1$$reg),
9515 (Assembler::Condition)$cmp$$cmpcode);
9516 %}
9517
9518 ins_pipe(icond_reg_reg);
9519 %}
9520
9521 instruct cmovUP_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src1, iRegP src2) %{
9522 match(Set dst (CMoveP (Binary cmp cr) (Binary src1 src2)));
9523
9524 ins_cost(INSN_COST * 2);
9525 format %{ "csel $dst, $src2, $src1 $cmp\t# unsigned, ptr" %}
9526
9527 ins_encode %{
9528 __ csel(as_Register($dst$$reg),
9529 as_Register($src2$$reg),
9530 as_Register($src1$$reg),
9531 (Assembler::Condition)$cmp$$cmpcode);
9532 %}
9533
9534 ins_pipe(icond_reg_reg);
9535 %}
9536
9537 // special cases where one arg is zero
9538
9539 instruct cmovP_reg_zero(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9540 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9541
9542 ins_cost(INSN_COST * 2);
9543 format %{ "csel $dst, zr, $src $cmp\t# signed, ptr" %}
9544
9545 ins_encode %{
9546 __ csel(as_Register($dst$$reg),
9547 zr,
9548 as_Register($src$$reg),
9549 (Assembler::Condition)$cmp$$cmpcode);
9550 %}
9551
9552 ins_pipe(icond_reg);
9553 %}
9554
9555 instruct cmovUP_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, iRegP src, immP0 zero) %{
9556 match(Set dst (CMoveP (Binary cmp cr) (Binary src zero)));
9557
9558 ins_cost(INSN_COST * 2);
9559 format %{ "csel $dst, zr, $src $cmp\t# unsigned, ptr" %}
9560
9561 ins_encode %{
9562 __ csel(as_Register($dst$$reg),
9563 zr,
9564 as_Register($src$$reg),
9565 (Assembler::Condition)$cmp$$cmpcode);
9566 %}
9567
9568 ins_pipe(icond_reg);
9569 %}
9570
9571 instruct cmovP_zero_reg(cmpOp cmp, rFlagsReg cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9572 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9573
9574 ins_cost(INSN_COST * 2);
9575 format %{ "csel $dst, $src, zr $cmp\t# signed, ptr" %}
9576
9577 ins_encode %{
9578 __ csel(as_Register($dst$$reg),
9579 as_Register($src$$reg),
9580 zr,
9581 (Assembler::Condition)$cmp$$cmpcode);
9582 %}
9583
9584 ins_pipe(icond_reg);
9585 %}
9586
9587 instruct cmovUP_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegPNoSp dst, immP0 zero, iRegP src) %{
9588 match(Set dst (CMoveP (Binary cmp cr) (Binary zero src)));
9589
9590 ins_cost(INSN_COST * 2);
9591 format %{ "csel $dst, $src, zr $cmp\t# unsigned, ptr" %}
9592
9593 ins_encode %{
9594 __ csel(as_Register($dst$$reg),
9595 as_Register($src$$reg),
9596 zr,
9597 (Assembler::Condition)$cmp$$cmpcode);
9598 %}
9599
9600 ins_pipe(icond_reg);
9601 %}
9602
9603 instruct cmovN_reg_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9604 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9605
9606 ins_cost(INSN_COST * 2);
9607 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9608
9609 ins_encode %{
9610 __ cselw(as_Register($dst$$reg),
9611 as_Register($src2$$reg),
9612 as_Register($src1$$reg),
9613 (Assembler::Condition)$cmp$$cmpcode);
9614 %}
9615
9616 ins_pipe(icond_reg_reg);
9617 %}
9618
9619 instruct cmovUN_reg_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src1, iRegN src2) %{
9620 match(Set dst (CMoveN (Binary cmp cr) (Binary src1 src2)));
9621
9622 ins_cost(INSN_COST * 2);
9623 format %{ "cselw $dst, $src2, $src1 $cmp\t# signed, compressed ptr" %}
9624
9625 ins_encode %{
9626 __ cselw(as_Register($dst$$reg),
9627 as_Register($src2$$reg),
9628 as_Register($src1$$reg),
9629 (Assembler::Condition)$cmp$$cmpcode);
9630 %}
9631
9632 ins_pipe(icond_reg_reg);
9633 %}
9634
9635 // special cases where one arg is zero
9636
9637 instruct cmovN_reg_zero(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9638 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9639
9640 ins_cost(INSN_COST * 2);
9641 format %{ "cselw $dst, zr, $src $cmp\t# signed, compressed ptr" %}
9642
9643 ins_encode %{
9644 __ cselw(as_Register($dst$$reg),
9645 zr,
9646 as_Register($src$$reg),
9647 (Assembler::Condition)$cmp$$cmpcode);
9648 %}
9649
9650 ins_pipe(icond_reg);
9651 %}
9652
9653 instruct cmovUN_reg_zero(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, iRegN src, immN0 zero) %{
9654 match(Set dst (CMoveN (Binary cmp cr) (Binary src zero)));
9655
9656 ins_cost(INSN_COST * 2);
9657 format %{ "cselw $dst, zr, $src $cmp\t# unsigned, compressed ptr" %}
9658
9659 ins_encode %{
9660 __ cselw(as_Register($dst$$reg),
9661 zr,
9662 as_Register($src$$reg),
9663 (Assembler::Condition)$cmp$$cmpcode);
9664 %}
9665
9666 ins_pipe(icond_reg);
9667 %}
9668
9669 instruct cmovN_zero_reg(cmpOp cmp, rFlagsReg cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9670 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9671
9672 ins_cost(INSN_COST * 2);
9673 format %{ "cselw $dst, $src, zr $cmp\t# signed, compressed ptr" %}
9674
9675 ins_encode %{
9676 __ cselw(as_Register($dst$$reg),
9677 as_Register($src$$reg),
9678 zr,
9679 (Assembler::Condition)$cmp$$cmpcode);
9680 %}
9681
9682 ins_pipe(icond_reg);
9683 %}
9684
9685 instruct cmovUN_zero_reg(cmpOpU cmp, rFlagsRegU cr, iRegNNoSp dst, immN0 zero, iRegN src) %{
9686 match(Set dst (CMoveN (Binary cmp cr) (Binary zero src)));
9687
9688 ins_cost(INSN_COST * 2);
9689 format %{ "cselw $dst, $src, zr $cmp\t# unsigned, compressed ptr" %}
9690
9691 ins_encode %{
9692 __ cselw(as_Register($dst$$reg),
9693 as_Register($src$$reg),
9694 zr,
9695 (Assembler::Condition)$cmp$$cmpcode);
9696 %}
9697
9698 ins_pipe(icond_reg);
9699 %}
9700
9701 instruct cmovF_reg(cmpOp cmp, rFlagsReg cr, vRegF dst, vRegF src1, vRegF src2)
9702 %{
9703 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9704
9705 ins_cost(INSN_COST * 3);
9706
9707 format %{ "fcsels $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9708 ins_encode %{
9709 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9710 __ fcsels(as_FloatRegister($dst$$reg),
9711 as_FloatRegister($src2$$reg),
9712 as_FloatRegister($src1$$reg),
9713 cond);
9714 %}
9715
9716 ins_pipe(fp_cond_reg_reg_s);
9717 %}
9718
9719 instruct cmovUF_reg(cmpOpU cmp, rFlagsRegU cr, vRegF dst, vRegF src1, vRegF src2)
9720 %{
9721 match(Set dst (CMoveF (Binary cmp cr) (Binary src1 src2)));
9722
9723 ins_cost(INSN_COST * 3);
9724
9725 format %{ "fcsels $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9726 ins_encode %{
9727 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9728 __ fcsels(as_FloatRegister($dst$$reg),
9729 as_FloatRegister($src2$$reg),
9730 as_FloatRegister($src1$$reg),
9731 cond);
9732 %}
9733
9734 ins_pipe(fp_cond_reg_reg_s);
9735 %}
9736
9737 instruct cmovD_reg(cmpOp cmp, rFlagsReg cr, vRegD dst, vRegD src1, vRegD src2)
9738 %{
9739 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9740
9741 ins_cost(INSN_COST * 3);
9742
9743 format %{ "fcseld $dst, $src1, $src2, $cmp\t# signed cmove float\n\t" %}
9744 ins_encode %{
9745 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9746 __ fcseld(as_FloatRegister($dst$$reg),
9747 as_FloatRegister($src2$$reg),
9748 as_FloatRegister($src1$$reg),
9749 cond);
9750 %}
9751
9752 ins_pipe(fp_cond_reg_reg_d);
9753 %}
9754
9755 instruct cmovUD_reg(cmpOpU cmp, rFlagsRegU cr, vRegD dst, vRegD src1, vRegD src2)
9756 %{
9757 match(Set dst (CMoveD (Binary cmp cr) (Binary src1 src2)));
9758
9759 ins_cost(INSN_COST * 3);
9760
9761 format %{ "fcseld $dst, $src1, $src2, $cmp\t# unsigned cmove float\n\t" %}
9762 ins_encode %{
9763 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
9764 __ fcseld(as_FloatRegister($dst$$reg),
9765 as_FloatRegister($src2$$reg),
9766 as_FloatRegister($src1$$reg),
9767 cond);
9768 %}
9769
9770 ins_pipe(fp_cond_reg_reg_d);
9771 %}
9772
9773 // ============================================================================
9774 // Arithmetic Instructions
9775 //
9776
9777 // Integer Addition
9778
9779 // TODO
9780 // these currently employ operations which do not set CR and hence are
9781 // not flagged as killing CR but we would like to isolate the cases
9782 // where we want to set flags from those where we don't. need to work
9783 // out how to do that.
9784
9785 instruct addI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9786 match(Set dst (AddI src1 src2));
9787
9788 ins_cost(INSN_COST);
9789 format %{ "addw $dst, $src1, $src2" %}
9790
9791 ins_encode %{
9792 __ addw(as_Register($dst$$reg),
9793 as_Register($src1$$reg),
9794 as_Register($src2$$reg));
9795 %}
9796
9797 ins_pipe(ialu_reg_reg);
9798 %}
9799
9800 instruct addI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9801 match(Set dst (AddI src1 src2));
9802
9803 ins_cost(INSN_COST);
9804 format %{ "addw $dst, $src1, $src2" %}
9805
9806 // use opcode to indicate that this is an add not a sub
9807 opcode(0x0);
9808
9809 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9810
9811 ins_pipe(ialu_reg_imm);
9812 %}
9813
9814 instruct addI_reg_imm_i2l(iRegINoSp dst, iRegL src1, immIAddSub src2) %{
9815 match(Set dst (AddI (ConvL2I src1) src2));
9816
9817 ins_cost(INSN_COST);
9818 format %{ "addw $dst, $src1, $src2" %}
9819
9820 // use opcode to indicate that this is an add not a sub
9821 opcode(0x0);
9822
9823 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9824
9825 ins_pipe(ialu_reg_imm);
9826 %}
9827
9828 // Pointer Addition
9829 instruct addP_reg_reg(iRegPNoSp dst, iRegPorL2P src1, iRegL src2) %{
9830 match(Set dst (AddP src1 src2));
9831
9832 ins_cost(INSN_COST);
9833 format %{ "add $dst, $src1, $src2\t# ptr" %}
9834
9835 ins_encode %{
9836 __ add(as_Register($dst$$reg),
9837 as_Register($src1$$reg),
9838 as_Register($src2$$reg));
9839 %}
9840
9841 ins_pipe(ialu_reg_reg);
9842 %}
9843
9844 instruct addP_reg_reg_ext(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2) %{
9845 match(Set dst (AddP src1 (ConvI2L src2)));
9846
9847 ins_cost(1.9 * INSN_COST);
9848 format %{ "add $dst, $src1, $src2, sxtw\t# ptr" %}
9849
9850 ins_encode %{
9851 __ add(as_Register($dst$$reg),
9852 as_Register($src1$$reg),
9853 as_Register($src2$$reg), ext::sxtw);
9854 %}
9855
9856 ins_pipe(ialu_reg_reg);
9857 %}
9858
9859 instruct addP_reg_reg_lsl(iRegPNoSp dst, iRegPorL2P src1, iRegL src2, immIScale scale) %{
9860 match(Set dst (AddP src1 (LShiftL src2 scale)));
9861
9862 ins_cost(1.9 * INSN_COST);
9863 format %{ "add $dst, $src1, $src2, LShiftL $scale\t# ptr" %}
9864
9865 ins_encode %{
9866 __ lea(as_Register($dst$$reg),
9867 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9868 Address::lsl($scale$$constant)));
9869 %}
9870
9871 ins_pipe(ialu_reg_reg_shift);
9872 %}
9873
9874 instruct addP_reg_reg_ext_shift(iRegPNoSp dst, iRegPorL2P src1, iRegIorL2I src2, immIScale scale) %{
9875 match(Set dst (AddP src1 (LShiftL (ConvI2L src2) scale)));
9876
9877 ins_cost(1.9 * INSN_COST);
9878 format %{ "add $dst, $src1, $src2, I2L $scale\t# ptr" %}
9879
9880 ins_encode %{
9881 __ lea(as_Register($dst$$reg),
9882 Address(as_Register($src1$$reg), as_Register($src2$$reg),
9883 Address::sxtw($scale$$constant)));
9884 %}
9885
9886 ins_pipe(ialu_reg_reg_shift);
9887 %}
9888
9889 instruct lshift_ext(iRegLNoSp dst, iRegIorL2I src, immI scale, rFlagsReg cr) %{
9890 match(Set dst (LShiftL (ConvI2L src) scale));
9891
9892 ins_cost(INSN_COST);
9893 format %{ "sbfiz $dst, $src, $scale & 63, -$scale & 63\t" %}
9894
9895 ins_encode %{
9896 __ sbfiz(as_Register($dst$$reg),
9897 as_Register($src$$reg),
9898 $scale$$constant & 63, MIN2(32, (int)((-$scale$$constant) & 63)));
9899 %}
9900
9901 ins_pipe(ialu_reg_shift);
9902 %}
9903
9904 // Pointer Immediate Addition
9905 // n.b. this needs to be more expensive than using an indirect memory
9906 // operand
9907 instruct addP_reg_imm(iRegPNoSp dst, iRegPorL2P src1, immLAddSub src2) %{
9908 match(Set dst (AddP src1 src2));
9909
9910 ins_cost(INSN_COST);
9911 format %{ "add $dst, $src1, $src2\t# ptr" %}
9912
9913 // use opcode to indicate that this is an add not a sub
9914 opcode(0x0);
9915
9916 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9917
9918 ins_pipe(ialu_reg_imm);
9919 %}
9920
9921 // Long Addition
9922 instruct addL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9923
9924 match(Set dst (AddL src1 src2));
9925
9926 ins_cost(INSN_COST);
9927 format %{ "add $dst, $src1, $src2" %}
9928
9929 ins_encode %{
9930 __ add(as_Register($dst$$reg),
9931 as_Register($src1$$reg),
9932 as_Register($src2$$reg));
9933 %}
9934
9935 ins_pipe(ialu_reg_reg);
9936 %}
9937
9938 // No constant pool entries requiredLong Immediate Addition.
9939 instruct addL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
9940 match(Set dst (AddL src1 src2));
9941
9942 ins_cost(INSN_COST);
9943 format %{ "add $dst, $src1, $src2" %}
9944
9945 // use opcode to indicate that this is an add not a sub
9946 opcode(0x0);
9947
9948 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
9949
9950 ins_pipe(ialu_reg_imm);
9951 %}
9952
9953 // Integer Subtraction
9954 instruct subI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
9955 match(Set dst (SubI src1 src2));
9956
9957 ins_cost(INSN_COST);
9958 format %{ "subw $dst, $src1, $src2" %}
9959
9960 ins_encode %{
9961 __ subw(as_Register($dst$$reg),
9962 as_Register($src1$$reg),
9963 as_Register($src2$$reg));
9964 %}
9965
9966 ins_pipe(ialu_reg_reg);
9967 %}
9968
9969 // Immediate Subtraction
9970 instruct subI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immIAddSub src2) %{
9971 match(Set dst (SubI src1 src2));
9972
9973 ins_cost(INSN_COST);
9974 format %{ "subw $dst, $src1, $src2" %}
9975
9976 // use opcode to indicate that this is a sub not an add
9977 opcode(0x1);
9978
9979 ins_encode(aarch64_enc_addsubw_imm(dst, src1, src2));
9980
9981 ins_pipe(ialu_reg_imm);
9982 %}
9983
9984 // Long Subtraction
9985 instruct subL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
9986
9987 match(Set dst (SubL src1 src2));
9988
9989 ins_cost(INSN_COST);
9990 format %{ "sub $dst, $src1, $src2" %}
9991
9992 ins_encode %{
9993 __ sub(as_Register($dst$$reg),
9994 as_Register($src1$$reg),
9995 as_Register($src2$$reg));
9996 %}
9997
9998 ins_pipe(ialu_reg_reg);
9999 %}
10000
10001 // No constant pool entries requiredLong Immediate Subtraction.
10002 instruct subL_reg_imm(iRegLNoSp dst, iRegL src1, immLAddSub src2) %{
10003 match(Set dst (SubL src1 src2));
10004
10005 ins_cost(INSN_COST);
10006 format %{ "sub$dst, $src1, $src2" %}
10007
10008 // use opcode to indicate that this is a sub not an add
10009 opcode(0x1);
10010
10011 ins_encode( aarch64_enc_addsub_imm(dst, src1, src2) );
10012
10013 ins_pipe(ialu_reg_imm);
10014 %}
10015
10016 // Integer Negation (special case for sub)
10017
10018 instruct negI_reg(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr) %{
10019 match(Set dst (SubI zero src));
10020
10021 ins_cost(INSN_COST);
10022 format %{ "negw $dst, $src\t# int" %}
10023
10024 ins_encode %{
10025 __ negw(as_Register($dst$$reg),
10026 as_Register($src$$reg));
10027 %}
10028
10029 ins_pipe(ialu_reg);
10030 %}
10031
10032 // Long Negation
10033
10034 instruct negL_reg(iRegLNoSp dst, iRegL src, immL0 zero, rFlagsReg cr) %{
10035 match(Set dst (SubL zero src));
10036
10037 ins_cost(INSN_COST);
10038 format %{ "neg $dst, $src\t# long" %}
10039
10040 ins_encode %{
10041 __ neg(as_Register($dst$$reg),
10042 as_Register($src$$reg));
10043 %}
10044
10045 ins_pipe(ialu_reg);
10046 %}
10047
10048 // Integer Multiply
10049
10050 instruct mulI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10051 match(Set dst (MulI src1 src2));
10052
10053 ins_cost(INSN_COST * 3);
10054 format %{ "mulw $dst, $src1, $src2" %}
10055
10056 ins_encode %{
10057 __ mulw(as_Register($dst$$reg),
10058 as_Register($src1$$reg),
10059 as_Register($src2$$reg));
10060 %}
10061
10062 ins_pipe(imul_reg_reg);
10063 %}
10064
10065 instruct smulI(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10066 match(Set dst (MulL (ConvI2L src1) (ConvI2L src2)));
10067
10068 ins_cost(INSN_COST * 3);
10069 format %{ "smull $dst, $src1, $src2" %}
10070
10071 ins_encode %{
10072 __ smull(as_Register($dst$$reg),
10073 as_Register($src1$$reg),
10074 as_Register($src2$$reg));
10075 %}
10076
10077 ins_pipe(imul_reg_reg);
10078 %}
10079
10080 // Long Multiply
10081
10082 instruct mulL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10083 match(Set dst (MulL src1 src2));
10084
10085 ins_cost(INSN_COST * 5);
10086 format %{ "mul $dst, $src1, $src2" %}
10087
10088 ins_encode %{
10089 __ mul(as_Register($dst$$reg),
10090 as_Register($src1$$reg),
10091 as_Register($src2$$reg));
10092 %}
10093
10094 ins_pipe(lmul_reg_reg);
10095 %}
10096
10097 instruct mulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10098 %{
10099 match(Set dst (MulHiL src1 src2));
10100
10101 ins_cost(INSN_COST * 7);
10102 format %{ "smulh $dst, $src1, $src2\t# mulhi" %}
10103
10104 ins_encode %{
10105 __ smulh(as_Register($dst$$reg),
10106 as_Register($src1$$reg),
10107 as_Register($src2$$reg));
10108 %}
10109
10110 ins_pipe(lmul_reg_reg);
10111 %}
10112
10113 instruct umulHiL_rReg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr)
10114 %{
10115 match(Set dst (UMulHiL src1 src2));
10116
10117 ins_cost(INSN_COST * 7);
10118 format %{ "umulh $dst, $src1, $src2\t# umulhi" %}
10119
10120 ins_encode %{
10121 __ umulh(as_Register($dst$$reg),
10122 as_Register($src1$$reg),
10123 as_Register($src2$$reg));
10124 %}
10125
10126 ins_pipe(lmul_reg_reg);
10127 %}
10128
10129 // Combined Integer Multiply & Add/Sub
10130
10131 instruct maddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10132 match(Set dst (AddI src3 (MulI src1 src2)));
10133
10134 ins_cost(INSN_COST * 3);
10135 format %{ "madd $dst, $src1, $src2, $src3" %}
10136
10137 ins_encode %{
10138 __ maddw(as_Register($dst$$reg),
10139 as_Register($src1$$reg),
10140 as_Register($src2$$reg),
10141 as_Register($src3$$reg));
10142 %}
10143
10144 ins_pipe(imac_reg_reg);
10145 %}
10146
10147 instruct msubI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3) %{
10148 match(Set dst (SubI src3 (MulI src1 src2)));
10149
10150 ins_cost(INSN_COST * 3);
10151 format %{ "msub $dst, $src1, $src2, $src3" %}
10152
10153 ins_encode %{
10154 __ msubw(as_Register($dst$$reg),
10155 as_Register($src1$$reg),
10156 as_Register($src2$$reg),
10157 as_Register($src3$$reg));
10158 %}
10159
10160 ins_pipe(imac_reg_reg);
10161 %}
10162
10163 // Combined Integer Multiply & Neg
10164
10165 instruct mnegI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI0 zero) %{
10166 match(Set dst (MulI (SubI zero src1) src2));
10167
10168 ins_cost(INSN_COST * 3);
10169 format %{ "mneg $dst, $src1, $src2" %}
10170
10171 ins_encode %{
10172 __ mnegw(as_Register($dst$$reg),
10173 as_Register($src1$$reg),
10174 as_Register($src2$$reg));
10175 %}
10176
10177 ins_pipe(imac_reg_reg);
10178 %}
10179
10180 // Combined Long Multiply & Add/Sub
10181
10182 instruct maddL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10183 match(Set dst (AddL src3 (MulL src1 src2)));
10184
10185 ins_cost(INSN_COST * 5);
10186 format %{ "madd $dst, $src1, $src2, $src3" %}
10187
10188 ins_encode %{
10189 __ madd(as_Register($dst$$reg),
10190 as_Register($src1$$reg),
10191 as_Register($src2$$reg),
10192 as_Register($src3$$reg));
10193 %}
10194
10195 ins_pipe(lmac_reg_reg);
10196 %}
10197
10198 instruct msubL(iRegLNoSp dst, iRegL src1, iRegL src2, iRegL src3) %{
10199 match(Set dst (SubL src3 (MulL src1 src2)));
10200
10201 ins_cost(INSN_COST * 5);
10202 format %{ "msub $dst, $src1, $src2, $src3" %}
10203
10204 ins_encode %{
10205 __ msub(as_Register($dst$$reg),
10206 as_Register($src1$$reg),
10207 as_Register($src2$$reg),
10208 as_Register($src3$$reg));
10209 %}
10210
10211 ins_pipe(lmac_reg_reg);
10212 %}
10213
10214 // Combined Long Multiply & Neg
10215
10216 instruct mnegL(iRegLNoSp dst, iRegL src1, iRegL src2, immL0 zero) %{
10217 match(Set dst (MulL (SubL zero src1) src2));
10218
10219 ins_cost(INSN_COST * 5);
10220 format %{ "mneg $dst, $src1, $src2" %}
10221
10222 ins_encode %{
10223 __ mneg(as_Register($dst$$reg),
10224 as_Register($src1$$reg),
10225 as_Register($src2$$reg));
10226 %}
10227
10228 ins_pipe(lmac_reg_reg);
10229 %}
10230
10231 // Combine Integer Signed Multiply & Add/Sub/Neg Long
10232
10233 instruct smaddL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10234 match(Set dst (AddL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10235
10236 ins_cost(INSN_COST * 3);
10237 format %{ "smaddl $dst, $src1, $src2, $src3" %}
10238
10239 ins_encode %{
10240 __ smaddl(as_Register($dst$$reg),
10241 as_Register($src1$$reg),
10242 as_Register($src2$$reg),
10243 as_Register($src3$$reg));
10244 %}
10245
10246 ins_pipe(imac_reg_reg);
10247 %}
10248
10249 instruct smsubL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegLNoSp src3) %{
10250 match(Set dst (SubL src3 (MulL (ConvI2L src1) (ConvI2L src2))));
10251
10252 ins_cost(INSN_COST * 3);
10253 format %{ "smsubl $dst, $src1, $src2, $src3" %}
10254
10255 ins_encode %{
10256 __ smsubl(as_Register($dst$$reg),
10257 as_Register($src1$$reg),
10258 as_Register($src2$$reg),
10259 as_Register($src3$$reg));
10260 %}
10261
10262 ins_pipe(imac_reg_reg);
10263 %}
10264
10265 instruct smnegL(iRegLNoSp dst, iRegIorL2I src1, iRegIorL2I src2, immL0 zero) %{
10266 match(Set dst (MulL (SubL zero (ConvI2L src1)) (ConvI2L src2)));
10267
10268 ins_cost(INSN_COST * 3);
10269 format %{ "smnegl $dst, $src1, $src2" %}
10270
10271 ins_encode %{
10272 __ smnegl(as_Register($dst$$reg),
10273 as_Register($src1$$reg),
10274 as_Register($src2$$reg));
10275 %}
10276
10277 ins_pipe(imac_reg_reg);
10278 %}
10279
10280 // Combined Multiply-Add Shorts into Integer (dst = src1 * src2 + src3 * src4)
10281
10282 instruct muladdS2I(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, iRegIorL2I src3, iRegIorL2I src4) %{
10283 match(Set dst (MulAddS2I (Binary src1 src2) (Binary src3 src4)));
10284
10285 ins_cost(INSN_COST * 5);
10286 format %{ "mulw rscratch1, $src1, $src2\n\t"
10287 "maddw $dst, $src3, $src4, rscratch1" %}
10288
10289 ins_encode %{
10290 __ mulw(rscratch1, as_Register($src1$$reg), as_Register($src2$$reg));
10291 __ maddw(as_Register($dst$$reg), as_Register($src3$$reg), as_Register($src4$$reg), rscratch1); %}
10292
10293 ins_pipe(imac_reg_reg);
10294 %}
10295
10296 // Integer Divide
10297
10298 instruct divI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10299 match(Set dst (DivI src1 src2));
10300
10301 ins_cost(INSN_COST * 19);
10302 format %{ "sdivw $dst, $src1, $src2" %}
10303
10304 ins_encode(aarch64_enc_divw(dst, src1, src2));
10305 ins_pipe(idiv_reg_reg);
10306 %}
10307
10308 // Long Divide
10309
10310 instruct divL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10311 match(Set dst (DivL src1 src2));
10312
10313 ins_cost(INSN_COST * 35);
10314 format %{ "sdiv $dst, $src1, $src2" %}
10315
10316 ins_encode(aarch64_enc_div(dst, src1, src2));
10317 ins_pipe(ldiv_reg_reg);
10318 %}
10319
10320 // Integer Remainder
10321
10322 instruct modI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10323 match(Set dst (ModI src1 src2));
10324
10325 ins_cost(INSN_COST * 22);
10326 format %{ "sdivw rscratch1, $src1, $src2\n\t"
10327 "msubw $dst, rscratch1, $src2, $src1" %}
10328
10329 ins_encode(aarch64_enc_modw(dst, src1, src2));
10330 ins_pipe(idiv_reg_reg);
10331 %}
10332
10333 // Long Remainder
10334
10335 instruct modL(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10336 match(Set dst (ModL src1 src2));
10337
10338 ins_cost(INSN_COST * 38);
10339 format %{ "sdiv rscratch1, $src1, $src2\n"
10340 "msub $dst, rscratch1, $src2, $src1" %}
10341
10342 ins_encode(aarch64_enc_mod(dst, src1, src2));
10343 ins_pipe(ldiv_reg_reg);
10344 %}
10345
10346 // Unsigned Integer Divide
10347
10348 instruct UdivI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10349 match(Set dst (UDivI src1 src2));
10350
10351 ins_cost(INSN_COST * 19);
10352 format %{ "udivw $dst, $src1, $src2" %}
10353
10354 ins_encode %{
10355 __ udivw($dst$$Register, $src1$$Register, $src2$$Register);
10356 %}
10357
10358 ins_pipe(idiv_reg_reg);
10359 %}
10360
10361 // Unsigned Long Divide
10362
10363 instruct UdivL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10364 match(Set dst (UDivL src1 src2));
10365
10366 ins_cost(INSN_COST * 35);
10367 format %{ "udiv $dst, $src1, $src2" %}
10368
10369 ins_encode %{
10370 __ udiv($dst$$Register, $src1$$Register, $src2$$Register);
10371 %}
10372
10373 ins_pipe(ldiv_reg_reg);
10374 %}
10375
10376 // Unsigned Integer Remainder
10377
10378 instruct UmodI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10379 match(Set dst (UModI src1 src2));
10380
10381 ins_cost(INSN_COST * 22);
10382 format %{ "udivw rscratch1, $src1, $src2\n\t"
10383 "msubw $dst, rscratch1, $src2, $src1" %}
10384
10385 ins_encode %{
10386 __ udivw(rscratch1, $src1$$Register, $src2$$Register);
10387 __ msubw($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10388 %}
10389
10390 ins_pipe(idiv_reg_reg);
10391 %}
10392
10393 // Unsigned Long Remainder
10394
10395 instruct UModL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
10396 match(Set dst (UModL src1 src2));
10397
10398 ins_cost(INSN_COST * 38);
10399 format %{ "udiv rscratch1, $src1, $src2\n"
10400 "msub $dst, rscratch1, $src2, $src1" %}
10401
10402 ins_encode %{
10403 __ udiv(rscratch1, $src1$$Register, $src2$$Register);
10404 __ msub($dst$$Register, rscratch1, $src2$$Register, $src1$$Register);
10405 %}
10406
10407 ins_pipe(ldiv_reg_reg);
10408 %}
10409
10410 // Integer Shifts
10411
10412 // Shift Left Register
10413 instruct lShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10414 match(Set dst (LShiftI src1 src2));
10415
10416 ins_cost(INSN_COST * 2);
10417 format %{ "lslvw $dst, $src1, $src2" %}
10418
10419 ins_encode %{
10420 __ lslvw(as_Register($dst$$reg),
10421 as_Register($src1$$reg),
10422 as_Register($src2$$reg));
10423 %}
10424
10425 ins_pipe(ialu_reg_reg_vshift);
10426 %}
10427
10428 // Shift Left Immediate
10429 instruct lShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10430 match(Set dst (LShiftI src1 src2));
10431
10432 ins_cost(INSN_COST);
10433 format %{ "lslw $dst, $src1, ($src2 & 0x1f)" %}
10434
10435 ins_encode %{
10436 __ lslw(as_Register($dst$$reg),
10437 as_Register($src1$$reg),
10438 $src2$$constant & 0x1f);
10439 %}
10440
10441 ins_pipe(ialu_reg_shift);
10442 %}
10443
10444 // Shift Right Logical Register
10445 instruct urShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10446 match(Set dst (URShiftI src1 src2));
10447
10448 ins_cost(INSN_COST * 2);
10449 format %{ "lsrvw $dst, $src1, $src2" %}
10450
10451 ins_encode %{
10452 __ lsrvw(as_Register($dst$$reg),
10453 as_Register($src1$$reg),
10454 as_Register($src2$$reg));
10455 %}
10456
10457 ins_pipe(ialu_reg_reg_vshift);
10458 %}
10459
10460 // Shift Right Logical Immediate
10461 instruct urShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10462 match(Set dst (URShiftI src1 src2));
10463
10464 ins_cost(INSN_COST);
10465 format %{ "lsrw $dst, $src1, ($src2 & 0x1f)" %}
10466
10467 ins_encode %{
10468 __ lsrw(as_Register($dst$$reg),
10469 as_Register($src1$$reg),
10470 $src2$$constant & 0x1f);
10471 %}
10472
10473 ins_pipe(ialu_reg_shift);
10474 %}
10475
10476 // Shift Right Arithmetic Register
10477 instruct rShiftI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
10478 match(Set dst (RShiftI src1 src2));
10479
10480 ins_cost(INSN_COST * 2);
10481 format %{ "asrvw $dst, $src1, $src2" %}
10482
10483 ins_encode %{
10484 __ asrvw(as_Register($dst$$reg),
10485 as_Register($src1$$reg),
10486 as_Register($src2$$reg));
10487 %}
10488
10489 ins_pipe(ialu_reg_reg_vshift);
10490 %}
10491
10492 // Shift Right Arithmetic Immediate
10493 instruct rShiftI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immI src2) %{
10494 match(Set dst (RShiftI src1 src2));
10495
10496 ins_cost(INSN_COST);
10497 format %{ "asrw $dst, $src1, ($src2 & 0x1f)" %}
10498
10499 ins_encode %{
10500 __ asrw(as_Register($dst$$reg),
10501 as_Register($src1$$reg),
10502 $src2$$constant & 0x1f);
10503 %}
10504
10505 ins_pipe(ialu_reg_shift);
10506 %}
10507
10508 // Combined Int Mask and Right Shift (using UBFM)
10509 // TODO
10510
10511 // Long Shifts
10512
10513 // Shift Left Register
10514 instruct lShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10515 match(Set dst (LShiftL src1 src2));
10516
10517 ins_cost(INSN_COST * 2);
10518 format %{ "lslv $dst, $src1, $src2" %}
10519
10520 ins_encode %{
10521 __ lslv(as_Register($dst$$reg),
10522 as_Register($src1$$reg),
10523 as_Register($src2$$reg));
10524 %}
10525
10526 ins_pipe(ialu_reg_reg_vshift);
10527 %}
10528
10529 // Shift Left Immediate
10530 instruct lShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10531 match(Set dst (LShiftL src1 src2));
10532
10533 ins_cost(INSN_COST);
10534 format %{ "lsl $dst, $src1, ($src2 & 0x3f)" %}
10535
10536 ins_encode %{
10537 __ lsl(as_Register($dst$$reg),
10538 as_Register($src1$$reg),
10539 $src2$$constant & 0x3f);
10540 %}
10541
10542 ins_pipe(ialu_reg_shift);
10543 %}
10544
10545 // Shift Right Logical Register
10546 instruct urShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10547 match(Set dst (URShiftL src1 src2));
10548
10549 ins_cost(INSN_COST * 2);
10550 format %{ "lsrv $dst, $src1, $src2" %}
10551
10552 ins_encode %{
10553 __ lsrv(as_Register($dst$$reg),
10554 as_Register($src1$$reg),
10555 as_Register($src2$$reg));
10556 %}
10557
10558 ins_pipe(ialu_reg_reg_vshift);
10559 %}
10560
10561 // Shift Right Logical Immediate
10562 instruct urShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10563 match(Set dst (URShiftL src1 src2));
10564
10565 ins_cost(INSN_COST);
10566 format %{ "lsr $dst, $src1, ($src2 & 0x3f)" %}
10567
10568 ins_encode %{
10569 __ lsr(as_Register($dst$$reg),
10570 as_Register($src1$$reg),
10571 $src2$$constant & 0x3f);
10572 %}
10573
10574 ins_pipe(ialu_reg_shift);
10575 %}
10576
10577 // A special-case pattern for card table stores.
10578 instruct urShiftP_reg_imm(iRegLNoSp dst, iRegP src1, immI src2) %{
10579 match(Set dst (URShiftL (CastP2X src1) src2));
10580
10581 ins_cost(INSN_COST);
10582 format %{ "lsr $dst, p2x($src1), ($src2 & 0x3f)" %}
10583
10584 ins_encode %{
10585 __ lsr(as_Register($dst$$reg),
10586 as_Register($src1$$reg),
10587 $src2$$constant & 0x3f);
10588 %}
10589
10590 ins_pipe(ialu_reg_shift);
10591 %}
10592
10593 // Shift Right Arithmetic Register
10594 instruct rShiftL_reg_reg(iRegLNoSp dst, iRegL src1, iRegIorL2I src2) %{
10595 match(Set dst (RShiftL src1 src2));
10596
10597 ins_cost(INSN_COST * 2);
10598 format %{ "asrv $dst, $src1, $src2" %}
10599
10600 ins_encode %{
10601 __ asrv(as_Register($dst$$reg),
10602 as_Register($src1$$reg),
10603 as_Register($src2$$reg));
10604 %}
10605
10606 ins_pipe(ialu_reg_reg_vshift);
10607 %}
10608
10609 // Shift Right Arithmetic Immediate
10610 instruct rShiftL_reg_imm(iRegLNoSp dst, iRegL src1, immI src2) %{
10611 match(Set dst (RShiftL src1 src2));
10612
10613 ins_cost(INSN_COST);
10614 format %{ "asr $dst, $src1, ($src2 & 0x3f)" %}
10615
10616 ins_encode %{
10617 __ asr(as_Register($dst$$reg),
10618 as_Register($src1$$reg),
10619 $src2$$constant & 0x3f);
10620 %}
10621
10622 ins_pipe(ialu_reg_shift);
10623 %}
10624
10625 // BEGIN This section of the file is automatically generated. Do not edit --------------
10626 // This section is generated from aarch64_ad.m4
10627
10628 // This pattern is automatically generated from aarch64_ad.m4
10629 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10630 instruct regL_not_reg(iRegLNoSp dst,
10631 iRegL src1, immL_M1 m1,
10632 rFlagsReg cr) %{
10633 match(Set dst (XorL src1 m1));
10634 ins_cost(INSN_COST);
10635 format %{ "eon $dst, $src1, zr" %}
10636
10637 ins_encode %{
10638 __ eon(as_Register($dst$$reg),
10639 as_Register($src1$$reg),
10640 zr,
10641 Assembler::LSL, 0);
10642 %}
10643
10644 ins_pipe(ialu_reg);
10645 %}
10646
10647 // This pattern is automatically generated from aarch64_ad.m4
10648 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10649 instruct regI_not_reg(iRegINoSp dst,
10650 iRegIorL2I src1, immI_M1 m1,
10651 rFlagsReg cr) %{
10652 match(Set dst (XorI src1 m1));
10653 ins_cost(INSN_COST);
10654 format %{ "eonw $dst, $src1, zr" %}
10655
10656 ins_encode %{
10657 __ eonw(as_Register($dst$$reg),
10658 as_Register($src1$$reg),
10659 zr,
10660 Assembler::LSL, 0);
10661 %}
10662
10663 ins_pipe(ialu_reg);
10664 %}
10665
10666 // This pattern is automatically generated from aarch64_ad.m4
10667 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10668 instruct NegI_reg_URShift_reg(iRegINoSp dst,
10669 immI0 zero, iRegIorL2I src1, immI src2) %{
10670 match(Set dst (SubI zero (URShiftI src1 src2)));
10671
10672 ins_cost(1.9 * INSN_COST);
10673 format %{ "negw $dst, $src1, LSR $src2" %}
10674
10675 ins_encode %{
10676 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10677 Assembler::LSR, $src2$$constant & 0x1f);
10678 %}
10679
10680 ins_pipe(ialu_reg_shift);
10681 %}
10682
10683 // This pattern is automatically generated from aarch64_ad.m4
10684 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10685 instruct NegI_reg_RShift_reg(iRegINoSp dst,
10686 immI0 zero, iRegIorL2I src1, immI src2) %{
10687 match(Set dst (SubI zero (RShiftI src1 src2)));
10688
10689 ins_cost(1.9 * INSN_COST);
10690 format %{ "negw $dst, $src1, ASR $src2" %}
10691
10692 ins_encode %{
10693 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10694 Assembler::ASR, $src2$$constant & 0x1f);
10695 %}
10696
10697 ins_pipe(ialu_reg_shift);
10698 %}
10699
10700 // This pattern is automatically generated from aarch64_ad.m4
10701 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10702 instruct NegI_reg_LShift_reg(iRegINoSp dst,
10703 immI0 zero, iRegIorL2I src1, immI src2) %{
10704 match(Set dst (SubI zero (LShiftI src1 src2)));
10705
10706 ins_cost(1.9 * INSN_COST);
10707 format %{ "negw $dst, $src1, LSL $src2" %}
10708
10709 ins_encode %{
10710 __ negw(as_Register($dst$$reg), as_Register($src1$$reg),
10711 Assembler::LSL, $src2$$constant & 0x1f);
10712 %}
10713
10714 ins_pipe(ialu_reg_shift);
10715 %}
10716
10717 // This pattern is automatically generated from aarch64_ad.m4
10718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10719 instruct NegL_reg_URShift_reg(iRegLNoSp dst,
10720 immL0 zero, iRegL src1, immI src2) %{
10721 match(Set dst (SubL zero (URShiftL src1 src2)));
10722
10723 ins_cost(1.9 * INSN_COST);
10724 format %{ "neg $dst, $src1, LSR $src2" %}
10725
10726 ins_encode %{
10727 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10728 Assembler::LSR, $src2$$constant & 0x3f);
10729 %}
10730
10731 ins_pipe(ialu_reg_shift);
10732 %}
10733
10734 // This pattern is automatically generated from aarch64_ad.m4
10735 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10736 instruct NegL_reg_RShift_reg(iRegLNoSp dst,
10737 immL0 zero, iRegL src1, immI src2) %{
10738 match(Set dst (SubL zero (RShiftL src1 src2)));
10739
10740 ins_cost(1.9 * INSN_COST);
10741 format %{ "neg $dst, $src1, ASR $src2" %}
10742
10743 ins_encode %{
10744 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10745 Assembler::ASR, $src2$$constant & 0x3f);
10746 %}
10747
10748 ins_pipe(ialu_reg_shift);
10749 %}
10750
10751 // This pattern is automatically generated from aarch64_ad.m4
10752 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10753 instruct NegL_reg_LShift_reg(iRegLNoSp dst,
10754 immL0 zero, iRegL src1, immI src2) %{
10755 match(Set dst (SubL zero (LShiftL src1 src2)));
10756
10757 ins_cost(1.9 * INSN_COST);
10758 format %{ "neg $dst, $src1, LSL $src2" %}
10759
10760 ins_encode %{
10761 __ neg(as_Register($dst$$reg), as_Register($src1$$reg),
10762 Assembler::LSL, $src2$$constant & 0x3f);
10763 %}
10764
10765 ins_pipe(ialu_reg_shift);
10766 %}
10767
10768 // This pattern is automatically generated from aarch64_ad.m4
10769 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10770 instruct AndI_reg_not_reg(iRegINoSp dst,
10771 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10772 match(Set dst (AndI src1 (XorI src2 m1)));
10773 ins_cost(INSN_COST);
10774 format %{ "bicw $dst, $src1, $src2" %}
10775
10776 ins_encode %{
10777 __ bicw(as_Register($dst$$reg),
10778 as_Register($src1$$reg),
10779 as_Register($src2$$reg),
10780 Assembler::LSL, 0);
10781 %}
10782
10783 ins_pipe(ialu_reg_reg);
10784 %}
10785
10786 // This pattern is automatically generated from aarch64_ad.m4
10787 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10788 instruct AndL_reg_not_reg(iRegLNoSp dst,
10789 iRegL src1, iRegL src2, immL_M1 m1) %{
10790 match(Set dst (AndL src1 (XorL src2 m1)));
10791 ins_cost(INSN_COST);
10792 format %{ "bic $dst, $src1, $src2" %}
10793
10794 ins_encode %{
10795 __ bic(as_Register($dst$$reg),
10796 as_Register($src1$$reg),
10797 as_Register($src2$$reg),
10798 Assembler::LSL, 0);
10799 %}
10800
10801 ins_pipe(ialu_reg_reg);
10802 %}
10803
10804 // This pattern is automatically generated from aarch64_ad.m4
10805 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10806 instruct OrI_reg_not_reg(iRegINoSp dst,
10807 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10808 match(Set dst (OrI src1 (XorI src2 m1)));
10809 ins_cost(INSN_COST);
10810 format %{ "ornw $dst, $src1, $src2" %}
10811
10812 ins_encode %{
10813 __ ornw(as_Register($dst$$reg),
10814 as_Register($src1$$reg),
10815 as_Register($src2$$reg),
10816 Assembler::LSL, 0);
10817 %}
10818
10819 ins_pipe(ialu_reg_reg);
10820 %}
10821
10822 // This pattern is automatically generated from aarch64_ad.m4
10823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10824 instruct OrL_reg_not_reg(iRegLNoSp dst,
10825 iRegL src1, iRegL src2, immL_M1 m1) %{
10826 match(Set dst (OrL src1 (XorL src2 m1)));
10827 ins_cost(INSN_COST);
10828 format %{ "orn $dst, $src1, $src2" %}
10829
10830 ins_encode %{
10831 __ orn(as_Register($dst$$reg),
10832 as_Register($src1$$reg),
10833 as_Register($src2$$reg),
10834 Assembler::LSL, 0);
10835 %}
10836
10837 ins_pipe(ialu_reg_reg);
10838 %}
10839
10840 // This pattern is automatically generated from aarch64_ad.m4
10841 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10842 instruct XorI_reg_not_reg(iRegINoSp dst,
10843 iRegIorL2I src1, iRegIorL2I src2, immI_M1 m1) %{
10844 match(Set dst (XorI m1 (XorI src2 src1)));
10845 ins_cost(INSN_COST);
10846 format %{ "eonw $dst, $src1, $src2" %}
10847
10848 ins_encode %{
10849 __ eonw(as_Register($dst$$reg),
10850 as_Register($src1$$reg),
10851 as_Register($src2$$reg),
10852 Assembler::LSL, 0);
10853 %}
10854
10855 ins_pipe(ialu_reg_reg);
10856 %}
10857
10858 // This pattern is automatically generated from aarch64_ad.m4
10859 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10860 instruct XorL_reg_not_reg(iRegLNoSp dst,
10861 iRegL src1, iRegL src2, immL_M1 m1) %{
10862 match(Set dst (XorL m1 (XorL src2 src1)));
10863 ins_cost(INSN_COST);
10864 format %{ "eon $dst, $src1, $src2" %}
10865
10866 ins_encode %{
10867 __ eon(as_Register($dst$$reg),
10868 as_Register($src1$$reg),
10869 as_Register($src2$$reg),
10870 Assembler::LSL, 0);
10871 %}
10872
10873 ins_pipe(ialu_reg_reg);
10874 %}
10875
10876 // This pattern is automatically generated from aarch64_ad.m4
10877 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10878 // val & (-1 ^ (val >>> shift)) ==> bicw
10879 instruct AndI_reg_URShift_not_reg(iRegINoSp dst,
10880 iRegIorL2I src1, iRegIorL2I src2,
10881 immI src3, immI_M1 src4) %{
10882 match(Set dst (AndI src1 (XorI(URShiftI src2 src3) src4)));
10883 ins_cost(1.9 * INSN_COST);
10884 format %{ "bicw $dst, $src1, $src2, LSR $src3" %}
10885
10886 ins_encode %{
10887 __ bicw(as_Register($dst$$reg),
10888 as_Register($src1$$reg),
10889 as_Register($src2$$reg),
10890 Assembler::LSR,
10891 $src3$$constant & 0x1f);
10892 %}
10893
10894 ins_pipe(ialu_reg_reg_shift);
10895 %}
10896
10897 // This pattern is automatically generated from aarch64_ad.m4
10898 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10899 // val & (-1 ^ (val >>> shift)) ==> bic
10900 instruct AndL_reg_URShift_not_reg(iRegLNoSp dst,
10901 iRegL src1, iRegL src2,
10902 immI src3, immL_M1 src4) %{
10903 match(Set dst (AndL src1 (XorL(URShiftL src2 src3) src4)));
10904 ins_cost(1.9 * INSN_COST);
10905 format %{ "bic $dst, $src1, $src2, LSR $src3" %}
10906
10907 ins_encode %{
10908 __ bic(as_Register($dst$$reg),
10909 as_Register($src1$$reg),
10910 as_Register($src2$$reg),
10911 Assembler::LSR,
10912 $src3$$constant & 0x3f);
10913 %}
10914
10915 ins_pipe(ialu_reg_reg_shift);
10916 %}
10917
10918 // This pattern is automatically generated from aarch64_ad.m4
10919 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10920 // val & (-1 ^ (val >> shift)) ==> bicw
10921 instruct AndI_reg_RShift_not_reg(iRegINoSp dst,
10922 iRegIorL2I src1, iRegIorL2I src2,
10923 immI src3, immI_M1 src4) %{
10924 match(Set dst (AndI src1 (XorI(RShiftI src2 src3) src4)));
10925 ins_cost(1.9 * INSN_COST);
10926 format %{ "bicw $dst, $src1, $src2, ASR $src3" %}
10927
10928 ins_encode %{
10929 __ bicw(as_Register($dst$$reg),
10930 as_Register($src1$$reg),
10931 as_Register($src2$$reg),
10932 Assembler::ASR,
10933 $src3$$constant & 0x1f);
10934 %}
10935
10936 ins_pipe(ialu_reg_reg_shift);
10937 %}
10938
10939 // This pattern is automatically generated from aarch64_ad.m4
10940 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10941 // val & (-1 ^ (val >> shift)) ==> bic
10942 instruct AndL_reg_RShift_not_reg(iRegLNoSp dst,
10943 iRegL src1, iRegL src2,
10944 immI src3, immL_M1 src4) %{
10945 match(Set dst (AndL src1 (XorL(RShiftL src2 src3) src4)));
10946 ins_cost(1.9 * INSN_COST);
10947 format %{ "bic $dst, $src1, $src2, ASR $src3" %}
10948
10949 ins_encode %{
10950 __ bic(as_Register($dst$$reg),
10951 as_Register($src1$$reg),
10952 as_Register($src2$$reg),
10953 Assembler::ASR,
10954 $src3$$constant & 0x3f);
10955 %}
10956
10957 ins_pipe(ialu_reg_reg_shift);
10958 %}
10959
10960 // This pattern is automatically generated from aarch64_ad.m4
10961 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10962 // val & (-1 ^ (val ror shift)) ==> bicw
10963 instruct AndI_reg_RotateRight_not_reg(iRegINoSp dst,
10964 iRegIorL2I src1, iRegIorL2I src2,
10965 immI src3, immI_M1 src4) %{
10966 match(Set dst (AndI src1 (XorI(RotateRight src2 src3) src4)));
10967 ins_cost(1.9 * INSN_COST);
10968 format %{ "bicw $dst, $src1, $src2, ROR $src3" %}
10969
10970 ins_encode %{
10971 __ bicw(as_Register($dst$$reg),
10972 as_Register($src1$$reg),
10973 as_Register($src2$$reg),
10974 Assembler::ROR,
10975 $src3$$constant & 0x1f);
10976 %}
10977
10978 ins_pipe(ialu_reg_reg_shift);
10979 %}
10980
10981 // This pattern is automatically generated from aarch64_ad.m4
10982 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
10983 // val & (-1 ^ (val ror shift)) ==> bic
10984 instruct AndL_reg_RotateRight_not_reg(iRegLNoSp dst,
10985 iRegL src1, iRegL src2,
10986 immI src3, immL_M1 src4) %{
10987 match(Set dst (AndL src1 (XorL(RotateRight src2 src3) src4)));
10988 ins_cost(1.9 * INSN_COST);
10989 format %{ "bic $dst, $src1, $src2, ROR $src3" %}
10990
10991 ins_encode %{
10992 __ bic(as_Register($dst$$reg),
10993 as_Register($src1$$reg),
10994 as_Register($src2$$reg),
10995 Assembler::ROR,
10996 $src3$$constant & 0x3f);
10997 %}
10998
10999 ins_pipe(ialu_reg_reg_shift);
11000 %}
11001
11002 // This pattern is automatically generated from aarch64_ad.m4
11003 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11004 // val & (-1 ^ (val << shift)) ==> bicw
11005 instruct AndI_reg_LShift_not_reg(iRegINoSp dst,
11006 iRegIorL2I src1, iRegIorL2I src2,
11007 immI src3, immI_M1 src4) %{
11008 match(Set dst (AndI src1 (XorI(LShiftI src2 src3) src4)));
11009 ins_cost(1.9 * INSN_COST);
11010 format %{ "bicw $dst, $src1, $src2, LSL $src3" %}
11011
11012 ins_encode %{
11013 __ bicw(as_Register($dst$$reg),
11014 as_Register($src1$$reg),
11015 as_Register($src2$$reg),
11016 Assembler::LSL,
11017 $src3$$constant & 0x1f);
11018 %}
11019
11020 ins_pipe(ialu_reg_reg_shift);
11021 %}
11022
11023 // This pattern is automatically generated from aarch64_ad.m4
11024 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11025 // val & (-1 ^ (val << shift)) ==> bic
11026 instruct AndL_reg_LShift_not_reg(iRegLNoSp dst,
11027 iRegL src1, iRegL src2,
11028 immI src3, immL_M1 src4) %{
11029 match(Set dst (AndL src1 (XorL(LShiftL src2 src3) src4)));
11030 ins_cost(1.9 * INSN_COST);
11031 format %{ "bic $dst, $src1, $src2, LSL $src3" %}
11032
11033 ins_encode %{
11034 __ bic(as_Register($dst$$reg),
11035 as_Register($src1$$reg),
11036 as_Register($src2$$reg),
11037 Assembler::LSL,
11038 $src3$$constant & 0x3f);
11039 %}
11040
11041 ins_pipe(ialu_reg_reg_shift);
11042 %}
11043
11044 // This pattern is automatically generated from aarch64_ad.m4
11045 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11046 // val ^ (-1 ^ (val >>> shift)) ==> eonw
11047 instruct XorI_reg_URShift_not_reg(iRegINoSp dst,
11048 iRegIorL2I src1, iRegIorL2I src2,
11049 immI src3, immI_M1 src4) %{
11050 match(Set dst (XorI src4 (XorI(URShiftI src2 src3) src1)));
11051 ins_cost(1.9 * INSN_COST);
11052 format %{ "eonw $dst, $src1, $src2, LSR $src3" %}
11053
11054 ins_encode %{
11055 __ eonw(as_Register($dst$$reg),
11056 as_Register($src1$$reg),
11057 as_Register($src2$$reg),
11058 Assembler::LSR,
11059 $src3$$constant & 0x1f);
11060 %}
11061
11062 ins_pipe(ialu_reg_reg_shift);
11063 %}
11064
11065 // This pattern is automatically generated from aarch64_ad.m4
11066 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11067 // val ^ (-1 ^ (val >>> shift)) ==> eon
11068 instruct XorL_reg_URShift_not_reg(iRegLNoSp dst,
11069 iRegL src1, iRegL src2,
11070 immI src3, immL_M1 src4) %{
11071 match(Set dst (XorL src4 (XorL(URShiftL src2 src3) src1)));
11072 ins_cost(1.9 * INSN_COST);
11073 format %{ "eon $dst, $src1, $src2, LSR $src3" %}
11074
11075 ins_encode %{
11076 __ eon(as_Register($dst$$reg),
11077 as_Register($src1$$reg),
11078 as_Register($src2$$reg),
11079 Assembler::LSR,
11080 $src3$$constant & 0x3f);
11081 %}
11082
11083 ins_pipe(ialu_reg_reg_shift);
11084 %}
11085
11086 // This pattern is automatically generated from aarch64_ad.m4
11087 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11088 // val ^ (-1 ^ (val >> shift)) ==> eonw
11089 instruct XorI_reg_RShift_not_reg(iRegINoSp dst,
11090 iRegIorL2I src1, iRegIorL2I src2,
11091 immI src3, immI_M1 src4) %{
11092 match(Set dst (XorI src4 (XorI(RShiftI src2 src3) src1)));
11093 ins_cost(1.9 * INSN_COST);
11094 format %{ "eonw $dst, $src1, $src2, ASR $src3" %}
11095
11096 ins_encode %{
11097 __ eonw(as_Register($dst$$reg),
11098 as_Register($src1$$reg),
11099 as_Register($src2$$reg),
11100 Assembler::ASR,
11101 $src3$$constant & 0x1f);
11102 %}
11103
11104 ins_pipe(ialu_reg_reg_shift);
11105 %}
11106
11107 // This pattern is automatically generated from aarch64_ad.m4
11108 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11109 // val ^ (-1 ^ (val >> shift)) ==> eon
11110 instruct XorL_reg_RShift_not_reg(iRegLNoSp dst,
11111 iRegL src1, iRegL src2,
11112 immI src3, immL_M1 src4) %{
11113 match(Set dst (XorL src4 (XorL(RShiftL src2 src3) src1)));
11114 ins_cost(1.9 * INSN_COST);
11115 format %{ "eon $dst, $src1, $src2, ASR $src3" %}
11116
11117 ins_encode %{
11118 __ eon(as_Register($dst$$reg),
11119 as_Register($src1$$reg),
11120 as_Register($src2$$reg),
11121 Assembler::ASR,
11122 $src3$$constant & 0x3f);
11123 %}
11124
11125 ins_pipe(ialu_reg_reg_shift);
11126 %}
11127
11128 // This pattern is automatically generated from aarch64_ad.m4
11129 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11130 // val ^ (-1 ^ (val ror shift)) ==> eonw
11131 instruct XorI_reg_RotateRight_not_reg(iRegINoSp dst,
11132 iRegIorL2I src1, iRegIorL2I src2,
11133 immI src3, immI_M1 src4) %{
11134 match(Set dst (XorI src4 (XorI(RotateRight src2 src3) src1)));
11135 ins_cost(1.9 * INSN_COST);
11136 format %{ "eonw $dst, $src1, $src2, ROR $src3" %}
11137
11138 ins_encode %{
11139 __ eonw(as_Register($dst$$reg),
11140 as_Register($src1$$reg),
11141 as_Register($src2$$reg),
11142 Assembler::ROR,
11143 $src3$$constant & 0x1f);
11144 %}
11145
11146 ins_pipe(ialu_reg_reg_shift);
11147 %}
11148
11149 // This pattern is automatically generated from aarch64_ad.m4
11150 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11151 // val ^ (-1 ^ (val ror shift)) ==> eon
11152 instruct XorL_reg_RotateRight_not_reg(iRegLNoSp dst,
11153 iRegL src1, iRegL src2,
11154 immI src3, immL_M1 src4) %{
11155 match(Set dst (XorL src4 (XorL(RotateRight src2 src3) src1)));
11156 ins_cost(1.9 * INSN_COST);
11157 format %{ "eon $dst, $src1, $src2, ROR $src3" %}
11158
11159 ins_encode %{
11160 __ eon(as_Register($dst$$reg),
11161 as_Register($src1$$reg),
11162 as_Register($src2$$reg),
11163 Assembler::ROR,
11164 $src3$$constant & 0x3f);
11165 %}
11166
11167 ins_pipe(ialu_reg_reg_shift);
11168 %}
11169
11170 // This pattern is automatically generated from aarch64_ad.m4
11171 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11172 // val ^ (-1 ^ (val << shift)) ==> eonw
11173 instruct XorI_reg_LShift_not_reg(iRegINoSp dst,
11174 iRegIorL2I src1, iRegIorL2I src2,
11175 immI src3, immI_M1 src4) %{
11176 match(Set dst (XorI src4 (XorI(LShiftI src2 src3) src1)));
11177 ins_cost(1.9 * INSN_COST);
11178 format %{ "eonw $dst, $src1, $src2, LSL $src3" %}
11179
11180 ins_encode %{
11181 __ eonw(as_Register($dst$$reg),
11182 as_Register($src1$$reg),
11183 as_Register($src2$$reg),
11184 Assembler::LSL,
11185 $src3$$constant & 0x1f);
11186 %}
11187
11188 ins_pipe(ialu_reg_reg_shift);
11189 %}
11190
11191 // This pattern is automatically generated from aarch64_ad.m4
11192 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11193 // val ^ (-1 ^ (val << shift)) ==> eon
11194 instruct XorL_reg_LShift_not_reg(iRegLNoSp dst,
11195 iRegL src1, iRegL src2,
11196 immI src3, immL_M1 src4) %{
11197 match(Set dst (XorL src4 (XorL(LShiftL src2 src3) src1)));
11198 ins_cost(1.9 * INSN_COST);
11199 format %{ "eon $dst, $src1, $src2, LSL $src3" %}
11200
11201 ins_encode %{
11202 __ eon(as_Register($dst$$reg),
11203 as_Register($src1$$reg),
11204 as_Register($src2$$reg),
11205 Assembler::LSL,
11206 $src3$$constant & 0x3f);
11207 %}
11208
11209 ins_pipe(ialu_reg_reg_shift);
11210 %}
11211
11212 // This pattern is automatically generated from aarch64_ad.m4
11213 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11214 // val | (-1 ^ (val >>> shift)) ==> ornw
11215 instruct OrI_reg_URShift_not_reg(iRegINoSp dst,
11216 iRegIorL2I src1, iRegIorL2I src2,
11217 immI src3, immI_M1 src4) %{
11218 match(Set dst (OrI src1 (XorI(URShiftI src2 src3) src4)));
11219 ins_cost(1.9 * INSN_COST);
11220 format %{ "ornw $dst, $src1, $src2, LSR $src3" %}
11221
11222 ins_encode %{
11223 __ ornw(as_Register($dst$$reg),
11224 as_Register($src1$$reg),
11225 as_Register($src2$$reg),
11226 Assembler::LSR,
11227 $src3$$constant & 0x1f);
11228 %}
11229
11230 ins_pipe(ialu_reg_reg_shift);
11231 %}
11232
11233 // This pattern is automatically generated from aarch64_ad.m4
11234 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11235 // val | (-1 ^ (val >>> shift)) ==> orn
11236 instruct OrL_reg_URShift_not_reg(iRegLNoSp dst,
11237 iRegL src1, iRegL src2,
11238 immI src3, immL_M1 src4) %{
11239 match(Set dst (OrL src1 (XorL(URShiftL src2 src3) src4)));
11240 ins_cost(1.9 * INSN_COST);
11241 format %{ "orn $dst, $src1, $src2, LSR $src3" %}
11242
11243 ins_encode %{
11244 __ orn(as_Register($dst$$reg),
11245 as_Register($src1$$reg),
11246 as_Register($src2$$reg),
11247 Assembler::LSR,
11248 $src3$$constant & 0x3f);
11249 %}
11250
11251 ins_pipe(ialu_reg_reg_shift);
11252 %}
11253
11254 // This pattern is automatically generated from aarch64_ad.m4
11255 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11256 // val | (-1 ^ (val >> shift)) ==> ornw
11257 instruct OrI_reg_RShift_not_reg(iRegINoSp dst,
11258 iRegIorL2I src1, iRegIorL2I src2,
11259 immI src3, immI_M1 src4) %{
11260 match(Set dst (OrI src1 (XorI(RShiftI src2 src3) src4)));
11261 ins_cost(1.9 * INSN_COST);
11262 format %{ "ornw $dst, $src1, $src2, ASR $src3" %}
11263
11264 ins_encode %{
11265 __ ornw(as_Register($dst$$reg),
11266 as_Register($src1$$reg),
11267 as_Register($src2$$reg),
11268 Assembler::ASR,
11269 $src3$$constant & 0x1f);
11270 %}
11271
11272 ins_pipe(ialu_reg_reg_shift);
11273 %}
11274
11275 // This pattern is automatically generated from aarch64_ad.m4
11276 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11277 // val | (-1 ^ (val >> shift)) ==> orn
11278 instruct OrL_reg_RShift_not_reg(iRegLNoSp dst,
11279 iRegL src1, iRegL src2,
11280 immI src3, immL_M1 src4) %{
11281 match(Set dst (OrL src1 (XorL(RShiftL src2 src3) src4)));
11282 ins_cost(1.9 * INSN_COST);
11283 format %{ "orn $dst, $src1, $src2, ASR $src3" %}
11284
11285 ins_encode %{
11286 __ orn(as_Register($dst$$reg),
11287 as_Register($src1$$reg),
11288 as_Register($src2$$reg),
11289 Assembler::ASR,
11290 $src3$$constant & 0x3f);
11291 %}
11292
11293 ins_pipe(ialu_reg_reg_shift);
11294 %}
11295
11296 // This pattern is automatically generated from aarch64_ad.m4
11297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11298 // val | (-1 ^ (val ror shift)) ==> ornw
11299 instruct OrI_reg_RotateRight_not_reg(iRegINoSp dst,
11300 iRegIorL2I src1, iRegIorL2I src2,
11301 immI src3, immI_M1 src4) %{
11302 match(Set dst (OrI src1 (XorI(RotateRight src2 src3) src4)));
11303 ins_cost(1.9 * INSN_COST);
11304 format %{ "ornw $dst, $src1, $src2, ROR $src3" %}
11305
11306 ins_encode %{
11307 __ ornw(as_Register($dst$$reg),
11308 as_Register($src1$$reg),
11309 as_Register($src2$$reg),
11310 Assembler::ROR,
11311 $src3$$constant & 0x1f);
11312 %}
11313
11314 ins_pipe(ialu_reg_reg_shift);
11315 %}
11316
11317 // This pattern is automatically generated from aarch64_ad.m4
11318 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11319 // val | (-1 ^ (val ror shift)) ==> orn
11320 instruct OrL_reg_RotateRight_not_reg(iRegLNoSp dst,
11321 iRegL src1, iRegL src2,
11322 immI src3, immL_M1 src4) %{
11323 match(Set dst (OrL src1 (XorL(RotateRight src2 src3) src4)));
11324 ins_cost(1.9 * INSN_COST);
11325 format %{ "orn $dst, $src1, $src2, ROR $src3" %}
11326
11327 ins_encode %{
11328 __ orn(as_Register($dst$$reg),
11329 as_Register($src1$$reg),
11330 as_Register($src2$$reg),
11331 Assembler::ROR,
11332 $src3$$constant & 0x3f);
11333 %}
11334
11335 ins_pipe(ialu_reg_reg_shift);
11336 %}
11337
11338 // This pattern is automatically generated from aarch64_ad.m4
11339 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11340 // val | (-1 ^ (val << shift)) ==> ornw
11341 instruct OrI_reg_LShift_not_reg(iRegINoSp dst,
11342 iRegIorL2I src1, iRegIorL2I src2,
11343 immI src3, immI_M1 src4) %{
11344 match(Set dst (OrI src1 (XorI(LShiftI src2 src3) src4)));
11345 ins_cost(1.9 * INSN_COST);
11346 format %{ "ornw $dst, $src1, $src2, LSL $src3" %}
11347
11348 ins_encode %{
11349 __ ornw(as_Register($dst$$reg),
11350 as_Register($src1$$reg),
11351 as_Register($src2$$reg),
11352 Assembler::LSL,
11353 $src3$$constant & 0x1f);
11354 %}
11355
11356 ins_pipe(ialu_reg_reg_shift);
11357 %}
11358
11359 // This pattern is automatically generated from aarch64_ad.m4
11360 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11361 // val | (-1 ^ (val << shift)) ==> orn
11362 instruct OrL_reg_LShift_not_reg(iRegLNoSp dst,
11363 iRegL src1, iRegL src2,
11364 immI src3, immL_M1 src4) %{
11365 match(Set dst (OrL src1 (XorL(LShiftL src2 src3) src4)));
11366 ins_cost(1.9 * INSN_COST);
11367 format %{ "orn $dst, $src1, $src2, LSL $src3" %}
11368
11369 ins_encode %{
11370 __ orn(as_Register($dst$$reg),
11371 as_Register($src1$$reg),
11372 as_Register($src2$$reg),
11373 Assembler::LSL,
11374 $src3$$constant & 0x3f);
11375 %}
11376
11377 ins_pipe(ialu_reg_reg_shift);
11378 %}
11379
11380 // This pattern is automatically generated from aarch64_ad.m4
11381 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11382 instruct AndI_reg_URShift_reg(iRegINoSp dst,
11383 iRegIorL2I src1, iRegIorL2I src2,
11384 immI src3) %{
11385 match(Set dst (AndI src1 (URShiftI src2 src3)));
11386
11387 ins_cost(1.9 * INSN_COST);
11388 format %{ "andw $dst, $src1, $src2, LSR $src3" %}
11389
11390 ins_encode %{
11391 __ andw(as_Register($dst$$reg),
11392 as_Register($src1$$reg),
11393 as_Register($src2$$reg),
11394 Assembler::LSR,
11395 $src3$$constant & 0x1f);
11396 %}
11397
11398 ins_pipe(ialu_reg_reg_shift);
11399 %}
11400
11401 // This pattern is automatically generated from aarch64_ad.m4
11402 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11403 instruct AndL_reg_URShift_reg(iRegLNoSp dst,
11404 iRegL src1, iRegL src2,
11405 immI src3) %{
11406 match(Set dst (AndL src1 (URShiftL src2 src3)));
11407
11408 ins_cost(1.9 * INSN_COST);
11409 format %{ "andr $dst, $src1, $src2, LSR $src3" %}
11410
11411 ins_encode %{
11412 __ andr(as_Register($dst$$reg),
11413 as_Register($src1$$reg),
11414 as_Register($src2$$reg),
11415 Assembler::LSR,
11416 $src3$$constant & 0x3f);
11417 %}
11418
11419 ins_pipe(ialu_reg_reg_shift);
11420 %}
11421
11422 // This pattern is automatically generated from aarch64_ad.m4
11423 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11424 instruct AndI_reg_RShift_reg(iRegINoSp dst,
11425 iRegIorL2I src1, iRegIorL2I src2,
11426 immI src3) %{
11427 match(Set dst (AndI src1 (RShiftI src2 src3)));
11428
11429 ins_cost(1.9 * INSN_COST);
11430 format %{ "andw $dst, $src1, $src2, ASR $src3" %}
11431
11432 ins_encode %{
11433 __ andw(as_Register($dst$$reg),
11434 as_Register($src1$$reg),
11435 as_Register($src2$$reg),
11436 Assembler::ASR,
11437 $src3$$constant & 0x1f);
11438 %}
11439
11440 ins_pipe(ialu_reg_reg_shift);
11441 %}
11442
11443 // This pattern is automatically generated from aarch64_ad.m4
11444 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11445 instruct AndL_reg_RShift_reg(iRegLNoSp dst,
11446 iRegL src1, iRegL src2,
11447 immI src3) %{
11448 match(Set dst (AndL src1 (RShiftL src2 src3)));
11449
11450 ins_cost(1.9 * INSN_COST);
11451 format %{ "andr $dst, $src1, $src2, ASR $src3" %}
11452
11453 ins_encode %{
11454 __ andr(as_Register($dst$$reg),
11455 as_Register($src1$$reg),
11456 as_Register($src2$$reg),
11457 Assembler::ASR,
11458 $src3$$constant & 0x3f);
11459 %}
11460
11461 ins_pipe(ialu_reg_reg_shift);
11462 %}
11463
11464 // This pattern is automatically generated from aarch64_ad.m4
11465 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11466 instruct AndI_reg_LShift_reg(iRegINoSp dst,
11467 iRegIorL2I src1, iRegIorL2I src2,
11468 immI src3) %{
11469 match(Set dst (AndI src1 (LShiftI src2 src3)));
11470
11471 ins_cost(1.9 * INSN_COST);
11472 format %{ "andw $dst, $src1, $src2, LSL $src3" %}
11473
11474 ins_encode %{
11475 __ andw(as_Register($dst$$reg),
11476 as_Register($src1$$reg),
11477 as_Register($src2$$reg),
11478 Assembler::LSL,
11479 $src3$$constant & 0x1f);
11480 %}
11481
11482 ins_pipe(ialu_reg_reg_shift);
11483 %}
11484
11485 // This pattern is automatically generated from aarch64_ad.m4
11486 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11487 instruct AndL_reg_LShift_reg(iRegLNoSp dst,
11488 iRegL src1, iRegL src2,
11489 immI src3) %{
11490 match(Set dst (AndL src1 (LShiftL src2 src3)));
11491
11492 ins_cost(1.9 * INSN_COST);
11493 format %{ "andr $dst, $src1, $src2, LSL $src3" %}
11494
11495 ins_encode %{
11496 __ andr(as_Register($dst$$reg),
11497 as_Register($src1$$reg),
11498 as_Register($src2$$reg),
11499 Assembler::LSL,
11500 $src3$$constant & 0x3f);
11501 %}
11502
11503 ins_pipe(ialu_reg_reg_shift);
11504 %}
11505
11506 // This pattern is automatically generated from aarch64_ad.m4
11507 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11508 instruct AndI_reg_RotateRight_reg(iRegINoSp dst,
11509 iRegIorL2I src1, iRegIorL2I src2,
11510 immI src3) %{
11511 match(Set dst (AndI src1 (RotateRight src2 src3)));
11512
11513 ins_cost(1.9 * INSN_COST);
11514 format %{ "andw $dst, $src1, $src2, ROR $src3" %}
11515
11516 ins_encode %{
11517 __ andw(as_Register($dst$$reg),
11518 as_Register($src1$$reg),
11519 as_Register($src2$$reg),
11520 Assembler::ROR,
11521 $src3$$constant & 0x1f);
11522 %}
11523
11524 ins_pipe(ialu_reg_reg_shift);
11525 %}
11526
11527 // This pattern is automatically generated from aarch64_ad.m4
11528 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11529 instruct AndL_reg_RotateRight_reg(iRegLNoSp dst,
11530 iRegL src1, iRegL src2,
11531 immI src3) %{
11532 match(Set dst (AndL src1 (RotateRight src2 src3)));
11533
11534 ins_cost(1.9 * INSN_COST);
11535 format %{ "andr $dst, $src1, $src2, ROR $src3" %}
11536
11537 ins_encode %{
11538 __ andr(as_Register($dst$$reg),
11539 as_Register($src1$$reg),
11540 as_Register($src2$$reg),
11541 Assembler::ROR,
11542 $src3$$constant & 0x3f);
11543 %}
11544
11545 ins_pipe(ialu_reg_reg_shift);
11546 %}
11547
11548 // This pattern is automatically generated from aarch64_ad.m4
11549 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11550 instruct XorI_reg_URShift_reg(iRegINoSp dst,
11551 iRegIorL2I src1, iRegIorL2I src2,
11552 immI src3) %{
11553 match(Set dst (XorI src1 (URShiftI src2 src3)));
11554
11555 ins_cost(1.9 * INSN_COST);
11556 format %{ "eorw $dst, $src1, $src2, LSR $src3" %}
11557
11558 ins_encode %{
11559 __ eorw(as_Register($dst$$reg),
11560 as_Register($src1$$reg),
11561 as_Register($src2$$reg),
11562 Assembler::LSR,
11563 $src3$$constant & 0x1f);
11564 %}
11565
11566 ins_pipe(ialu_reg_reg_shift);
11567 %}
11568
11569 // This pattern is automatically generated from aarch64_ad.m4
11570 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11571 instruct XorL_reg_URShift_reg(iRegLNoSp dst,
11572 iRegL src1, iRegL src2,
11573 immI src3) %{
11574 match(Set dst (XorL src1 (URShiftL src2 src3)));
11575
11576 ins_cost(1.9 * INSN_COST);
11577 format %{ "eor $dst, $src1, $src2, LSR $src3" %}
11578
11579 ins_encode %{
11580 __ eor(as_Register($dst$$reg),
11581 as_Register($src1$$reg),
11582 as_Register($src2$$reg),
11583 Assembler::LSR,
11584 $src3$$constant & 0x3f);
11585 %}
11586
11587 ins_pipe(ialu_reg_reg_shift);
11588 %}
11589
11590 // This pattern is automatically generated from aarch64_ad.m4
11591 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11592 instruct XorI_reg_RShift_reg(iRegINoSp dst,
11593 iRegIorL2I src1, iRegIorL2I src2,
11594 immI src3) %{
11595 match(Set dst (XorI src1 (RShiftI src2 src3)));
11596
11597 ins_cost(1.9 * INSN_COST);
11598 format %{ "eorw $dst, $src1, $src2, ASR $src3" %}
11599
11600 ins_encode %{
11601 __ eorw(as_Register($dst$$reg),
11602 as_Register($src1$$reg),
11603 as_Register($src2$$reg),
11604 Assembler::ASR,
11605 $src3$$constant & 0x1f);
11606 %}
11607
11608 ins_pipe(ialu_reg_reg_shift);
11609 %}
11610
11611 // This pattern is automatically generated from aarch64_ad.m4
11612 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11613 instruct XorL_reg_RShift_reg(iRegLNoSp dst,
11614 iRegL src1, iRegL src2,
11615 immI src3) %{
11616 match(Set dst (XorL src1 (RShiftL src2 src3)));
11617
11618 ins_cost(1.9 * INSN_COST);
11619 format %{ "eor $dst, $src1, $src2, ASR $src3" %}
11620
11621 ins_encode %{
11622 __ eor(as_Register($dst$$reg),
11623 as_Register($src1$$reg),
11624 as_Register($src2$$reg),
11625 Assembler::ASR,
11626 $src3$$constant & 0x3f);
11627 %}
11628
11629 ins_pipe(ialu_reg_reg_shift);
11630 %}
11631
11632 // This pattern is automatically generated from aarch64_ad.m4
11633 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11634 instruct XorI_reg_LShift_reg(iRegINoSp dst,
11635 iRegIorL2I src1, iRegIorL2I src2,
11636 immI src3) %{
11637 match(Set dst (XorI src1 (LShiftI src2 src3)));
11638
11639 ins_cost(1.9 * INSN_COST);
11640 format %{ "eorw $dst, $src1, $src2, LSL $src3" %}
11641
11642 ins_encode %{
11643 __ eorw(as_Register($dst$$reg),
11644 as_Register($src1$$reg),
11645 as_Register($src2$$reg),
11646 Assembler::LSL,
11647 $src3$$constant & 0x1f);
11648 %}
11649
11650 ins_pipe(ialu_reg_reg_shift);
11651 %}
11652
11653 // This pattern is automatically generated from aarch64_ad.m4
11654 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11655 instruct XorL_reg_LShift_reg(iRegLNoSp dst,
11656 iRegL src1, iRegL src2,
11657 immI src3) %{
11658 match(Set dst (XorL src1 (LShiftL src2 src3)));
11659
11660 ins_cost(1.9 * INSN_COST);
11661 format %{ "eor $dst, $src1, $src2, LSL $src3" %}
11662
11663 ins_encode %{
11664 __ eor(as_Register($dst$$reg),
11665 as_Register($src1$$reg),
11666 as_Register($src2$$reg),
11667 Assembler::LSL,
11668 $src3$$constant & 0x3f);
11669 %}
11670
11671 ins_pipe(ialu_reg_reg_shift);
11672 %}
11673
11674 // This pattern is automatically generated from aarch64_ad.m4
11675 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11676 instruct XorI_reg_RotateRight_reg(iRegINoSp dst,
11677 iRegIorL2I src1, iRegIorL2I src2,
11678 immI src3) %{
11679 match(Set dst (XorI src1 (RotateRight src2 src3)));
11680
11681 ins_cost(1.9 * INSN_COST);
11682 format %{ "eorw $dst, $src1, $src2, ROR $src3" %}
11683
11684 ins_encode %{
11685 __ eorw(as_Register($dst$$reg),
11686 as_Register($src1$$reg),
11687 as_Register($src2$$reg),
11688 Assembler::ROR,
11689 $src3$$constant & 0x1f);
11690 %}
11691
11692 ins_pipe(ialu_reg_reg_shift);
11693 %}
11694
11695 // This pattern is automatically generated from aarch64_ad.m4
11696 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11697 instruct XorL_reg_RotateRight_reg(iRegLNoSp dst,
11698 iRegL src1, iRegL src2,
11699 immI src3) %{
11700 match(Set dst (XorL src1 (RotateRight src2 src3)));
11701
11702 ins_cost(1.9 * INSN_COST);
11703 format %{ "eor $dst, $src1, $src2, ROR $src3" %}
11704
11705 ins_encode %{
11706 __ eor(as_Register($dst$$reg),
11707 as_Register($src1$$reg),
11708 as_Register($src2$$reg),
11709 Assembler::ROR,
11710 $src3$$constant & 0x3f);
11711 %}
11712
11713 ins_pipe(ialu_reg_reg_shift);
11714 %}
11715
11716 // This pattern is automatically generated from aarch64_ad.m4
11717 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11718 instruct OrI_reg_URShift_reg(iRegINoSp dst,
11719 iRegIorL2I src1, iRegIorL2I src2,
11720 immI src3) %{
11721 match(Set dst (OrI src1 (URShiftI src2 src3)));
11722
11723 ins_cost(1.9 * INSN_COST);
11724 format %{ "orrw $dst, $src1, $src2, LSR $src3" %}
11725
11726 ins_encode %{
11727 __ orrw(as_Register($dst$$reg),
11728 as_Register($src1$$reg),
11729 as_Register($src2$$reg),
11730 Assembler::LSR,
11731 $src3$$constant & 0x1f);
11732 %}
11733
11734 ins_pipe(ialu_reg_reg_shift);
11735 %}
11736
11737 // This pattern is automatically generated from aarch64_ad.m4
11738 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11739 instruct OrL_reg_URShift_reg(iRegLNoSp dst,
11740 iRegL src1, iRegL src2,
11741 immI src3) %{
11742 match(Set dst (OrL src1 (URShiftL src2 src3)));
11743
11744 ins_cost(1.9 * INSN_COST);
11745 format %{ "orr $dst, $src1, $src2, LSR $src3" %}
11746
11747 ins_encode %{
11748 __ orr(as_Register($dst$$reg),
11749 as_Register($src1$$reg),
11750 as_Register($src2$$reg),
11751 Assembler::LSR,
11752 $src3$$constant & 0x3f);
11753 %}
11754
11755 ins_pipe(ialu_reg_reg_shift);
11756 %}
11757
11758 // This pattern is automatically generated from aarch64_ad.m4
11759 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11760 instruct OrI_reg_RShift_reg(iRegINoSp dst,
11761 iRegIorL2I src1, iRegIorL2I src2,
11762 immI src3) %{
11763 match(Set dst (OrI src1 (RShiftI src2 src3)));
11764
11765 ins_cost(1.9 * INSN_COST);
11766 format %{ "orrw $dst, $src1, $src2, ASR $src3" %}
11767
11768 ins_encode %{
11769 __ orrw(as_Register($dst$$reg),
11770 as_Register($src1$$reg),
11771 as_Register($src2$$reg),
11772 Assembler::ASR,
11773 $src3$$constant & 0x1f);
11774 %}
11775
11776 ins_pipe(ialu_reg_reg_shift);
11777 %}
11778
11779 // This pattern is automatically generated from aarch64_ad.m4
11780 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11781 instruct OrL_reg_RShift_reg(iRegLNoSp dst,
11782 iRegL src1, iRegL src2,
11783 immI src3) %{
11784 match(Set dst (OrL src1 (RShiftL src2 src3)));
11785
11786 ins_cost(1.9 * INSN_COST);
11787 format %{ "orr $dst, $src1, $src2, ASR $src3" %}
11788
11789 ins_encode %{
11790 __ orr(as_Register($dst$$reg),
11791 as_Register($src1$$reg),
11792 as_Register($src2$$reg),
11793 Assembler::ASR,
11794 $src3$$constant & 0x3f);
11795 %}
11796
11797 ins_pipe(ialu_reg_reg_shift);
11798 %}
11799
11800 // This pattern is automatically generated from aarch64_ad.m4
11801 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11802 instruct OrI_reg_LShift_reg(iRegINoSp dst,
11803 iRegIorL2I src1, iRegIorL2I src2,
11804 immI src3) %{
11805 match(Set dst (OrI src1 (LShiftI src2 src3)));
11806
11807 ins_cost(1.9 * INSN_COST);
11808 format %{ "orrw $dst, $src1, $src2, LSL $src3" %}
11809
11810 ins_encode %{
11811 __ orrw(as_Register($dst$$reg),
11812 as_Register($src1$$reg),
11813 as_Register($src2$$reg),
11814 Assembler::LSL,
11815 $src3$$constant & 0x1f);
11816 %}
11817
11818 ins_pipe(ialu_reg_reg_shift);
11819 %}
11820
11821 // This pattern is automatically generated from aarch64_ad.m4
11822 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11823 instruct OrL_reg_LShift_reg(iRegLNoSp dst,
11824 iRegL src1, iRegL src2,
11825 immI src3) %{
11826 match(Set dst (OrL src1 (LShiftL src2 src3)));
11827
11828 ins_cost(1.9 * INSN_COST);
11829 format %{ "orr $dst, $src1, $src2, LSL $src3" %}
11830
11831 ins_encode %{
11832 __ orr(as_Register($dst$$reg),
11833 as_Register($src1$$reg),
11834 as_Register($src2$$reg),
11835 Assembler::LSL,
11836 $src3$$constant & 0x3f);
11837 %}
11838
11839 ins_pipe(ialu_reg_reg_shift);
11840 %}
11841
11842 // This pattern is automatically generated from aarch64_ad.m4
11843 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11844 instruct OrI_reg_RotateRight_reg(iRegINoSp dst,
11845 iRegIorL2I src1, iRegIorL2I src2,
11846 immI src3) %{
11847 match(Set dst (OrI src1 (RotateRight src2 src3)));
11848
11849 ins_cost(1.9 * INSN_COST);
11850 format %{ "orrw $dst, $src1, $src2, ROR $src3" %}
11851
11852 ins_encode %{
11853 __ orrw(as_Register($dst$$reg),
11854 as_Register($src1$$reg),
11855 as_Register($src2$$reg),
11856 Assembler::ROR,
11857 $src3$$constant & 0x1f);
11858 %}
11859
11860 ins_pipe(ialu_reg_reg_shift);
11861 %}
11862
11863 // This pattern is automatically generated from aarch64_ad.m4
11864 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11865 instruct OrL_reg_RotateRight_reg(iRegLNoSp dst,
11866 iRegL src1, iRegL src2,
11867 immI src3) %{
11868 match(Set dst (OrL src1 (RotateRight src2 src3)));
11869
11870 ins_cost(1.9 * INSN_COST);
11871 format %{ "orr $dst, $src1, $src2, ROR $src3" %}
11872
11873 ins_encode %{
11874 __ orr(as_Register($dst$$reg),
11875 as_Register($src1$$reg),
11876 as_Register($src2$$reg),
11877 Assembler::ROR,
11878 $src3$$constant & 0x3f);
11879 %}
11880
11881 ins_pipe(ialu_reg_reg_shift);
11882 %}
11883
11884 // This pattern is automatically generated from aarch64_ad.m4
11885 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11886 instruct AddI_reg_URShift_reg(iRegINoSp dst,
11887 iRegIorL2I src1, iRegIorL2I src2,
11888 immI src3) %{
11889 match(Set dst (AddI src1 (URShiftI src2 src3)));
11890
11891 ins_cost(1.9 * INSN_COST);
11892 format %{ "addw $dst, $src1, $src2, LSR $src3" %}
11893
11894 ins_encode %{
11895 __ addw(as_Register($dst$$reg),
11896 as_Register($src1$$reg),
11897 as_Register($src2$$reg),
11898 Assembler::LSR,
11899 $src3$$constant & 0x1f);
11900 %}
11901
11902 ins_pipe(ialu_reg_reg_shift);
11903 %}
11904
11905 // This pattern is automatically generated from aarch64_ad.m4
11906 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11907 instruct AddL_reg_URShift_reg(iRegLNoSp dst,
11908 iRegL src1, iRegL src2,
11909 immI src3) %{
11910 match(Set dst (AddL src1 (URShiftL src2 src3)));
11911
11912 ins_cost(1.9 * INSN_COST);
11913 format %{ "add $dst, $src1, $src2, LSR $src3" %}
11914
11915 ins_encode %{
11916 __ add(as_Register($dst$$reg),
11917 as_Register($src1$$reg),
11918 as_Register($src2$$reg),
11919 Assembler::LSR,
11920 $src3$$constant & 0x3f);
11921 %}
11922
11923 ins_pipe(ialu_reg_reg_shift);
11924 %}
11925
11926 // This pattern is automatically generated from aarch64_ad.m4
11927 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11928 instruct AddI_reg_RShift_reg(iRegINoSp dst,
11929 iRegIorL2I src1, iRegIorL2I src2,
11930 immI src3) %{
11931 match(Set dst (AddI src1 (RShiftI src2 src3)));
11932
11933 ins_cost(1.9 * INSN_COST);
11934 format %{ "addw $dst, $src1, $src2, ASR $src3" %}
11935
11936 ins_encode %{
11937 __ addw(as_Register($dst$$reg),
11938 as_Register($src1$$reg),
11939 as_Register($src2$$reg),
11940 Assembler::ASR,
11941 $src3$$constant & 0x1f);
11942 %}
11943
11944 ins_pipe(ialu_reg_reg_shift);
11945 %}
11946
11947 // This pattern is automatically generated from aarch64_ad.m4
11948 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11949 instruct AddL_reg_RShift_reg(iRegLNoSp dst,
11950 iRegL src1, iRegL src2,
11951 immI src3) %{
11952 match(Set dst (AddL src1 (RShiftL src2 src3)));
11953
11954 ins_cost(1.9 * INSN_COST);
11955 format %{ "add $dst, $src1, $src2, ASR $src3" %}
11956
11957 ins_encode %{
11958 __ add(as_Register($dst$$reg),
11959 as_Register($src1$$reg),
11960 as_Register($src2$$reg),
11961 Assembler::ASR,
11962 $src3$$constant & 0x3f);
11963 %}
11964
11965 ins_pipe(ialu_reg_reg_shift);
11966 %}
11967
11968 // This pattern is automatically generated from aarch64_ad.m4
11969 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11970 instruct AddI_reg_LShift_reg(iRegINoSp dst,
11971 iRegIorL2I src1, iRegIorL2I src2,
11972 immI src3) %{
11973 match(Set dst (AddI src1 (LShiftI src2 src3)));
11974
11975 ins_cost(1.9 * INSN_COST);
11976 format %{ "addw $dst, $src1, $src2, LSL $src3" %}
11977
11978 ins_encode %{
11979 __ addw(as_Register($dst$$reg),
11980 as_Register($src1$$reg),
11981 as_Register($src2$$reg),
11982 Assembler::LSL,
11983 $src3$$constant & 0x1f);
11984 %}
11985
11986 ins_pipe(ialu_reg_reg_shift);
11987 %}
11988
11989 // This pattern is automatically generated from aarch64_ad.m4
11990 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
11991 instruct AddL_reg_LShift_reg(iRegLNoSp dst,
11992 iRegL src1, iRegL src2,
11993 immI src3) %{
11994 match(Set dst (AddL src1 (LShiftL src2 src3)));
11995
11996 ins_cost(1.9 * INSN_COST);
11997 format %{ "add $dst, $src1, $src2, LSL $src3" %}
11998
11999 ins_encode %{
12000 __ add(as_Register($dst$$reg),
12001 as_Register($src1$$reg),
12002 as_Register($src2$$reg),
12003 Assembler::LSL,
12004 $src3$$constant & 0x3f);
12005 %}
12006
12007 ins_pipe(ialu_reg_reg_shift);
12008 %}
12009
12010 // This pattern is automatically generated from aarch64_ad.m4
12011 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12012 instruct SubI_reg_URShift_reg(iRegINoSp dst,
12013 iRegIorL2I src1, iRegIorL2I src2,
12014 immI src3) %{
12015 match(Set dst (SubI src1 (URShiftI src2 src3)));
12016
12017 ins_cost(1.9 * INSN_COST);
12018 format %{ "subw $dst, $src1, $src2, LSR $src3" %}
12019
12020 ins_encode %{
12021 __ subw(as_Register($dst$$reg),
12022 as_Register($src1$$reg),
12023 as_Register($src2$$reg),
12024 Assembler::LSR,
12025 $src3$$constant & 0x1f);
12026 %}
12027
12028 ins_pipe(ialu_reg_reg_shift);
12029 %}
12030
12031 // This pattern is automatically generated from aarch64_ad.m4
12032 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12033 instruct SubL_reg_URShift_reg(iRegLNoSp dst,
12034 iRegL src1, iRegL src2,
12035 immI src3) %{
12036 match(Set dst (SubL src1 (URShiftL src2 src3)));
12037
12038 ins_cost(1.9 * INSN_COST);
12039 format %{ "sub $dst, $src1, $src2, LSR $src3" %}
12040
12041 ins_encode %{
12042 __ sub(as_Register($dst$$reg),
12043 as_Register($src1$$reg),
12044 as_Register($src2$$reg),
12045 Assembler::LSR,
12046 $src3$$constant & 0x3f);
12047 %}
12048
12049 ins_pipe(ialu_reg_reg_shift);
12050 %}
12051
12052 // This pattern is automatically generated from aarch64_ad.m4
12053 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12054 instruct SubI_reg_RShift_reg(iRegINoSp dst,
12055 iRegIorL2I src1, iRegIorL2I src2,
12056 immI src3) %{
12057 match(Set dst (SubI src1 (RShiftI src2 src3)));
12058
12059 ins_cost(1.9 * INSN_COST);
12060 format %{ "subw $dst, $src1, $src2, ASR $src3" %}
12061
12062 ins_encode %{
12063 __ subw(as_Register($dst$$reg),
12064 as_Register($src1$$reg),
12065 as_Register($src2$$reg),
12066 Assembler::ASR,
12067 $src3$$constant & 0x1f);
12068 %}
12069
12070 ins_pipe(ialu_reg_reg_shift);
12071 %}
12072
12073 // This pattern is automatically generated from aarch64_ad.m4
12074 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12075 instruct SubL_reg_RShift_reg(iRegLNoSp dst,
12076 iRegL src1, iRegL src2,
12077 immI src3) %{
12078 match(Set dst (SubL src1 (RShiftL src2 src3)));
12079
12080 ins_cost(1.9 * INSN_COST);
12081 format %{ "sub $dst, $src1, $src2, ASR $src3" %}
12082
12083 ins_encode %{
12084 __ sub(as_Register($dst$$reg),
12085 as_Register($src1$$reg),
12086 as_Register($src2$$reg),
12087 Assembler::ASR,
12088 $src3$$constant & 0x3f);
12089 %}
12090
12091 ins_pipe(ialu_reg_reg_shift);
12092 %}
12093
12094 // This pattern is automatically generated from aarch64_ad.m4
12095 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12096 instruct SubI_reg_LShift_reg(iRegINoSp dst,
12097 iRegIorL2I src1, iRegIorL2I src2,
12098 immI src3) %{
12099 match(Set dst (SubI src1 (LShiftI src2 src3)));
12100
12101 ins_cost(1.9 * INSN_COST);
12102 format %{ "subw $dst, $src1, $src2, LSL $src3" %}
12103
12104 ins_encode %{
12105 __ subw(as_Register($dst$$reg),
12106 as_Register($src1$$reg),
12107 as_Register($src2$$reg),
12108 Assembler::LSL,
12109 $src3$$constant & 0x1f);
12110 %}
12111
12112 ins_pipe(ialu_reg_reg_shift);
12113 %}
12114
12115 // This pattern is automatically generated from aarch64_ad.m4
12116 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12117 instruct SubL_reg_LShift_reg(iRegLNoSp dst,
12118 iRegL src1, iRegL src2,
12119 immI src3) %{
12120 match(Set dst (SubL src1 (LShiftL src2 src3)));
12121
12122 ins_cost(1.9 * INSN_COST);
12123 format %{ "sub $dst, $src1, $src2, LSL $src3" %}
12124
12125 ins_encode %{
12126 __ sub(as_Register($dst$$reg),
12127 as_Register($src1$$reg),
12128 as_Register($src2$$reg),
12129 Assembler::LSL,
12130 $src3$$constant & 0x3f);
12131 %}
12132
12133 ins_pipe(ialu_reg_reg_shift);
12134 %}
12135
12136 // This pattern is automatically generated from aarch64_ad.m4
12137 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12138
12139 // Shift Left followed by Shift Right.
12140 // This idiom is used by the compiler for the i2b bytecode etc.
12141 instruct sbfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12142 %{
12143 match(Set dst (RShiftL (LShiftL src lshift_count) rshift_count));
12144 ins_cost(INSN_COST * 2);
12145 format %{ "sbfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12146 ins_encode %{
12147 int lshift = $lshift_count$$constant & 63;
12148 int rshift = $rshift_count$$constant & 63;
12149 int s = 63 - lshift;
12150 int r = (rshift - lshift) & 63;
12151 __ sbfm(as_Register($dst$$reg),
12152 as_Register($src$$reg),
12153 r, s);
12154 %}
12155
12156 ins_pipe(ialu_reg_shift);
12157 %}
12158
12159 // This pattern is automatically generated from aarch64_ad.m4
12160 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12161
12162 // Shift Left followed by Shift Right.
12163 // This idiom is used by the compiler for the i2b bytecode etc.
12164 instruct sbfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12165 %{
12166 match(Set dst (RShiftI (LShiftI src lshift_count) rshift_count));
12167 ins_cost(INSN_COST * 2);
12168 format %{ "sbfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12169 ins_encode %{
12170 int lshift = $lshift_count$$constant & 31;
12171 int rshift = $rshift_count$$constant & 31;
12172 int s = 31 - lshift;
12173 int r = (rshift - lshift) & 31;
12174 __ sbfmw(as_Register($dst$$reg),
12175 as_Register($src$$reg),
12176 r, s);
12177 %}
12178
12179 ins_pipe(ialu_reg_shift);
12180 %}
12181
12182 // This pattern is automatically generated from aarch64_ad.m4
12183 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12184
12185 // Shift Left followed by Shift Right.
12186 // This idiom is used by the compiler for the i2b bytecode etc.
12187 instruct ubfmL(iRegLNoSp dst, iRegL src, immI lshift_count, immI rshift_count)
12188 %{
12189 match(Set dst (URShiftL (LShiftL src lshift_count) rshift_count));
12190 ins_cost(INSN_COST * 2);
12191 format %{ "ubfm $dst, $src, $rshift_count - $lshift_count, #63 - $lshift_count" %}
12192 ins_encode %{
12193 int lshift = $lshift_count$$constant & 63;
12194 int rshift = $rshift_count$$constant & 63;
12195 int s = 63 - lshift;
12196 int r = (rshift - lshift) & 63;
12197 __ ubfm(as_Register($dst$$reg),
12198 as_Register($src$$reg),
12199 r, s);
12200 %}
12201
12202 ins_pipe(ialu_reg_shift);
12203 %}
12204
12205 // This pattern is automatically generated from aarch64_ad.m4
12206 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12207
12208 // Shift Left followed by Shift Right.
12209 // This idiom is used by the compiler for the i2b bytecode etc.
12210 instruct ubfmwI(iRegINoSp dst, iRegIorL2I src, immI lshift_count, immI rshift_count)
12211 %{
12212 match(Set dst (URShiftI (LShiftI src lshift_count) rshift_count));
12213 ins_cost(INSN_COST * 2);
12214 format %{ "ubfmw $dst, $src, $rshift_count - $lshift_count, #31 - $lshift_count" %}
12215 ins_encode %{
12216 int lshift = $lshift_count$$constant & 31;
12217 int rshift = $rshift_count$$constant & 31;
12218 int s = 31 - lshift;
12219 int r = (rshift - lshift) & 31;
12220 __ ubfmw(as_Register($dst$$reg),
12221 as_Register($src$$reg),
12222 r, s);
12223 %}
12224
12225 ins_pipe(ialu_reg_shift);
12226 %}
12227
12228 // Bitfield extract with shift & mask
12229
12230 // This pattern is automatically generated from aarch64_ad.m4
12231 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12232 instruct ubfxwI(iRegINoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12233 %{
12234 match(Set dst (AndI (URShiftI src rshift) mask));
12235 // Make sure we are not going to exceed what ubfxw can do.
12236 predicate((exact_log2(n->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12237
12238 ins_cost(INSN_COST);
12239 format %{ "ubfxw $dst, $src, $rshift, $mask" %}
12240 ins_encode %{
12241 int rshift = $rshift$$constant & 31;
12242 intptr_t mask = $mask$$constant;
12243 int width = exact_log2(mask+1);
12244 __ ubfxw(as_Register($dst$$reg),
12245 as_Register($src$$reg), rshift, width);
12246 %}
12247 ins_pipe(ialu_reg_shift);
12248 %}
12249
12250 // This pattern is automatically generated from aarch64_ad.m4
12251 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12252 instruct ubfxL(iRegLNoSp dst, iRegL src, immI rshift, immL_bitmask mask)
12253 %{
12254 match(Set dst (AndL (URShiftL src rshift) mask));
12255 // Make sure we are not going to exceed what ubfx can do.
12256 predicate((exact_log2_long(n->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= (63 + 1));
12257
12258 ins_cost(INSN_COST);
12259 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12260 ins_encode %{
12261 int rshift = $rshift$$constant & 63;
12262 intptr_t mask = $mask$$constant;
12263 int width = exact_log2_long(mask+1);
12264 __ ubfx(as_Register($dst$$reg),
12265 as_Register($src$$reg), rshift, width);
12266 %}
12267 ins_pipe(ialu_reg_shift);
12268 %}
12269
12270
12271 // This pattern is automatically generated from aarch64_ad.m4
12272 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12273
12274 // We can use ubfx when extending an And with a mask when we know mask
12275 // is positive. We know that because immI_bitmask guarantees it.
12276 instruct ubfxIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI rshift, immI_bitmask mask)
12277 %{
12278 match(Set dst (ConvI2L (AndI (URShiftI src rshift) mask)));
12279 // Make sure we are not going to exceed what ubfxw can do.
12280 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(1)->in(2)->get_int() & 31)) <= (31 + 1));
12281
12282 ins_cost(INSN_COST * 2);
12283 format %{ "ubfx $dst, $src, $rshift, $mask" %}
12284 ins_encode %{
12285 int rshift = $rshift$$constant & 31;
12286 intptr_t mask = $mask$$constant;
12287 int width = exact_log2(mask+1);
12288 __ ubfx(as_Register($dst$$reg),
12289 as_Register($src$$reg), rshift, width);
12290 %}
12291 ins_pipe(ialu_reg_shift);
12292 %}
12293
12294
12295 // This pattern is automatically generated from aarch64_ad.m4
12296 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12297
12298 // We can use ubfiz when masking by a positive number and then left shifting the result.
12299 // We know that the mask is positive because immI_bitmask guarantees it.
12300 instruct ubfizwI(iRegINoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12301 %{
12302 match(Set dst (LShiftI (AndI src mask) lshift));
12303 predicate((exact_log2(n->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 31)) <= (31 + 1));
12304
12305 ins_cost(INSN_COST);
12306 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12307 ins_encode %{
12308 int lshift = $lshift$$constant & 31;
12309 intptr_t mask = $mask$$constant;
12310 int width = exact_log2(mask+1);
12311 __ ubfizw(as_Register($dst$$reg),
12312 as_Register($src$$reg), lshift, width);
12313 %}
12314 ins_pipe(ialu_reg_shift);
12315 %}
12316
12317 // This pattern is automatically generated from aarch64_ad.m4
12318 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12319
12320 // We can use ubfiz when masking by a positive number and then left shifting the result.
12321 // We know that the mask is positive because immL_bitmask guarantees it.
12322 instruct ubfizL(iRegLNoSp dst, iRegL src, immI lshift, immL_bitmask mask)
12323 %{
12324 match(Set dst (LShiftL (AndL src mask) lshift));
12325 predicate((exact_log2_long(n->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12326
12327 ins_cost(INSN_COST);
12328 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12329 ins_encode %{
12330 int lshift = $lshift$$constant & 63;
12331 intptr_t mask = $mask$$constant;
12332 int width = exact_log2_long(mask+1);
12333 __ ubfiz(as_Register($dst$$reg),
12334 as_Register($src$$reg), lshift, width);
12335 %}
12336 ins_pipe(ialu_reg_shift);
12337 %}
12338
12339 // This pattern is automatically generated from aarch64_ad.m4
12340 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12341
12342 // We can use ubfiz when masking by a positive number and then left shifting the result.
12343 // We know that the mask is positive because immI_bitmask guarantees it.
12344 instruct ubfizwIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12345 %{
12346 match(Set dst (ConvI2L (LShiftI (AndI src mask) lshift)));
12347 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(1)->in(2)->get_int() & 31)) <= 31);
12348
12349 ins_cost(INSN_COST);
12350 format %{ "ubfizw $dst, $src, $lshift, $mask" %}
12351 ins_encode %{
12352 int lshift = $lshift$$constant & 31;
12353 intptr_t mask = $mask$$constant;
12354 int width = exact_log2(mask+1);
12355 __ ubfizw(as_Register($dst$$reg),
12356 as_Register($src$$reg), lshift, width);
12357 %}
12358 ins_pipe(ialu_reg_shift);
12359 %}
12360
12361 // This pattern is automatically generated from aarch64_ad.m4
12362 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12363
12364 // We can use ubfiz when masking by a positive number and then left shifting the result.
12365 // We know that the mask is positive because immL_bitmask guarantees it.
12366 instruct ubfizLConvL2I(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12367 %{
12368 match(Set dst (ConvL2I (LShiftL (AndL src mask) lshift)));
12369 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(1)->in(2)->get_int() & 63)) <= 31);
12370
12371 ins_cost(INSN_COST);
12372 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12373 ins_encode %{
12374 int lshift = $lshift$$constant & 63;
12375 intptr_t mask = $mask$$constant;
12376 int width = exact_log2_long(mask+1);
12377 __ ubfiz(as_Register($dst$$reg),
12378 as_Register($src$$reg), lshift, width);
12379 %}
12380 ins_pipe(ialu_reg_shift);
12381 %}
12382
12383
12384 // This pattern is automatically generated from aarch64_ad.m4
12385 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12386
12387 // If there is a convert I to L block between and AndI and a LShiftL, we can also match ubfiz
12388 instruct ubfizIConvI2L(iRegLNoSp dst, iRegIorL2I src, immI lshift, immI_bitmask mask)
12389 %{
12390 match(Set dst (LShiftL (ConvI2L (AndI src mask)) lshift));
12391 predicate((exact_log2(n->in(1)->in(1)->in(2)->get_int() + 1) + (n->in(2)->get_int() & 63)) <= (63 + 1));
12392
12393 ins_cost(INSN_COST);
12394 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12395 ins_encode %{
12396 int lshift = $lshift$$constant & 63;
12397 intptr_t mask = $mask$$constant;
12398 int width = exact_log2(mask+1);
12399 __ ubfiz(as_Register($dst$$reg),
12400 as_Register($src$$reg), lshift, width);
12401 %}
12402 ins_pipe(ialu_reg_shift);
12403 %}
12404
12405 // This pattern is automatically generated from aarch64_ad.m4
12406 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12407
12408 // If there is a convert L to I block between and AndL and a LShiftI, we can also match ubfiz
12409 instruct ubfizLConvL2Ix(iRegINoSp dst, iRegL src, immI lshift, immL_positive_bitmaskI mask)
12410 %{
12411 match(Set dst (LShiftI (ConvL2I (AndL src mask)) lshift));
12412 predicate((exact_log2_long(n->in(1)->in(1)->in(2)->get_long() + 1) + (n->in(2)->get_int() & 31)) <= 31);
12413
12414 ins_cost(INSN_COST);
12415 format %{ "ubfiz $dst, $src, $lshift, $mask" %}
12416 ins_encode %{
12417 int lshift = $lshift$$constant & 31;
12418 intptr_t mask = $mask$$constant;
12419 int width = exact_log2(mask+1);
12420 __ ubfiz(as_Register($dst$$reg),
12421 as_Register($src$$reg), lshift, width);
12422 %}
12423 ins_pipe(ialu_reg_shift);
12424 %}
12425
12426 // This pattern is automatically generated from aarch64_ad.m4
12427 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12428
12429 // Can skip int2long conversions after AND with small bitmask
12430 instruct ubfizIConvI2LAndI(iRegLNoSp dst, iRegI src, immI_bitmask msk)
12431 %{
12432 match(Set dst (ConvI2L (AndI src msk)));
12433 ins_cost(INSN_COST);
12434 format %{ "ubfiz $dst, $src, 0, exact_log2($msk + 1) " %}
12435 ins_encode %{
12436 __ ubfiz(as_Register($dst$$reg), as_Register($src$$reg), 0, exact_log2($msk$$constant + 1));
12437 %}
12438 ins_pipe(ialu_reg_shift);
12439 %}
12440
12441
12442 // Rotations
12443
12444 // This pattern is automatically generated from aarch64_ad.m4
12445 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12446 instruct extrOrL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12447 %{
12448 match(Set dst (OrL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12449 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12450
12451 ins_cost(INSN_COST);
12452 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12453
12454 ins_encode %{
12455 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12456 $rshift$$constant & 63);
12457 %}
12458 ins_pipe(ialu_reg_reg_extr);
12459 %}
12460
12461
12462 // This pattern is automatically generated from aarch64_ad.m4
12463 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12464 instruct extrOrI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12465 %{
12466 match(Set dst (OrI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12467 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12468
12469 ins_cost(INSN_COST);
12470 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12471
12472 ins_encode %{
12473 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12474 $rshift$$constant & 31);
12475 %}
12476 ins_pipe(ialu_reg_reg_extr);
12477 %}
12478
12479
12480 // This pattern is automatically generated from aarch64_ad.m4
12481 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12482 instruct extrAddL(iRegLNoSp dst, iRegL src1, iRegL src2, immI lshift, immI rshift, rFlagsReg cr)
12483 %{
12484 match(Set dst (AddL (LShiftL src1 lshift) (URShiftL src2 rshift)));
12485 predicate(0 == (((n->in(1)->in(2)->get_int() & 63) + (n->in(2)->in(2)->get_int() & 63)) & 63));
12486
12487 ins_cost(INSN_COST);
12488 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12489
12490 ins_encode %{
12491 __ extr(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12492 $rshift$$constant & 63);
12493 %}
12494 ins_pipe(ialu_reg_reg_extr);
12495 %}
12496
12497
12498 // This pattern is automatically generated from aarch64_ad.m4
12499 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12500 instruct extrAddI(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI lshift, immI rshift, rFlagsReg cr)
12501 %{
12502 match(Set dst (AddI (LShiftI src1 lshift) (URShiftI src2 rshift)));
12503 predicate(0 == (((n->in(1)->in(2)->get_int() & 31) + (n->in(2)->in(2)->get_int() & 31)) & 31));
12504
12505 ins_cost(INSN_COST);
12506 format %{ "extr $dst, $src1, $src2, #$rshift" %}
12507
12508 ins_encode %{
12509 __ extrw(as_Register($dst$$reg), as_Register($src1$$reg), as_Register($src2$$reg),
12510 $rshift$$constant & 31);
12511 %}
12512 ins_pipe(ialu_reg_reg_extr);
12513 %}
12514
12515 // This pattern is automatically generated from aarch64_ad.m4
12516 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12517 instruct rorI_imm(iRegINoSp dst, iRegI src, immI shift)
12518 %{
12519 match(Set dst (RotateRight src shift));
12520
12521 ins_cost(INSN_COST);
12522 format %{ "ror $dst, $src, $shift" %}
12523
12524 ins_encode %{
12525 __ extrw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12526 $shift$$constant & 0x1f);
12527 %}
12528 ins_pipe(ialu_reg_reg_vshift);
12529 %}
12530
12531 // This pattern is automatically generated from aarch64_ad.m4
12532 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12533 instruct rorL_imm(iRegLNoSp dst, iRegL src, immI shift)
12534 %{
12535 match(Set dst (RotateRight src shift));
12536
12537 ins_cost(INSN_COST);
12538 format %{ "ror $dst, $src, $shift" %}
12539
12540 ins_encode %{
12541 __ extr(as_Register($dst$$reg), as_Register($src$$reg), as_Register($src$$reg),
12542 $shift$$constant & 0x3f);
12543 %}
12544 ins_pipe(ialu_reg_reg_vshift);
12545 %}
12546
12547 // This pattern is automatically generated from aarch64_ad.m4
12548 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12549 instruct rorI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12550 %{
12551 match(Set dst (RotateRight src shift));
12552
12553 ins_cost(INSN_COST);
12554 format %{ "ror $dst, $src, $shift" %}
12555
12556 ins_encode %{
12557 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12558 %}
12559 ins_pipe(ialu_reg_reg_vshift);
12560 %}
12561
12562 // This pattern is automatically generated from aarch64_ad.m4
12563 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12564 instruct rorL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12565 %{
12566 match(Set dst (RotateRight src shift));
12567
12568 ins_cost(INSN_COST);
12569 format %{ "ror $dst, $src, $shift" %}
12570
12571 ins_encode %{
12572 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), as_Register($shift$$reg));
12573 %}
12574 ins_pipe(ialu_reg_reg_vshift);
12575 %}
12576
12577 // This pattern is automatically generated from aarch64_ad.m4
12578 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12579 instruct rolI_reg(iRegINoSp dst, iRegI src, iRegI shift)
12580 %{
12581 match(Set dst (RotateLeft src shift));
12582
12583 ins_cost(INSN_COST);
12584 format %{ "rol $dst, $src, $shift" %}
12585
12586 ins_encode %{
12587 __ subw(rscratch1, zr, as_Register($shift$$reg));
12588 __ rorvw(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12589 %}
12590 ins_pipe(ialu_reg_reg_vshift);
12591 %}
12592
12593 // This pattern is automatically generated from aarch64_ad.m4
12594 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12595 instruct rolL_reg(iRegLNoSp dst, iRegL src, iRegI shift)
12596 %{
12597 match(Set dst (RotateLeft src shift));
12598
12599 ins_cost(INSN_COST);
12600 format %{ "rol $dst, $src, $shift" %}
12601
12602 ins_encode %{
12603 __ subw(rscratch1, zr, as_Register($shift$$reg));
12604 __ rorv(as_Register($dst$$reg), as_Register($src$$reg), rscratch1);
12605 %}
12606 ins_pipe(ialu_reg_reg_vshift);
12607 %}
12608
12609
12610 // Add/subtract (extended)
12611
12612 // This pattern is automatically generated from aarch64_ad.m4
12613 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12614 instruct AddExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12615 %{
12616 match(Set dst (AddL src1 (ConvI2L src2)));
12617 ins_cost(INSN_COST);
12618 format %{ "add $dst, $src1, $src2, sxtw" %}
12619
12620 ins_encode %{
12621 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12622 as_Register($src2$$reg), ext::sxtw);
12623 %}
12624 ins_pipe(ialu_reg_reg);
12625 %}
12626
12627 // This pattern is automatically generated from aarch64_ad.m4
12628 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12629 instruct SubExtI(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, rFlagsReg cr)
12630 %{
12631 match(Set dst (SubL src1 (ConvI2L src2)));
12632 ins_cost(INSN_COST);
12633 format %{ "sub $dst, $src1, $src2, sxtw" %}
12634
12635 ins_encode %{
12636 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12637 as_Register($src2$$reg), ext::sxtw);
12638 %}
12639 ins_pipe(ialu_reg_reg);
12640 %}
12641
12642 // This pattern is automatically generated from aarch64_ad.m4
12643 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12644 instruct AddExtI_sxth(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_16 lshift, immI_16 rshift, rFlagsReg cr)
12645 %{
12646 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12647 ins_cost(INSN_COST);
12648 format %{ "add $dst, $src1, $src2, sxth" %}
12649
12650 ins_encode %{
12651 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12652 as_Register($src2$$reg), ext::sxth);
12653 %}
12654 ins_pipe(ialu_reg_reg);
12655 %}
12656
12657 // This pattern is automatically generated from aarch64_ad.m4
12658 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12659 instruct AddExtI_sxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12660 %{
12661 match(Set dst (AddI src1 (RShiftI (LShiftI src2 lshift) rshift)));
12662 ins_cost(INSN_COST);
12663 format %{ "add $dst, $src1, $src2, sxtb" %}
12664
12665 ins_encode %{
12666 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12667 as_Register($src2$$reg), ext::sxtb);
12668 %}
12669 ins_pipe(ialu_reg_reg);
12670 %}
12671
12672 // This pattern is automatically generated from aarch64_ad.m4
12673 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12674 instruct AddExtI_uxtb(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_24 lshift, immI_24 rshift, rFlagsReg cr)
12675 %{
12676 match(Set dst (AddI src1 (URShiftI (LShiftI src2 lshift) rshift)));
12677 ins_cost(INSN_COST);
12678 format %{ "add $dst, $src1, $src2, uxtb" %}
12679
12680 ins_encode %{
12681 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12682 as_Register($src2$$reg), ext::uxtb);
12683 %}
12684 ins_pipe(ialu_reg_reg);
12685 %}
12686
12687 // This pattern is automatically generated from aarch64_ad.m4
12688 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12689 instruct AddExtL_sxth(iRegLNoSp dst, iRegL src1, iRegL src2, immI_48 lshift, immI_48 rshift, rFlagsReg cr)
12690 %{
12691 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12692 ins_cost(INSN_COST);
12693 format %{ "add $dst, $src1, $src2, sxth" %}
12694
12695 ins_encode %{
12696 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12697 as_Register($src2$$reg), ext::sxth);
12698 %}
12699 ins_pipe(ialu_reg_reg);
12700 %}
12701
12702 // This pattern is automatically generated from aarch64_ad.m4
12703 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12704 instruct AddExtL_sxtw(iRegLNoSp dst, iRegL src1, iRegL src2, immI_32 lshift, immI_32 rshift, rFlagsReg cr)
12705 %{
12706 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12707 ins_cost(INSN_COST);
12708 format %{ "add $dst, $src1, $src2, sxtw" %}
12709
12710 ins_encode %{
12711 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12712 as_Register($src2$$reg), ext::sxtw);
12713 %}
12714 ins_pipe(ialu_reg_reg);
12715 %}
12716
12717 // This pattern is automatically generated from aarch64_ad.m4
12718 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12719 instruct AddExtL_sxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12720 %{
12721 match(Set dst (AddL src1 (RShiftL (LShiftL src2 lshift) rshift)));
12722 ins_cost(INSN_COST);
12723 format %{ "add $dst, $src1, $src2, sxtb" %}
12724
12725 ins_encode %{
12726 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12727 as_Register($src2$$reg), ext::sxtb);
12728 %}
12729 ins_pipe(ialu_reg_reg);
12730 %}
12731
12732 // This pattern is automatically generated from aarch64_ad.m4
12733 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12734 instruct AddExtL_uxtb(iRegLNoSp dst, iRegL src1, iRegL src2, immI_56 lshift, immI_56 rshift, rFlagsReg cr)
12735 %{
12736 match(Set dst (AddL src1 (URShiftL (LShiftL src2 lshift) rshift)));
12737 ins_cost(INSN_COST);
12738 format %{ "add $dst, $src1, $src2, uxtb" %}
12739
12740 ins_encode %{
12741 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12742 as_Register($src2$$reg), ext::uxtb);
12743 %}
12744 ins_pipe(ialu_reg_reg);
12745 %}
12746
12747 // This pattern is automatically generated from aarch64_ad.m4
12748 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12749 instruct AddExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12750 %{
12751 match(Set dst (AddI src1 (AndI src2 mask)));
12752 ins_cost(INSN_COST);
12753 format %{ "addw $dst, $src1, $src2, uxtb" %}
12754
12755 ins_encode %{
12756 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12757 as_Register($src2$$reg), ext::uxtb);
12758 %}
12759 ins_pipe(ialu_reg_reg);
12760 %}
12761
12762 // This pattern is automatically generated from aarch64_ad.m4
12763 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12764 instruct AddExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12765 %{
12766 match(Set dst (AddI src1 (AndI src2 mask)));
12767 ins_cost(INSN_COST);
12768 format %{ "addw $dst, $src1, $src2, uxth" %}
12769
12770 ins_encode %{
12771 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12772 as_Register($src2$$reg), ext::uxth);
12773 %}
12774 ins_pipe(ialu_reg_reg);
12775 %}
12776
12777 // This pattern is automatically generated from aarch64_ad.m4
12778 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12779 instruct AddExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12780 %{
12781 match(Set dst (AddL src1 (AndL src2 mask)));
12782 ins_cost(INSN_COST);
12783 format %{ "add $dst, $src1, $src2, uxtb" %}
12784
12785 ins_encode %{
12786 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12787 as_Register($src2$$reg), ext::uxtb);
12788 %}
12789 ins_pipe(ialu_reg_reg);
12790 %}
12791
12792 // This pattern is automatically generated from aarch64_ad.m4
12793 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12794 instruct AddExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12795 %{
12796 match(Set dst (AddL src1 (AndL src2 mask)));
12797 ins_cost(INSN_COST);
12798 format %{ "add $dst, $src1, $src2, uxth" %}
12799
12800 ins_encode %{
12801 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12802 as_Register($src2$$reg), ext::uxth);
12803 %}
12804 ins_pipe(ialu_reg_reg);
12805 %}
12806
12807 // This pattern is automatically generated from aarch64_ad.m4
12808 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12809 instruct AddExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12810 %{
12811 match(Set dst (AddL src1 (AndL src2 mask)));
12812 ins_cost(INSN_COST);
12813 format %{ "add $dst, $src1, $src2, uxtw" %}
12814
12815 ins_encode %{
12816 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12817 as_Register($src2$$reg), ext::uxtw);
12818 %}
12819 ins_pipe(ialu_reg_reg);
12820 %}
12821
12822 // This pattern is automatically generated from aarch64_ad.m4
12823 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12824 instruct SubExtI_uxtb_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, rFlagsReg cr)
12825 %{
12826 match(Set dst (SubI src1 (AndI src2 mask)));
12827 ins_cost(INSN_COST);
12828 format %{ "subw $dst, $src1, $src2, uxtb" %}
12829
12830 ins_encode %{
12831 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12832 as_Register($src2$$reg), ext::uxtb);
12833 %}
12834 ins_pipe(ialu_reg_reg);
12835 %}
12836
12837 // This pattern is automatically generated from aarch64_ad.m4
12838 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12839 instruct SubExtI_uxth_and(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, rFlagsReg cr)
12840 %{
12841 match(Set dst (SubI src1 (AndI src2 mask)));
12842 ins_cost(INSN_COST);
12843 format %{ "subw $dst, $src1, $src2, uxth" %}
12844
12845 ins_encode %{
12846 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
12847 as_Register($src2$$reg), ext::uxth);
12848 %}
12849 ins_pipe(ialu_reg_reg);
12850 %}
12851
12852 // This pattern is automatically generated from aarch64_ad.m4
12853 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12854 instruct SubExtL_uxtb_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, rFlagsReg cr)
12855 %{
12856 match(Set dst (SubL src1 (AndL src2 mask)));
12857 ins_cost(INSN_COST);
12858 format %{ "sub $dst, $src1, $src2, uxtb" %}
12859
12860 ins_encode %{
12861 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12862 as_Register($src2$$reg), ext::uxtb);
12863 %}
12864 ins_pipe(ialu_reg_reg);
12865 %}
12866
12867 // This pattern is automatically generated from aarch64_ad.m4
12868 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12869 instruct SubExtL_uxth_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, rFlagsReg cr)
12870 %{
12871 match(Set dst (SubL src1 (AndL src2 mask)));
12872 ins_cost(INSN_COST);
12873 format %{ "sub $dst, $src1, $src2, uxth" %}
12874
12875 ins_encode %{
12876 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12877 as_Register($src2$$reg), ext::uxth);
12878 %}
12879 ins_pipe(ialu_reg_reg);
12880 %}
12881
12882 // This pattern is automatically generated from aarch64_ad.m4
12883 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12884 instruct SubExtL_uxtw_and(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, rFlagsReg cr)
12885 %{
12886 match(Set dst (SubL src1 (AndL src2 mask)));
12887 ins_cost(INSN_COST);
12888 format %{ "sub $dst, $src1, $src2, uxtw" %}
12889
12890 ins_encode %{
12891 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12892 as_Register($src2$$reg), ext::uxtw);
12893 %}
12894 ins_pipe(ialu_reg_reg);
12895 %}
12896
12897
12898 // This pattern is automatically generated from aarch64_ad.m4
12899 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12900 instruct AddExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12901 %{
12902 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12903 ins_cost(1.9 * INSN_COST);
12904 format %{ "add $dst, $src1, $src2, sxtb #lshift2" %}
12905
12906 ins_encode %{
12907 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12908 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12909 %}
12910 ins_pipe(ialu_reg_reg_shift);
12911 %}
12912
12913 // This pattern is automatically generated from aarch64_ad.m4
12914 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12915 instruct AddExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12916 %{
12917 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12918 ins_cost(1.9 * INSN_COST);
12919 format %{ "add $dst, $src1, $src2, sxth #lshift2" %}
12920
12921 ins_encode %{
12922 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12923 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12924 %}
12925 ins_pipe(ialu_reg_reg_shift);
12926 %}
12927
12928 // This pattern is automatically generated from aarch64_ad.m4
12929 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12930 instruct AddExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12931 %{
12932 match(Set dst (AddL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12933 ins_cost(1.9 * INSN_COST);
12934 format %{ "add $dst, $src1, $src2, sxtw #lshift2" %}
12935
12936 ins_encode %{
12937 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
12938 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12939 %}
12940 ins_pipe(ialu_reg_reg_shift);
12941 %}
12942
12943 // This pattern is automatically generated from aarch64_ad.m4
12944 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12945 instruct SubExtL_sxtb_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_56 lshift1, immI_56 rshift1, rFlagsReg cr)
12946 %{
12947 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12948 ins_cost(1.9 * INSN_COST);
12949 format %{ "sub $dst, $src1, $src2, sxtb #lshift2" %}
12950
12951 ins_encode %{
12952 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12953 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12954 %}
12955 ins_pipe(ialu_reg_reg_shift);
12956 %}
12957
12958 // This pattern is automatically generated from aarch64_ad.m4
12959 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12960 instruct SubExtL_sxth_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_48 lshift1, immI_48 rshift1, rFlagsReg cr)
12961 %{
12962 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12963 ins_cost(1.9 * INSN_COST);
12964 format %{ "sub $dst, $src1, $src2, sxth #lshift2" %}
12965
12966 ins_encode %{
12967 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12968 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
12969 %}
12970 ins_pipe(ialu_reg_reg_shift);
12971 %}
12972
12973 // This pattern is automatically generated from aarch64_ad.m4
12974 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12975 instruct SubExtL_sxtw_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immIExt lshift2, immI_32 lshift1, immI_32 rshift1, rFlagsReg cr)
12976 %{
12977 match(Set dst (SubL src1 (LShiftL (RShiftL (LShiftL src2 lshift1) rshift1) lshift2)));
12978 ins_cost(1.9 * INSN_COST);
12979 format %{ "sub $dst, $src1, $src2, sxtw #lshift2" %}
12980
12981 ins_encode %{
12982 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
12983 as_Register($src2$$reg), ext::sxtw, ($lshift2$$constant));
12984 %}
12985 ins_pipe(ialu_reg_reg_shift);
12986 %}
12987
12988 // This pattern is automatically generated from aarch64_ad.m4
12989 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
12990 instruct AddExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
12991 %{
12992 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
12993 ins_cost(1.9 * INSN_COST);
12994 format %{ "addw $dst, $src1, $src2, sxtb #lshift2" %}
12995
12996 ins_encode %{
12997 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
12998 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
12999 %}
13000 ins_pipe(ialu_reg_reg_shift);
13001 %}
13002
13003 // This pattern is automatically generated from aarch64_ad.m4
13004 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13005 instruct AddExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13006 %{
13007 match(Set dst (AddI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13008 ins_cost(1.9 * INSN_COST);
13009 format %{ "addw $dst, $src1, $src2, sxth #lshift2" %}
13010
13011 ins_encode %{
13012 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13013 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13014 %}
13015 ins_pipe(ialu_reg_reg_shift);
13016 %}
13017
13018 // This pattern is automatically generated from aarch64_ad.m4
13019 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13020 instruct SubExtI_sxtb_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_24 lshift1, immI_24 rshift1, rFlagsReg cr)
13021 %{
13022 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13023 ins_cost(1.9 * INSN_COST);
13024 format %{ "subw $dst, $src1, $src2, sxtb #lshift2" %}
13025
13026 ins_encode %{
13027 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13028 as_Register($src2$$reg), ext::sxtb, ($lshift2$$constant));
13029 %}
13030 ins_pipe(ialu_reg_reg_shift);
13031 %}
13032
13033 // This pattern is automatically generated from aarch64_ad.m4
13034 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13035 instruct SubExtI_sxth_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immIExt lshift2, immI_16 lshift1, immI_16 rshift1, rFlagsReg cr)
13036 %{
13037 match(Set dst (SubI src1 (LShiftI (RShiftI (LShiftI src2 lshift1) rshift1) lshift2)));
13038 ins_cost(1.9 * INSN_COST);
13039 format %{ "subw $dst, $src1, $src2, sxth #lshift2" %}
13040
13041 ins_encode %{
13042 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13043 as_Register($src2$$reg), ext::sxth, ($lshift2$$constant));
13044 %}
13045 ins_pipe(ialu_reg_reg_shift);
13046 %}
13047
13048 // This pattern is automatically generated from aarch64_ad.m4
13049 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13050 instruct AddExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13051 %{
13052 match(Set dst (AddL src1 (LShiftL (ConvI2L src2) lshift)));
13053 ins_cost(1.9 * INSN_COST);
13054 format %{ "add $dst, $src1, $src2, sxtw #lshift" %}
13055
13056 ins_encode %{
13057 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13058 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13059 %}
13060 ins_pipe(ialu_reg_reg_shift);
13061 %}
13062
13063 // This pattern is automatically generated from aarch64_ad.m4
13064 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13065 instruct SubExtI_shift(iRegLNoSp dst, iRegL src1, iRegIorL2I src2, immIExt lshift, rFlagsReg cr)
13066 %{
13067 match(Set dst (SubL src1 (LShiftL (ConvI2L src2) lshift)));
13068 ins_cost(1.9 * INSN_COST);
13069 format %{ "sub $dst, $src1, $src2, sxtw #lshift" %}
13070
13071 ins_encode %{
13072 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13073 as_Register($src2$$reg), ext::sxtw, ($lshift$$constant));
13074 %}
13075 ins_pipe(ialu_reg_reg_shift);
13076 %}
13077
13078 // This pattern is automatically generated from aarch64_ad.m4
13079 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13080 instruct AddExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13081 %{
13082 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13083 ins_cost(1.9 * INSN_COST);
13084 format %{ "add $dst, $src1, $src2, uxtb #lshift" %}
13085
13086 ins_encode %{
13087 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13088 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13089 %}
13090 ins_pipe(ialu_reg_reg_shift);
13091 %}
13092
13093 // This pattern is automatically generated from aarch64_ad.m4
13094 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13095 instruct AddExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13096 %{
13097 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13098 ins_cost(1.9 * INSN_COST);
13099 format %{ "add $dst, $src1, $src2, uxth #lshift" %}
13100
13101 ins_encode %{
13102 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13103 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13104 %}
13105 ins_pipe(ialu_reg_reg_shift);
13106 %}
13107
13108 // This pattern is automatically generated from aarch64_ad.m4
13109 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13110 instruct AddExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13111 %{
13112 match(Set dst (AddL src1 (LShiftL (AndL src2 mask) lshift)));
13113 ins_cost(1.9 * INSN_COST);
13114 format %{ "add $dst, $src1, $src2, uxtw #lshift" %}
13115
13116 ins_encode %{
13117 __ add(as_Register($dst$$reg), as_Register($src1$$reg),
13118 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13119 %}
13120 ins_pipe(ialu_reg_reg_shift);
13121 %}
13122
13123 // This pattern is automatically generated from aarch64_ad.m4
13124 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13125 instruct SubExtL_uxtb_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_255 mask, immIExt lshift, rFlagsReg cr)
13126 %{
13127 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13128 ins_cost(1.9 * INSN_COST);
13129 format %{ "sub $dst, $src1, $src2, uxtb #lshift" %}
13130
13131 ins_encode %{
13132 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13133 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13134 %}
13135 ins_pipe(ialu_reg_reg_shift);
13136 %}
13137
13138 // This pattern is automatically generated from aarch64_ad.m4
13139 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13140 instruct SubExtL_uxth_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_65535 mask, immIExt lshift, rFlagsReg cr)
13141 %{
13142 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13143 ins_cost(1.9 * INSN_COST);
13144 format %{ "sub $dst, $src1, $src2, uxth #lshift" %}
13145
13146 ins_encode %{
13147 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13148 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13149 %}
13150 ins_pipe(ialu_reg_reg_shift);
13151 %}
13152
13153 // This pattern is automatically generated from aarch64_ad.m4
13154 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13155 instruct SubExtL_uxtw_and_shift(iRegLNoSp dst, iRegL src1, iRegL src2, immL_4294967295 mask, immIExt lshift, rFlagsReg cr)
13156 %{
13157 match(Set dst (SubL src1 (LShiftL (AndL src2 mask) lshift)));
13158 ins_cost(1.9 * INSN_COST);
13159 format %{ "sub $dst, $src1, $src2, uxtw #lshift" %}
13160
13161 ins_encode %{
13162 __ sub(as_Register($dst$$reg), as_Register($src1$$reg),
13163 as_Register($src2$$reg), ext::uxtw, ($lshift$$constant));
13164 %}
13165 ins_pipe(ialu_reg_reg_shift);
13166 %}
13167
13168 // This pattern is automatically generated from aarch64_ad.m4
13169 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13170 instruct AddExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13171 %{
13172 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13173 ins_cost(1.9 * INSN_COST);
13174 format %{ "addw $dst, $src1, $src2, uxtb #lshift" %}
13175
13176 ins_encode %{
13177 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13178 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13179 %}
13180 ins_pipe(ialu_reg_reg_shift);
13181 %}
13182
13183 // This pattern is automatically generated from aarch64_ad.m4
13184 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13185 instruct AddExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13186 %{
13187 match(Set dst (AddI src1 (LShiftI (AndI src2 mask) lshift)));
13188 ins_cost(1.9 * INSN_COST);
13189 format %{ "addw $dst, $src1, $src2, uxth #lshift" %}
13190
13191 ins_encode %{
13192 __ addw(as_Register($dst$$reg), as_Register($src1$$reg),
13193 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13194 %}
13195 ins_pipe(ialu_reg_reg_shift);
13196 %}
13197
13198 // This pattern is automatically generated from aarch64_ad.m4
13199 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13200 instruct SubExtI_uxtb_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_255 mask, immIExt lshift, rFlagsReg cr)
13201 %{
13202 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13203 ins_cost(1.9 * INSN_COST);
13204 format %{ "subw $dst, $src1, $src2, uxtb #lshift" %}
13205
13206 ins_encode %{
13207 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13208 as_Register($src2$$reg), ext::uxtb, ($lshift$$constant));
13209 %}
13210 ins_pipe(ialu_reg_reg_shift);
13211 %}
13212
13213 // This pattern is automatically generated from aarch64_ad.m4
13214 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13215 instruct SubExtI_uxth_and_shift(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, immI_65535 mask, immIExt lshift, rFlagsReg cr)
13216 %{
13217 match(Set dst (SubI src1 (LShiftI (AndI src2 mask) lshift)));
13218 ins_cost(1.9 * INSN_COST);
13219 format %{ "subw $dst, $src1, $src2, uxth #lshift" %}
13220
13221 ins_encode %{
13222 __ subw(as_Register($dst$$reg), as_Register($src1$$reg),
13223 as_Register($src2$$reg), ext::uxth, ($lshift$$constant));
13224 %}
13225 ins_pipe(ialu_reg_reg_shift);
13226 %}
13227
13228 // This pattern is automatically generated from aarch64_ad.m4
13229 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13230 instruct cmovI_reg_reg_lt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13231 %{
13232 effect(DEF dst, USE src1, USE src2, USE cr);
13233 ins_cost(INSN_COST * 2);
13234 format %{ "cselw $dst, $src1, $src2 lt\t" %}
13235
13236 ins_encode %{
13237 __ cselw($dst$$Register,
13238 $src1$$Register,
13239 $src2$$Register,
13240 Assembler::LT);
13241 %}
13242 ins_pipe(icond_reg_reg);
13243 %}
13244
13245 // This pattern is automatically generated from aarch64_ad.m4
13246 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13247 instruct cmovI_reg_reg_gt(iRegINoSp dst, iRegI src1, iRegI src2, rFlagsReg cr)
13248 %{
13249 effect(DEF dst, USE src1, USE src2, USE cr);
13250 ins_cost(INSN_COST * 2);
13251 format %{ "cselw $dst, $src1, $src2 gt\t" %}
13252
13253 ins_encode %{
13254 __ cselw($dst$$Register,
13255 $src1$$Register,
13256 $src2$$Register,
13257 Assembler::GT);
13258 %}
13259 ins_pipe(icond_reg_reg);
13260 %}
13261
13262 // This pattern is automatically generated from aarch64_ad.m4
13263 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13264 instruct cmovI_reg_imm0_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13265 %{
13266 effect(DEF dst, USE src1, USE cr);
13267 ins_cost(INSN_COST * 2);
13268 format %{ "cselw $dst, $src1, zr lt\t" %}
13269
13270 ins_encode %{
13271 __ cselw($dst$$Register,
13272 $src1$$Register,
13273 zr,
13274 Assembler::LT);
13275 %}
13276 ins_pipe(icond_reg);
13277 %}
13278
13279 // This pattern is automatically generated from aarch64_ad.m4
13280 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13281 instruct cmovI_reg_imm0_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13282 %{
13283 effect(DEF dst, USE src1, USE cr);
13284 ins_cost(INSN_COST * 2);
13285 format %{ "cselw $dst, $src1, zr gt\t" %}
13286
13287 ins_encode %{
13288 __ cselw($dst$$Register,
13289 $src1$$Register,
13290 zr,
13291 Assembler::GT);
13292 %}
13293 ins_pipe(icond_reg);
13294 %}
13295
13296 // This pattern is automatically generated from aarch64_ad.m4
13297 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13298 instruct cmovI_reg_imm1_le(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13299 %{
13300 effect(DEF dst, USE src1, USE cr);
13301 ins_cost(INSN_COST * 2);
13302 format %{ "csincw $dst, $src1, zr le\t" %}
13303
13304 ins_encode %{
13305 __ csincw($dst$$Register,
13306 $src1$$Register,
13307 zr,
13308 Assembler::LE);
13309 %}
13310 ins_pipe(icond_reg);
13311 %}
13312
13313 // This pattern is automatically generated from aarch64_ad.m4
13314 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13315 instruct cmovI_reg_imm1_gt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13316 %{
13317 effect(DEF dst, USE src1, USE cr);
13318 ins_cost(INSN_COST * 2);
13319 format %{ "csincw $dst, $src1, zr gt\t" %}
13320
13321 ins_encode %{
13322 __ csincw($dst$$Register,
13323 $src1$$Register,
13324 zr,
13325 Assembler::GT);
13326 %}
13327 ins_pipe(icond_reg);
13328 %}
13329
13330 // This pattern is automatically generated from aarch64_ad.m4
13331 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13332 instruct cmovI_reg_immM1_lt(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13333 %{
13334 effect(DEF dst, USE src1, USE cr);
13335 ins_cost(INSN_COST * 2);
13336 format %{ "csinvw $dst, $src1, zr lt\t" %}
13337
13338 ins_encode %{
13339 __ csinvw($dst$$Register,
13340 $src1$$Register,
13341 zr,
13342 Assembler::LT);
13343 %}
13344 ins_pipe(icond_reg);
13345 %}
13346
13347 // This pattern is automatically generated from aarch64_ad.m4
13348 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13349 instruct cmovI_reg_immM1_ge(iRegINoSp dst, iRegI src1, rFlagsReg cr)
13350 %{
13351 effect(DEF dst, USE src1, USE cr);
13352 ins_cost(INSN_COST * 2);
13353 format %{ "csinvw $dst, $src1, zr ge\t" %}
13354
13355 ins_encode %{
13356 __ csinvw($dst$$Register,
13357 $src1$$Register,
13358 zr,
13359 Assembler::GE);
13360 %}
13361 ins_pipe(icond_reg);
13362 %}
13363
13364 // This pattern is automatically generated from aarch64_ad.m4
13365 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13366 instruct minI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13367 %{
13368 match(Set dst (MinI src imm));
13369 ins_cost(INSN_COST * 3);
13370 expand %{
13371 rFlagsReg cr;
13372 compI_reg_imm0(cr, src);
13373 cmovI_reg_imm0_lt(dst, src, cr);
13374 %}
13375 %}
13376
13377 // This pattern is automatically generated from aarch64_ad.m4
13378 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13379 instruct minI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13380 %{
13381 match(Set dst (MinI imm src));
13382 ins_cost(INSN_COST * 3);
13383 expand %{
13384 rFlagsReg cr;
13385 compI_reg_imm0(cr, src);
13386 cmovI_reg_imm0_lt(dst, src, cr);
13387 %}
13388 %}
13389
13390 // This pattern is automatically generated from aarch64_ad.m4
13391 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13392 instruct minI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13393 %{
13394 match(Set dst (MinI src imm));
13395 ins_cost(INSN_COST * 3);
13396 expand %{
13397 rFlagsReg cr;
13398 compI_reg_imm0(cr, src);
13399 cmovI_reg_imm1_le(dst, src, cr);
13400 %}
13401 %}
13402
13403 // This pattern is automatically generated from aarch64_ad.m4
13404 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13405 instruct minI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13406 %{
13407 match(Set dst (MinI imm src));
13408 ins_cost(INSN_COST * 3);
13409 expand %{
13410 rFlagsReg cr;
13411 compI_reg_imm0(cr, src);
13412 cmovI_reg_imm1_le(dst, src, cr);
13413 %}
13414 %}
13415
13416 // This pattern is automatically generated from aarch64_ad.m4
13417 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13418 instruct minI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13419 %{
13420 match(Set dst (MinI src imm));
13421 ins_cost(INSN_COST * 3);
13422 expand %{
13423 rFlagsReg cr;
13424 compI_reg_imm0(cr, src);
13425 cmovI_reg_immM1_lt(dst, src, cr);
13426 %}
13427 %}
13428
13429 // This pattern is automatically generated from aarch64_ad.m4
13430 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13431 instruct minI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13432 %{
13433 match(Set dst (MinI imm src));
13434 ins_cost(INSN_COST * 3);
13435 expand %{
13436 rFlagsReg cr;
13437 compI_reg_imm0(cr, src);
13438 cmovI_reg_immM1_lt(dst, src, cr);
13439 %}
13440 %}
13441
13442 // This pattern is automatically generated from aarch64_ad.m4
13443 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13444 instruct maxI_reg_imm0(iRegINoSp dst, iRegIorL2I src, immI0 imm)
13445 %{
13446 match(Set dst (MaxI src imm));
13447 ins_cost(INSN_COST * 3);
13448 expand %{
13449 rFlagsReg cr;
13450 compI_reg_imm0(cr, src);
13451 cmovI_reg_imm0_gt(dst, src, cr);
13452 %}
13453 %}
13454
13455 // This pattern is automatically generated from aarch64_ad.m4
13456 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13457 instruct maxI_imm0_reg(iRegINoSp dst, immI0 imm, iRegIorL2I src)
13458 %{
13459 match(Set dst (MaxI imm src));
13460 ins_cost(INSN_COST * 3);
13461 expand %{
13462 rFlagsReg cr;
13463 compI_reg_imm0(cr, src);
13464 cmovI_reg_imm0_gt(dst, src, cr);
13465 %}
13466 %}
13467
13468 // This pattern is automatically generated from aarch64_ad.m4
13469 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13470 instruct maxI_reg_imm1(iRegINoSp dst, iRegIorL2I src, immI_1 imm)
13471 %{
13472 match(Set dst (MaxI src imm));
13473 ins_cost(INSN_COST * 3);
13474 expand %{
13475 rFlagsReg cr;
13476 compI_reg_imm0(cr, src);
13477 cmovI_reg_imm1_gt(dst, src, cr);
13478 %}
13479 %}
13480
13481 // This pattern is automatically generated from aarch64_ad.m4
13482 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13483 instruct maxI_imm1_reg(iRegINoSp dst, immI_1 imm, iRegIorL2I src)
13484 %{
13485 match(Set dst (MaxI imm src));
13486 ins_cost(INSN_COST * 3);
13487 expand %{
13488 rFlagsReg cr;
13489 compI_reg_imm0(cr, src);
13490 cmovI_reg_imm1_gt(dst, src, cr);
13491 %}
13492 %}
13493
13494 // This pattern is automatically generated from aarch64_ad.m4
13495 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13496 instruct maxI_reg_immM1(iRegINoSp dst, iRegIorL2I src, immI_M1 imm)
13497 %{
13498 match(Set dst (MaxI src imm));
13499 ins_cost(INSN_COST * 3);
13500 expand %{
13501 rFlagsReg cr;
13502 compI_reg_imm0(cr, src);
13503 cmovI_reg_immM1_ge(dst, src, cr);
13504 %}
13505 %}
13506
13507 // This pattern is automatically generated from aarch64_ad.m4
13508 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13509 instruct maxI_immM1_reg(iRegINoSp dst, immI_M1 imm, iRegIorL2I src)
13510 %{
13511 match(Set dst (MaxI imm src));
13512 ins_cost(INSN_COST * 3);
13513 expand %{
13514 rFlagsReg cr;
13515 compI_reg_imm0(cr, src);
13516 cmovI_reg_immM1_ge(dst, src, cr);
13517 %}
13518 %}
13519
13520 // This pattern is automatically generated from aarch64_ad.m4
13521 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13522 instruct bits_reverse_I(iRegINoSp dst, iRegIorL2I src)
13523 %{
13524 match(Set dst (ReverseI src));
13525 ins_cost(INSN_COST);
13526 format %{ "rbitw $dst, $src" %}
13527 ins_encode %{
13528 __ rbitw($dst$$Register, $src$$Register);
13529 %}
13530 ins_pipe(ialu_reg);
13531 %}
13532
13533 // This pattern is automatically generated from aarch64_ad.m4
13534 // DO NOT EDIT ANYTHING IN THIS SECTION OF THE FILE
13535 instruct bits_reverse_L(iRegLNoSp dst, iRegL src)
13536 %{
13537 match(Set dst (ReverseL src));
13538 ins_cost(INSN_COST);
13539 format %{ "rbit $dst, $src" %}
13540 ins_encode %{
13541 __ rbit($dst$$Register, $src$$Register);
13542 %}
13543 ins_pipe(ialu_reg);
13544 %}
13545
13546
13547 // END This section of the file is automatically generated. Do not edit --------------
13548
13549
13550 // ============================================================================
13551 // Floating Point Arithmetic Instructions
13552
13553 instruct addF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13554 match(Set dst (AddF src1 src2));
13555
13556 ins_cost(INSN_COST * 5);
13557 format %{ "fadds $dst, $src1, $src2" %}
13558
13559 ins_encode %{
13560 __ fadds(as_FloatRegister($dst$$reg),
13561 as_FloatRegister($src1$$reg),
13562 as_FloatRegister($src2$$reg));
13563 %}
13564
13565 ins_pipe(fp_dop_reg_reg_s);
13566 %}
13567
13568 instruct addD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13569 match(Set dst (AddD src1 src2));
13570
13571 ins_cost(INSN_COST * 5);
13572 format %{ "faddd $dst, $src1, $src2" %}
13573
13574 ins_encode %{
13575 __ faddd(as_FloatRegister($dst$$reg),
13576 as_FloatRegister($src1$$reg),
13577 as_FloatRegister($src2$$reg));
13578 %}
13579
13580 ins_pipe(fp_dop_reg_reg_d);
13581 %}
13582
13583 instruct subF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13584 match(Set dst (SubF src1 src2));
13585
13586 ins_cost(INSN_COST * 5);
13587 format %{ "fsubs $dst, $src1, $src2" %}
13588
13589 ins_encode %{
13590 __ fsubs(as_FloatRegister($dst$$reg),
13591 as_FloatRegister($src1$$reg),
13592 as_FloatRegister($src2$$reg));
13593 %}
13594
13595 ins_pipe(fp_dop_reg_reg_s);
13596 %}
13597
13598 instruct subD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13599 match(Set dst (SubD src1 src2));
13600
13601 ins_cost(INSN_COST * 5);
13602 format %{ "fsubd $dst, $src1, $src2" %}
13603
13604 ins_encode %{
13605 __ fsubd(as_FloatRegister($dst$$reg),
13606 as_FloatRegister($src1$$reg),
13607 as_FloatRegister($src2$$reg));
13608 %}
13609
13610 ins_pipe(fp_dop_reg_reg_d);
13611 %}
13612
13613 instruct mulF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13614 match(Set dst (MulF src1 src2));
13615
13616 ins_cost(INSN_COST * 6);
13617 format %{ "fmuls $dst, $src1, $src2" %}
13618
13619 ins_encode %{
13620 __ fmuls(as_FloatRegister($dst$$reg),
13621 as_FloatRegister($src1$$reg),
13622 as_FloatRegister($src2$$reg));
13623 %}
13624
13625 ins_pipe(fp_dop_reg_reg_s);
13626 %}
13627
13628 instruct mulD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13629 match(Set dst (MulD src1 src2));
13630
13631 ins_cost(INSN_COST * 6);
13632 format %{ "fmuld $dst, $src1, $src2" %}
13633
13634 ins_encode %{
13635 __ fmuld(as_FloatRegister($dst$$reg),
13636 as_FloatRegister($src1$$reg),
13637 as_FloatRegister($src2$$reg));
13638 %}
13639
13640 ins_pipe(fp_dop_reg_reg_d);
13641 %}
13642
13643 // src1 * src2 + src3
13644 instruct maddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13645 match(Set dst (FmaF src3 (Binary src1 src2)));
13646
13647 format %{ "fmadds $dst, $src1, $src2, $src3" %}
13648
13649 ins_encode %{
13650 assert(UseFMA, "Needs FMA instructions support.");
13651 __ fmadds(as_FloatRegister($dst$$reg),
13652 as_FloatRegister($src1$$reg),
13653 as_FloatRegister($src2$$reg),
13654 as_FloatRegister($src3$$reg));
13655 %}
13656
13657 ins_pipe(pipe_class_default);
13658 %}
13659
13660 // src1 * src2 + src3
13661 instruct maddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13662 match(Set dst (FmaD src3 (Binary src1 src2)));
13663
13664 format %{ "fmaddd $dst, $src1, $src2, $src3" %}
13665
13666 ins_encode %{
13667 assert(UseFMA, "Needs FMA instructions support.");
13668 __ fmaddd(as_FloatRegister($dst$$reg),
13669 as_FloatRegister($src1$$reg),
13670 as_FloatRegister($src2$$reg),
13671 as_FloatRegister($src3$$reg));
13672 %}
13673
13674 ins_pipe(pipe_class_default);
13675 %}
13676
13677 // src1 * (-src2) + src3
13678 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13679 instruct msubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13680 match(Set dst (FmaF src3 (Binary src1 (NegF src2))));
13681
13682 format %{ "fmsubs $dst, $src1, $src2, $src3" %}
13683
13684 ins_encode %{
13685 assert(UseFMA, "Needs FMA instructions support.");
13686 __ fmsubs(as_FloatRegister($dst$$reg),
13687 as_FloatRegister($src1$$reg),
13688 as_FloatRegister($src2$$reg),
13689 as_FloatRegister($src3$$reg));
13690 %}
13691
13692 ins_pipe(pipe_class_default);
13693 %}
13694
13695 // src1 * (-src2) + src3
13696 // "(-src1) * src2 + src3" has been idealized to "src2 * (-src1) + src3"
13697 instruct msubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13698 match(Set dst (FmaD src3 (Binary src1 (NegD src2))));
13699
13700 format %{ "fmsubd $dst, $src1, $src2, $src3" %}
13701
13702 ins_encode %{
13703 assert(UseFMA, "Needs FMA instructions support.");
13704 __ fmsubd(as_FloatRegister($dst$$reg),
13705 as_FloatRegister($src1$$reg),
13706 as_FloatRegister($src2$$reg),
13707 as_FloatRegister($src3$$reg));
13708 %}
13709
13710 ins_pipe(pipe_class_default);
13711 %}
13712
13713 // src1 * (-src2) - src3
13714 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13715 instruct mnaddF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3) %{
13716 match(Set dst (FmaF (NegF src3) (Binary src1 (NegF src2))));
13717
13718 format %{ "fnmadds $dst, $src1, $src2, $src3" %}
13719
13720 ins_encode %{
13721 assert(UseFMA, "Needs FMA instructions support.");
13722 __ fnmadds(as_FloatRegister($dst$$reg),
13723 as_FloatRegister($src1$$reg),
13724 as_FloatRegister($src2$$reg),
13725 as_FloatRegister($src3$$reg));
13726 %}
13727
13728 ins_pipe(pipe_class_default);
13729 %}
13730
13731 // src1 * (-src2) - src3
13732 // "(-src1) * src2 - src3" has been idealized to "src2 * (-src1) - src3"
13733 instruct mnaddD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3) %{
13734 match(Set dst (FmaD (NegD src3) (Binary src1 (NegD src2))));
13735
13736 format %{ "fnmaddd $dst, $src1, $src2, $src3" %}
13737
13738 ins_encode %{
13739 assert(UseFMA, "Needs FMA instructions support.");
13740 __ fnmaddd(as_FloatRegister($dst$$reg),
13741 as_FloatRegister($src1$$reg),
13742 as_FloatRegister($src2$$reg),
13743 as_FloatRegister($src3$$reg));
13744 %}
13745
13746 ins_pipe(pipe_class_default);
13747 %}
13748
13749 // src1 * src2 - src3
13750 instruct mnsubF_reg_reg(vRegF dst, vRegF src1, vRegF src2, vRegF src3, immF0 zero) %{
13751 match(Set dst (FmaF (NegF src3) (Binary src1 src2)));
13752
13753 format %{ "fnmsubs $dst, $src1, $src2, $src3" %}
13754
13755 ins_encode %{
13756 assert(UseFMA, "Needs FMA instructions support.");
13757 __ fnmsubs(as_FloatRegister($dst$$reg),
13758 as_FloatRegister($src1$$reg),
13759 as_FloatRegister($src2$$reg),
13760 as_FloatRegister($src3$$reg));
13761 %}
13762
13763 ins_pipe(pipe_class_default);
13764 %}
13765
13766 // src1 * src2 - src3
13767 instruct mnsubD_reg_reg(vRegD dst, vRegD src1, vRegD src2, vRegD src3, immD0 zero) %{
13768 match(Set dst (FmaD (NegD src3) (Binary src1 src2)));
13769
13770 format %{ "fnmsubd $dst, $src1, $src2, $src3" %}
13771
13772 ins_encode %{
13773 assert(UseFMA, "Needs FMA instructions support.");
13774 // n.b. insn name should be fnmsubd
13775 __ fnmsub(as_FloatRegister($dst$$reg),
13776 as_FloatRegister($src1$$reg),
13777 as_FloatRegister($src2$$reg),
13778 as_FloatRegister($src3$$reg));
13779 %}
13780
13781 ins_pipe(pipe_class_default);
13782 %}
13783
13784
13785 // Math.max(FF)F
13786 instruct maxF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13787 match(Set dst (MaxF src1 src2));
13788
13789 format %{ "fmaxs $dst, $src1, $src2" %}
13790 ins_encode %{
13791 __ fmaxs(as_FloatRegister($dst$$reg),
13792 as_FloatRegister($src1$$reg),
13793 as_FloatRegister($src2$$reg));
13794 %}
13795
13796 ins_pipe(fp_dop_reg_reg_s);
13797 %}
13798
13799 // Math.min(FF)F
13800 instruct minF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13801 match(Set dst (MinF src1 src2));
13802
13803 format %{ "fmins $dst, $src1, $src2" %}
13804 ins_encode %{
13805 __ fmins(as_FloatRegister($dst$$reg),
13806 as_FloatRegister($src1$$reg),
13807 as_FloatRegister($src2$$reg));
13808 %}
13809
13810 ins_pipe(fp_dop_reg_reg_s);
13811 %}
13812
13813 // Math.max(DD)D
13814 instruct maxD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13815 match(Set dst (MaxD src1 src2));
13816
13817 format %{ "fmaxd $dst, $src1, $src2" %}
13818 ins_encode %{
13819 __ fmaxd(as_FloatRegister($dst$$reg),
13820 as_FloatRegister($src1$$reg),
13821 as_FloatRegister($src2$$reg));
13822 %}
13823
13824 ins_pipe(fp_dop_reg_reg_d);
13825 %}
13826
13827 // Math.min(DD)D
13828 instruct minD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13829 match(Set dst (MinD src1 src2));
13830
13831 format %{ "fmind $dst, $src1, $src2" %}
13832 ins_encode %{
13833 __ fmind(as_FloatRegister($dst$$reg),
13834 as_FloatRegister($src1$$reg),
13835 as_FloatRegister($src2$$reg));
13836 %}
13837
13838 ins_pipe(fp_dop_reg_reg_d);
13839 %}
13840
13841
13842 instruct divF_reg_reg(vRegF dst, vRegF src1, vRegF src2) %{
13843 match(Set dst (DivF src1 src2));
13844
13845 ins_cost(INSN_COST * 18);
13846 format %{ "fdivs $dst, $src1, $src2" %}
13847
13848 ins_encode %{
13849 __ fdivs(as_FloatRegister($dst$$reg),
13850 as_FloatRegister($src1$$reg),
13851 as_FloatRegister($src2$$reg));
13852 %}
13853
13854 ins_pipe(fp_div_s);
13855 %}
13856
13857 instruct divD_reg_reg(vRegD dst, vRegD src1, vRegD src2) %{
13858 match(Set dst (DivD src1 src2));
13859
13860 ins_cost(INSN_COST * 32);
13861 format %{ "fdivd $dst, $src1, $src2" %}
13862
13863 ins_encode %{
13864 __ fdivd(as_FloatRegister($dst$$reg),
13865 as_FloatRegister($src1$$reg),
13866 as_FloatRegister($src2$$reg));
13867 %}
13868
13869 ins_pipe(fp_div_d);
13870 %}
13871
13872 instruct negF_reg_reg(vRegF dst, vRegF src) %{
13873 match(Set dst (NegF src));
13874
13875 ins_cost(INSN_COST * 3);
13876 format %{ "fneg $dst, $src" %}
13877
13878 ins_encode %{
13879 __ fnegs(as_FloatRegister($dst$$reg),
13880 as_FloatRegister($src$$reg));
13881 %}
13882
13883 ins_pipe(fp_uop_s);
13884 %}
13885
13886 instruct negD_reg_reg(vRegD dst, vRegD src) %{
13887 match(Set dst (NegD src));
13888
13889 ins_cost(INSN_COST * 3);
13890 format %{ "fnegd $dst, $src" %}
13891
13892 ins_encode %{
13893 __ fnegd(as_FloatRegister($dst$$reg),
13894 as_FloatRegister($src$$reg));
13895 %}
13896
13897 ins_pipe(fp_uop_d);
13898 %}
13899
13900 instruct absI_reg(iRegINoSp dst, iRegIorL2I src, rFlagsReg cr)
13901 %{
13902 match(Set dst (AbsI src));
13903
13904 effect(KILL cr);
13905 ins_cost(INSN_COST * 2);
13906 format %{ "cmpw $src, zr\n\t"
13907 "cnegw $dst, $src, Assembler::LT\t# int abs"
13908 %}
13909
13910 ins_encode %{
13911 __ cmpw(as_Register($src$$reg), zr);
13912 __ cnegw(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13913 %}
13914 ins_pipe(pipe_class_default);
13915 %}
13916
13917 instruct absL_reg(iRegLNoSp dst, iRegL src, rFlagsReg cr)
13918 %{
13919 match(Set dst (AbsL src));
13920
13921 effect(KILL cr);
13922 ins_cost(INSN_COST * 2);
13923 format %{ "cmp $src, zr\n\t"
13924 "cneg $dst, $src, Assembler::LT\t# long abs"
13925 %}
13926
13927 ins_encode %{
13928 __ cmp(as_Register($src$$reg), zr);
13929 __ cneg(as_Register($dst$$reg), as_Register($src$$reg), Assembler::LT);
13930 %}
13931 ins_pipe(pipe_class_default);
13932 %}
13933
13934 instruct absF_reg(vRegF dst, vRegF src) %{
13935 match(Set dst (AbsF src));
13936
13937 ins_cost(INSN_COST * 3);
13938 format %{ "fabss $dst, $src" %}
13939 ins_encode %{
13940 __ fabss(as_FloatRegister($dst$$reg),
13941 as_FloatRegister($src$$reg));
13942 %}
13943
13944 ins_pipe(fp_uop_s);
13945 %}
13946
13947 instruct absD_reg(vRegD dst, vRegD src) %{
13948 match(Set dst (AbsD src));
13949
13950 ins_cost(INSN_COST * 3);
13951 format %{ "fabsd $dst, $src" %}
13952 ins_encode %{
13953 __ fabsd(as_FloatRegister($dst$$reg),
13954 as_FloatRegister($src$$reg));
13955 %}
13956
13957 ins_pipe(fp_uop_d);
13958 %}
13959
13960 instruct absdF_reg(vRegF dst, vRegF src1, vRegF src2) %{
13961 match(Set dst (AbsF (SubF src1 src2)));
13962
13963 ins_cost(INSN_COST * 3);
13964 format %{ "fabds $dst, $src1, $src2" %}
13965 ins_encode %{
13966 __ fabds(as_FloatRegister($dst$$reg),
13967 as_FloatRegister($src1$$reg),
13968 as_FloatRegister($src2$$reg));
13969 %}
13970
13971 ins_pipe(fp_uop_s);
13972 %}
13973
13974 instruct absdD_reg(vRegD dst, vRegD src1, vRegD src2) %{
13975 match(Set dst (AbsD (SubD src1 src2)));
13976
13977 ins_cost(INSN_COST * 3);
13978 format %{ "fabdd $dst, $src1, $src2" %}
13979 ins_encode %{
13980 __ fabdd(as_FloatRegister($dst$$reg),
13981 as_FloatRegister($src1$$reg),
13982 as_FloatRegister($src2$$reg));
13983 %}
13984
13985 ins_pipe(fp_uop_d);
13986 %}
13987
13988 instruct sqrtD_reg(vRegD dst, vRegD src) %{
13989 match(Set dst (SqrtD src));
13990
13991 ins_cost(INSN_COST * 50);
13992 format %{ "fsqrtd $dst, $src" %}
13993 ins_encode %{
13994 __ fsqrtd(as_FloatRegister($dst$$reg),
13995 as_FloatRegister($src$$reg));
13996 %}
13997
13998 ins_pipe(fp_div_s);
13999 %}
14000
14001 instruct sqrtF_reg(vRegF dst, vRegF src) %{
14002 match(Set dst (SqrtF src));
14003
14004 ins_cost(INSN_COST * 50);
14005 format %{ "fsqrts $dst, $src" %}
14006 ins_encode %{
14007 __ fsqrts(as_FloatRegister($dst$$reg),
14008 as_FloatRegister($src$$reg));
14009 %}
14010
14011 ins_pipe(fp_div_d);
14012 %}
14013
14014 // Math.rint, floor, ceil
14015 instruct roundD_reg(vRegD dst, vRegD src, immI rmode) %{
14016 match(Set dst (RoundDoubleMode src rmode));
14017 format %{ "frint $dst, $src, $rmode" %}
14018 ins_encode %{
14019 switch ($rmode$$constant) {
14020 case RoundDoubleModeNode::rmode_rint:
14021 __ frintnd(as_FloatRegister($dst$$reg),
14022 as_FloatRegister($src$$reg));
14023 break;
14024 case RoundDoubleModeNode::rmode_floor:
14025 __ frintmd(as_FloatRegister($dst$$reg),
14026 as_FloatRegister($src$$reg));
14027 break;
14028 case RoundDoubleModeNode::rmode_ceil:
14029 __ frintpd(as_FloatRegister($dst$$reg),
14030 as_FloatRegister($src$$reg));
14031 break;
14032 }
14033 %}
14034 ins_pipe(fp_uop_d);
14035 %}
14036
14037 instruct copySignD_reg(vRegD dst, vRegD src1, vRegD src2, vRegD zero) %{
14038 match(Set dst (CopySignD src1 (Binary src2 zero)));
14039 effect(TEMP_DEF dst, USE src1, USE src2, USE zero);
14040 format %{ "CopySignD $dst $src1 $src2" %}
14041 ins_encode %{
14042 FloatRegister dst = as_FloatRegister($dst$$reg),
14043 src1 = as_FloatRegister($src1$$reg),
14044 src2 = as_FloatRegister($src2$$reg),
14045 zero = as_FloatRegister($zero$$reg);
14046 __ fnegd(dst, zero);
14047 __ bsl(dst, __ T8B, src2, src1);
14048 %}
14049 ins_pipe(fp_uop_d);
14050 %}
14051
14052 instruct copySignF_reg(vRegF dst, vRegF src1, vRegF src2) %{
14053 match(Set dst (CopySignF src1 src2));
14054 effect(TEMP_DEF dst, USE src1, USE src2);
14055 format %{ "CopySignF $dst $src1 $src2" %}
14056 ins_encode %{
14057 FloatRegister dst = as_FloatRegister($dst$$reg),
14058 src1 = as_FloatRegister($src1$$reg),
14059 src2 = as_FloatRegister($src2$$reg);
14060 __ movi(dst, __ T2S, 0x80, 24);
14061 __ bsl(dst, __ T8B, src2, src1);
14062 %}
14063 ins_pipe(fp_uop_d);
14064 %}
14065
14066 instruct signumD_reg(vRegD dst, vRegD src, vRegD zero, vRegD one) %{
14067 match(Set dst (SignumD src (Binary zero one)));
14068 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14069 format %{ "signumD $dst, $src" %}
14070 ins_encode %{
14071 FloatRegister src = as_FloatRegister($src$$reg),
14072 dst = as_FloatRegister($dst$$reg),
14073 zero = as_FloatRegister($zero$$reg),
14074 one = as_FloatRegister($one$$reg);
14075 __ facgtd(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14076 __ ushrd(dst, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14077 // Bit selection instruction gets bit from "one" for each enabled bit in
14078 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14079 // NaN the whole "src" will be copied because "dst" is zero. For all other
14080 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14081 // from "src", and all other bits are copied from 1.0.
14082 __ bsl(dst, __ T8B, one, src);
14083 %}
14084 ins_pipe(fp_uop_d);
14085 %}
14086
14087 instruct signumF_reg(vRegF dst, vRegF src, vRegF zero, vRegF one) %{
14088 match(Set dst (SignumF src (Binary zero one)));
14089 effect(TEMP_DEF dst, USE src, USE zero, USE one);
14090 format %{ "signumF $dst, $src" %}
14091 ins_encode %{
14092 FloatRegister src = as_FloatRegister($src$$reg),
14093 dst = as_FloatRegister($dst$$reg),
14094 zero = as_FloatRegister($zero$$reg),
14095 one = as_FloatRegister($one$$reg);
14096 __ facgts(dst, src, zero); // dst=0 for +-0.0 and NaN. 0xFFF..F otherwise
14097 __ ushr(dst, __ T2S, dst, 1); // dst=0 for +-0.0 and NaN. 0x7FF..F otherwise
14098 // Bit selection instruction gets bit from "one" for each enabled bit in
14099 // "dst", otherwise gets a bit from "src". For "src" that contains +-0.0 or
14100 // NaN the whole "src" will be copied because "dst" is zero. For all other
14101 // "src" values dst is 0x7FF..F, which means only the sign bit is copied
14102 // from "src", and all other bits are copied from 1.0.
14103 __ bsl(dst, __ T8B, one, src);
14104 %}
14105 ins_pipe(fp_uop_d);
14106 %}
14107
14108 instruct onspinwait() %{
14109 match(OnSpinWait);
14110 ins_cost(INSN_COST);
14111
14112 format %{ "onspinwait" %}
14113
14114 ins_encode %{
14115 __ spin_wait();
14116 %}
14117 ins_pipe(pipe_class_empty);
14118 %}
14119
14120 // ============================================================================
14121 // Logical Instructions
14122
14123 // Integer Logical Instructions
14124
14125 // And Instructions
14126
14127
14128 instruct andI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2, rFlagsReg cr) %{
14129 match(Set dst (AndI src1 src2));
14130
14131 format %{ "andw $dst, $src1, $src2\t# int" %}
14132
14133 ins_cost(INSN_COST);
14134 ins_encode %{
14135 __ andw(as_Register($dst$$reg),
14136 as_Register($src1$$reg),
14137 as_Register($src2$$reg));
14138 %}
14139
14140 ins_pipe(ialu_reg_reg);
14141 %}
14142
14143 instruct andI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2, rFlagsReg cr) %{
14144 match(Set dst (AndI src1 src2));
14145
14146 format %{ "andsw $dst, $src1, $src2\t# int" %}
14147
14148 ins_cost(INSN_COST);
14149 ins_encode %{
14150 __ andw(as_Register($dst$$reg),
14151 as_Register($src1$$reg),
14152 (uint64_t)($src2$$constant));
14153 %}
14154
14155 ins_pipe(ialu_reg_imm);
14156 %}
14157
14158 // Or Instructions
14159
14160 instruct orI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14161 match(Set dst (OrI src1 src2));
14162
14163 format %{ "orrw $dst, $src1, $src2\t# int" %}
14164
14165 ins_cost(INSN_COST);
14166 ins_encode %{
14167 __ orrw(as_Register($dst$$reg),
14168 as_Register($src1$$reg),
14169 as_Register($src2$$reg));
14170 %}
14171
14172 ins_pipe(ialu_reg_reg);
14173 %}
14174
14175 instruct orI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14176 match(Set dst (OrI src1 src2));
14177
14178 format %{ "orrw $dst, $src1, $src2\t# int" %}
14179
14180 ins_cost(INSN_COST);
14181 ins_encode %{
14182 __ orrw(as_Register($dst$$reg),
14183 as_Register($src1$$reg),
14184 (uint64_t)($src2$$constant));
14185 %}
14186
14187 ins_pipe(ialu_reg_imm);
14188 %}
14189
14190 // Xor Instructions
14191
14192 instruct xorI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2) %{
14193 match(Set dst (XorI src1 src2));
14194
14195 format %{ "eorw $dst, $src1, $src2\t# int" %}
14196
14197 ins_cost(INSN_COST);
14198 ins_encode %{
14199 __ eorw(as_Register($dst$$reg),
14200 as_Register($src1$$reg),
14201 as_Register($src2$$reg));
14202 %}
14203
14204 ins_pipe(ialu_reg_reg);
14205 %}
14206
14207 instruct xorI_reg_imm(iRegINoSp dst, iRegIorL2I src1, immILog src2) %{
14208 match(Set dst (XorI src1 src2));
14209
14210 format %{ "eorw $dst, $src1, $src2\t# int" %}
14211
14212 ins_cost(INSN_COST);
14213 ins_encode %{
14214 __ eorw(as_Register($dst$$reg),
14215 as_Register($src1$$reg),
14216 (uint64_t)($src2$$constant));
14217 %}
14218
14219 ins_pipe(ialu_reg_imm);
14220 %}
14221
14222 // Long Logical Instructions
14223 // TODO
14224
14225 instruct andL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2, rFlagsReg cr) %{
14226 match(Set dst (AndL src1 src2));
14227
14228 format %{ "and $dst, $src1, $src2\t# int" %}
14229
14230 ins_cost(INSN_COST);
14231 ins_encode %{
14232 __ andr(as_Register($dst$$reg),
14233 as_Register($src1$$reg),
14234 as_Register($src2$$reg));
14235 %}
14236
14237 ins_pipe(ialu_reg_reg);
14238 %}
14239
14240 instruct andL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2, rFlagsReg cr) %{
14241 match(Set dst (AndL src1 src2));
14242
14243 format %{ "and $dst, $src1, $src2\t# int" %}
14244
14245 ins_cost(INSN_COST);
14246 ins_encode %{
14247 __ andr(as_Register($dst$$reg),
14248 as_Register($src1$$reg),
14249 (uint64_t)($src2$$constant));
14250 %}
14251
14252 ins_pipe(ialu_reg_imm);
14253 %}
14254
14255 // Or Instructions
14256
14257 instruct orL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14258 match(Set dst (OrL src1 src2));
14259
14260 format %{ "orr $dst, $src1, $src2\t# int" %}
14261
14262 ins_cost(INSN_COST);
14263 ins_encode %{
14264 __ orr(as_Register($dst$$reg),
14265 as_Register($src1$$reg),
14266 as_Register($src2$$reg));
14267 %}
14268
14269 ins_pipe(ialu_reg_reg);
14270 %}
14271
14272 instruct orL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14273 match(Set dst (OrL src1 src2));
14274
14275 format %{ "orr $dst, $src1, $src2\t# int" %}
14276
14277 ins_cost(INSN_COST);
14278 ins_encode %{
14279 __ orr(as_Register($dst$$reg),
14280 as_Register($src1$$reg),
14281 (uint64_t)($src2$$constant));
14282 %}
14283
14284 ins_pipe(ialu_reg_imm);
14285 %}
14286
14287 // Xor Instructions
14288
14289 instruct xorL_reg_reg(iRegLNoSp dst, iRegL src1, iRegL src2) %{
14290 match(Set dst (XorL src1 src2));
14291
14292 format %{ "eor $dst, $src1, $src2\t# int" %}
14293
14294 ins_cost(INSN_COST);
14295 ins_encode %{
14296 __ eor(as_Register($dst$$reg),
14297 as_Register($src1$$reg),
14298 as_Register($src2$$reg));
14299 %}
14300
14301 ins_pipe(ialu_reg_reg);
14302 %}
14303
14304 instruct xorL_reg_imm(iRegLNoSp dst, iRegL src1, immLLog src2) %{
14305 match(Set dst (XorL src1 src2));
14306
14307 ins_cost(INSN_COST);
14308 format %{ "eor $dst, $src1, $src2\t# int" %}
14309
14310 ins_encode %{
14311 __ eor(as_Register($dst$$reg),
14312 as_Register($src1$$reg),
14313 (uint64_t)($src2$$constant));
14314 %}
14315
14316 ins_pipe(ialu_reg_imm);
14317 %}
14318
14319 instruct convI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src)
14320 %{
14321 match(Set dst (ConvI2L src));
14322
14323 ins_cost(INSN_COST);
14324 format %{ "sxtw $dst, $src\t# i2l" %}
14325 ins_encode %{
14326 __ sbfm($dst$$Register, $src$$Register, 0, 31);
14327 %}
14328 ins_pipe(ialu_reg_shift);
14329 %}
14330
14331 // this pattern occurs in bigmath arithmetic
14332 instruct convUI2L_reg_reg(iRegLNoSp dst, iRegIorL2I src, immL_32bits mask)
14333 %{
14334 match(Set dst (AndL (ConvI2L src) mask));
14335
14336 ins_cost(INSN_COST);
14337 format %{ "ubfm $dst, $src, 0, 31\t# ui2l" %}
14338 ins_encode %{
14339 __ ubfm($dst$$Register, $src$$Register, 0, 31);
14340 %}
14341
14342 ins_pipe(ialu_reg_shift);
14343 %}
14344
14345 instruct convL2I_reg(iRegINoSp dst, iRegL src) %{
14346 match(Set dst (ConvL2I src));
14347
14348 ins_cost(INSN_COST);
14349 format %{ "movw $dst, $src \t// l2i" %}
14350
14351 ins_encode %{
14352 __ movw(as_Register($dst$$reg), as_Register($src$$reg));
14353 %}
14354
14355 ins_pipe(ialu_reg);
14356 %}
14357
14358 instruct convD2F_reg(vRegF dst, vRegD src) %{
14359 match(Set dst (ConvD2F src));
14360
14361 ins_cost(INSN_COST * 5);
14362 format %{ "fcvtd $dst, $src \t// d2f" %}
14363
14364 ins_encode %{
14365 __ fcvtd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14366 %}
14367
14368 ins_pipe(fp_d2f);
14369 %}
14370
14371 instruct convF2D_reg(vRegD dst, vRegF src) %{
14372 match(Set dst (ConvF2D src));
14373
14374 ins_cost(INSN_COST * 5);
14375 format %{ "fcvts $dst, $src \t// f2d" %}
14376
14377 ins_encode %{
14378 __ fcvts(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
14379 %}
14380
14381 ins_pipe(fp_f2d);
14382 %}
14383
14384 instruct convF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14385 match(Set dst (ConvF2I src));
14386
14387 ins_cost(INSN_COST * 5);
14388 format %{ "fcvtzsw $dst, $src \t// f2i" %}
14389
14390 ins_encode %{
14391 __ fcvtzsw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14392 %}
14393
14394 ins_pipe(fp_f2i);
14395 %}
14396
14397 instruct convF2L_reg_reg(iRegLNoSp dst, vRegF src) %{
14398 match(Set dst (ConvF2L src));
14399
14400 ins_cost(INSN_COST * 5);
14401 format %{ "fcvtzs $dst, $src \t// f2l" %}
14402
14403 ins_encode %{
14404 __ fcvtzs(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14405 %}
14406
14407 ins_pipe(fp_f2l);
14408 %}
14409
14410 instruct convF2HF_reg_reg(iRegINoSp dst, vRegF src, vRegF tmp) %{
14411 match(Set dst (ConvF2HF src));
14412 format %{ "fcvt $tmp, $src\t# convert single to half precision\n\t"
14413 "smov $dst, $tmp\t# move result from $tmp to $dst"
14414 %}
14415 effect(TEMP tmp);
14416 ins_encode %{
14417 __ flt_to_flt16($dst$$Register, $src$$FloatRegister, $tmp$$FloatRegister);
14418 %}
14419 ins_pipe(pipe_slow);
14420 %}
14421
14422 instruct convHF2F_reg_reg(vRegF dst, iRegINoSp src, vRegF tmp) %{
14423 match(Set dst (ConvHF2F src));
14424 format %{ "mov $tmp, $src\t# move source from $src to $tmp\n\t"
14425 "fcvt $dst, $tmp\t# convert half to single precision"
14426 %}
14427 effect(TEMP tmp);
14428 ins_encode %{
14429 __ flt16_to_flt($dst$$FloatRegister, $src$$Register, $tmp$$FloatRegister);
14430 %}
14431 ins_pipe(pipe_slow);
14432 %}
14433
14434 instruct convI2F_reg_reg(vRegF dst, iRegIorL2I src) %{
14435 match(Set dst (ConvI2F src));
14436
14437 ins_cost(INSN_COST * 5);
14438 format %{ "scvtfws $dst, $src \t// i2f" %}
14439
14440 ins_encode %{
14441 __ scvtfws(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14442 %}
14443
14444 ins_pipe(fp_i2f);
14445 %}
14446
14447 instruct convL2F_reg_reg(vRegF dst, iRegL src) %{
14448 match(Set dst (ConvL2F src));
14449
14450 ins_cost(INSN_COST * 5);
14451 format %{ "scvtfs $dst, $src \t// l2f" %}
14452
14453 ins_encode %{
14454 __ scvtfs(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14455 %}
14456
14457 ins_pipe(fp_l2f);
14458 %}
14459
14460 instruct convD2I_reg_reg(iRegINoSp dst, vRegD src) %{
14461 match(Set dst (ConvD2I src));
14462
14463 ins_cost(INSN_COST * 5);
14464 format %{ "fcvtzdw $dst, $src \t// d2i" %}
14465
14466 ins_encode %{
14467 __ fcvtzdw(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14468 %}
14469
14470 ins_pipe(fp_d2i);
14471 %}
14472
14473 instruct convD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14474 match(Set dst (ConvD2L src));
14475
14476 ins_cost(INSN_COST * 5);
14477 format %{ "fcvtzd $dst, $src \t// d2l" %}
14478
14479 ins_encode %{
14480 __ fcvtzd(as_Register($dst$$reg), as_FloatRegister($src$$reg));
14481 %}
14482
14483 ins_pipe(fp_d2l);
14484 %}
14485
14486 instruct convI2D_reg_reg(vRegD dst, iRegIorL2I src) %{
14487 match(Set dst (ConvI2D src));
14488
14489 ins_cost(INSN_COST * 5);
14490 format %{ "scvtfwd $dst, $src \t// i2d" %}
14491
14492 ins_encode %{
14493 __ scvtfwd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14494 %}
14495
14496 ins_pipe(fp_i2d);
14497 %}
14498
14499 instruct convL2D_reg_reg(vRegD dst, iRegL src) %{
14500 match(Set dst (ConvL2D src));
14501
14502 ins_cost(INSN_COST * 5);
14503 format %{ "scvtfd $dst, $src \t// l2d" %}
14504
14505 ins_encode %{
14506 __ scvtfd(as_FloatRegister($dst$$reg), as_Register($src$$reg));
14507 %}
14508
14509 ins_pipe(fp_l2d);
14510 %}
14511
14512 instruct round_double_reg(iRegLNoSp dst, vRegD src, vRegD ftmp, rFlagsReg cr)
14513 %{
14514 match(Set dst (RoundD src));
14515 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14516 format %{ "java_round_double $dst,$src"%}
14517 ins_encode %{
14518 __ java_round_double($dst$$Register, as_FloatRegister($src$$reg),
14519 as_FloatRegister($ftmp$$reg));
14520 %}
14521 ins_pipe(pipe_slow);
14522 %}
14523
14524 instruct round_float_reg(iRegINoSp dst, vRegF src, vRegF ftmp, rFlagsReg cr)
14525 %{
14526 match(Set dst (RoundF src));
14527 effect(TEMP_DEF dst, TEMP ftmp, KILL cr);
14528 format %{ "java_round_float $dst,$src"%}
14529 ins_encode %{
14530 __ java_round_float($dst$$Register, as_FloatRegister($src$$reg),
14531 as_FloatRegister($ftmp$$reg));
14532 %}
14533 ins_pipe(pipe_slow);
14534 %}
14535
14536 // stack <-> reg and reg <-> reg shuffles with no conversion
14537
14538 instruct MoveF2I_stack_reg(iRegINoSp dst, stackSlotF src) %{
14539
14540 match(Set dst (MoveF2I src));
14541
14542 effect(DEF dst, USE src);
14543
14544 ins_cost(4 * INSN_COST);
14545
14546 format %{ "ldrw $dst, $src\t# MoveF2I_stack_reg" %}
14547
14548 ins_encode %{
14549 __ ldrw($dst$$Register, Address(sp, $src$$disp));
14550 %}
14551
14552 ins_pipe(iload_reg_reg);
14553
14554 %}
14555
14556 instruct MoveI2F_stack_reg(vRegF dst, stackSlotI src) %{
14557
14558 match(Set dst (MoveI2F src));
14559
14560 effect(DEF dst, USE src);
14561
14562 ins_cost(4 * INSN_COST);
14563
14564 format %{ "ldrs $dst, $src\t# MoveI2F_stack_reg" %}
14565
14566 ins_encode %{
14567 __ ldrs(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14568 %}
14569
14570 ins_pipe(pipe_class_memory);
14571
14572 %}
14573
14574 instruct MoveD2L_stack_reg(iRegLNoSp dst, stackSlotD src) %{
14575
14576 match(Set dst (MoveD2L src));
14577
14578 effect(DEF dst, USE src);
14579
14580 ins_cost(4 * INSN_COST);
14581
14582 format %{ "ldr $dst, $src\t# MoveD2L_stack_reg" %}
14583
14584 ins_encode %{
14585 __ ldr($dst$$Register, Address(sp, $src$$disp));
14586 %}
14587
14588 ins_pipe(iload_reg_reg);
14589
14590 %}
14591
14592 instruct MoveL2D_stack_reg(vRegD dst, stackSlotL src) %{
14593
14594 match(Set dst (MoveL2D src));
14595
14596 effect(DEF dst, USE src);
14597
14598 ins_cost(4 * INSN_COST);
14599
14600 format %{ "ldrd $dst, $src\t# MoveL2D_stack_reg" %}
14601
14602 ins_encode %{
14603 __ ldrd(as_FloatRegister($dst$$reg), Address(sp, $src$$disp));
14604 %}
14605
14606 ins_pipe(pipe_class_memory);
14607
14608 %}
14609
14610 instruct MoveF2I_reg_stack(stackSlotI dst, vRegF src) %{
14611
14612 match(Set dst (MoveF2I src));
14613
14614 effect(DEF dst, USE src);
14615
14616 ins_cost(INSN_COST);
14617
14618 format %{ "strs $src, $dst\t# MoveF2I_reg_stack" %}
14619
14620 ins_encode %{
14621 __ strs(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14622 %}
14623
14624 ins_pipe(pipe_class_memory);
14625
14626 %}
14627
14628 instruct MoveI2F_reg_stack(stackSlotF dst, iRegI src) %{
14629
14630 match(Set dst (MoveI2F src));
14631
14632 effect(DEF dst, USE src);
14633
14634 ins_cost(INSN_COST);
14635
14636 format %{ "strw $src, $dst\t# MoveI2F_reg_stack" %}
14637
14638 ins_encode %{
14639 __ strw($src$$Register, Address(sp, $dst$$disp));
14640 %}
14641
14642 ins_pipe(istore_reg_reg);
14643
14644 %}
14645
14646 instruct MoveD2L_reg_stack(stackSlotL dst, vRegD src) %{
14647
14648 match(Set dst (MoveD2L src));
14649
14650 effect(DEF dst, USE src);
14651
14652 ins_cost(INSN_COST);
14653
14654 format %{ "strd $dst, $src\t# MoveD2L_reg_stack" %}
14655
14656 ins_encode %{
14657 __ strd(as_FloatRegister($src$$reg), Address(sp, $dst$$disp));
14658 %}
14659
14660 ins_pipe(pipe_class_memory);
14661
14662 %}
14663
14664 instruct MoveL2D_reg_stack(stackSlotD dst, iRegL src) %{
14665
14666 match(Set dst (MoveL2D src));
14667
14668 effect(DEF dst, USE src);
14669
14670 ins_cost(INSN_COST);
14671
14672 format %{ "str $src, $dst\t# MoveL2D_reg_stack" %}
14673
14674 ins_encode %{
14675 __ str($src$$Register, Address(sp, $dst$$disp));
14676 %}
14677
14678 ins_pipe(istore_reg_reg);
14679
14680 %}
14681
14682 instruct MoveF2I_reg_reg(iRegINoSp dst, vRegF src) %{
14683
14684 match(Set dst (MoveF2I src));
14685
14686 effect(DEF dst, USE src);
14687
14688 ins_cost(INSN_COST);
14689
14690 format %{ "fmovs $dst, $src\t# MoveF2I_reg_reg" %}
14691
14692 ins_encode %{
14693 __ fmovs($dst$$Register, as_FloatRegister($src$$reg));
14694 %}
14695
14696 ins_pipe(fp_f2i);
14697
14698 %}
14699
14700 instruct MoveI2F_reg_reg(vRegF dst, iRegI src) %{
14701
14702 match(Set dst (MoveI2F src));
14703
14704 effect(DEF dst, USE src);
14705
14706 ins_cost(INSN_COST);
14707
14708 format %{ "fmovs $dst, $src\t# MoveI2F_reg_reg" %}
14709
14710 ins_encode %{
14711 __ fmovs(as_FloatRegister($dst$$reg), $src$$Register);
14712 %}
14713
14714 ins_pipe(fp_i2f);
14715
14716 %}
14717
14718 instruct MoveD2L_reg_reg(iRegLNoSp dst, vRegD src) %{
14719
14720 match(Set dst (MoveD2L src));
14721
14722 effect(DEF dst, USE src);
14723
14724 ins_cost(INSN_COST);
14725
14726 format %{ "fmovd $dst, $src\t# MoveD2L_reg_reg" %}
14727
14728 ins_encode %{
14729 __ fmovd($dst$$Register, as_FloatRegister($src$$reg));
14730 %}
14731
14732 ins_pipe(fp_d2l);
14733
14734 %}
14735
14736 instruct MoveL2D_reg_reg(vRegD dst, iRegL src) %{
14737
14738 match(Set dst (MoveL2D src));
14739
14740 effect(DEF dst, USE src);
14741
14742 ins_cost(INSN_COST);
14743
14744 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14745
14746 ins_encode %{
14747 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14748 %}
14749
14750 ins_pipe(fp_l2d);
14751
14752 %}
14753
14754 // ============================================================================
14755 // clearing of an array
14756
14757 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14758 %{
14759 match(Set dummy (ClearArray cnt base));
14760 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14761
14762 ins_cost(4 * INSN_COST);
14763 format %{ "ClearArray $cnt, $base" %}
14764
14765 ins_encode %{
14766 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14767 if (tpc == nullptr) {
14768 ciEnv::current()->record_failure("CodeCache is full");
14769 return;
14770 }
14771 %}
14772
14773 ins_pipe(pipe_class_memory);
14774 %}
14775
14776 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14777 %{
14778 predicate((uint64_t)n->in(2)->get_long()
14779 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14780 match(Set dummy (ClearArray cnt base));
14781 effect(TEMP temp, USE_KILL base, KILL cr);
14782
14783 ins_cost(4 * INSN_COST);
14784 format %{ "ClearArray $cnt, $base" %}
14785
14786 ins_encode %{
14787 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14788 if (tpc == nullptr) {
14789 ciEnv::current()->record_failure("CodeCache is full");
14790 return;
14791 }
14792 %}
14793
14794 ins_pipe(pipe_class_memory);
14795 %}
14796
14797 // ============================================================================
14798 // Overflow Math Instructions
14799
14800 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14801 %{
14802 match(Set cr (OverflowAddI op1 op2));
14803
14804 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14805 ins_cost(INSN_COST);
14806 ins_encode %{
14807 __ cmnw($op1$$Register, $op2$$Register);
14808 %}
14809
14810 ins_pipe(icmp_reg_reg);
14811 %}
14812
14813 instruct overflowAddI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14814 %{
14815 match(Set cr (OverflowAddI op1 op2));
14816
14817 format %{ "cmnw $op1, $op2\t# overflow check int" %}
14818 ins_cost(INSN_COST);
14819 ins_encode %{
14820 __ cmnw($op1$$Register, $op2$$constant);
14821 %}
14822
14823 ins_pipe(icmp_reg_imm);
14824 %}
14825
14826 instruct overflowAddL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14827 %{
14828 match(Set cr (OverflowAddL op1 op2));
14829
14830 format %{ "cmn $op1, $op2\t# overflow check long" %}
14831 ins_cost(INSN_COST);
14832 ins_encode %{
14833 __ cmn($op1$$Register, $op2$$Register);
14834 %}
14835
14836 ins_pipe(icmp_reg_reg);
14837 %}
14838
14839 instruct overflowAddL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14840 %{
14841 match(Set cr (OverflowAddL op1 op2));
14842
14843 format %{ "adds zr, $op1, $op2\t# overflow check long" %}
14844 ins_cost(INSN_COST);
14845 ins_encode %{
14846 __ adds(zr, $op1$$Register, $op2$$constant);
14847 %}
14848
14849 ins_pipe(icmp_reg_imm);
14850 %}
14851
14852 instruct overflowSubI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14853 %{
14854 match(Set cr (OverflowSubI op1 op2));
14855
14856 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14857 ins_cost(INSN_COST);
14858 ins_encode %{
14859 __ cmpw($op1$$Register, $op2$$Register);
14860 %}
14861
14862 ins_pipe(icmp_reg_reg);
14863 %}
14864
14865 instruct overflowSubI_reg_imm(rFlagsReg cr, iRegIorL2I op1, immIAddSub op2)
14866 %{
14867 match(Set cr (OverflowSubI op1 op2));
14868
14869 format %{ "cmpw $op1, $op2\t# overflow check int" %}
14870 ins_cost(INSN_COST);
14871 ins_encode %{
14872 __ cmpw($op1$$Register, $op2$$constant);
14873 %}
14874
14875 ins_pipe(icmp_reg_imm);
14876 %}
14877
14878 instruct overflowSubL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14879 %{
14880 match(Set cr (OverflowSubL op1 op2));
14881
14882 format %{ "cmp $op1, $op2\t# overflow check long" %}
14883 ins_cost(INSN_COST);
14884 ins_encode %{
14885 __ cmp($op1$$Register, $op2$$Register);
14886 %}
14887
14888 ins_pipe(icmp_reg_reg);
14889 %}
14890
14891 instruct overflowSubL_reg_imm(rFlagsReg cr, iRegL op1, immLAddSub op2)
14892 %{
14893 match(Set cr (OverflowSubL op1 op2));
14894
14895 format %{ "cmp $op1, $op2\t# overflow check long" %}
14896 ins_cost(INSN_COST);
14897 ins_encode %{
14898 __ subs(zr, $op1$$Register, $op2$$constant);
14899 %}
14900
14901 ins_pipe(icmp_reg_imm);
14902 %}
14903
14904 instruct overflowNegI_reg(rFlagsReg cr, immI0 zero, iRegIorL2I op1)
14905 %{
14906 match(Set cr (OverflowSubI zero op1));
14907
14908 format %{ "cmpw zr, $op1\t# overflow check int" %}
14909 ins_cost(INSN_COST);
14910 ins_encode %{
14911 __ cmpw(zr, $op1$$Register);
14912 %}
14913
14914 ins_pipe(icmp_reg_imm);
14915 %}
14916
14917 instruct overflowNegL_reg(rFlagsReg cr, immI0 zero, iRegL op1)
14918 %{
14919 match(Set cr (OverflowSubL zero op1));
14920
14921 format %{ "cmp zr, $op1\t# overflow check long" %}
14922 ins_cost(INSN_COST);
14923 ins_encode %{
14924 __ cmp(zr, $op1$$Register);
14925 %}
14926
14927 ins_pipe(icmp_reg_imm);
14928 %}
14929
14930 instruct overflowMulI_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
14931 %{
14932 match(Set cr (OverflowMulI op1 op2));
14933
14934 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14935 "cmp rscratch1, rscratch1, sxtw\n\t"
14936 "movw rscratch1, #0x80000000\n\t"
14937 "cselw rscratch1, rscratch1, zr, NE\n\t"
14938 "cmpw rscratch1, #1" %}
14939 ins_cost(5 * INSN_COST);
14940 ins_encode %{
14941 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14942 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14943 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14944 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14945 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14946 %}
14947
14948 ins_pipe(pipe_slow);
14949 %}
14950
14951 instruct overflowMulI_reg_branch(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, label labl, rFlagsReg cr)
14952 %{
14953 match(If cmp (OverflowMulI op1 op2));
14954 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
14955 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
14956 effect(USE labl, KILL cr);
14957
14958 format %{ "smull rscratch1, $op1, $op2\t# overflow check int\n\t"
14959 "cmp rscratch1, rscratch1, sxtw\n\t"
14960 "b$cmp $labl" %}
14961 ins_cost(3 * INSN_COST); // Branch is rare so treat as INSN_COST
14962 ins_encode %{
14963 Label* L = $labl$$label;
14964 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
14965 __ smull(rscratch1, $op1$$Register, $op2$$Register);
14966 __ subs(zr, rscratch1, rscratch1, ext::sxtw); // NE => overflow
14967 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
14968 %}
14969
14970 ins_pipe(pipe_serial);
14971 %}
14972
14973 instruct overflowMulL_reg(rFlagsReg cr, iRegL op1, iRegL op2)
14974 %{
14975 match(Set cr (OverflowMulL op1 op2));
14976
14977 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
14978 "smulh rscratch2, $op1, $op2\n\t"
14979 "cmp rscratch2, rscratch1, ASR #63\n\t"
14980 "movw rscratch1, #0x80000000\n\t"
14981 "cselw rscratch1, rscratch1, zr, NE\n\t"
14982 "cmpw rscratch1, #1" %}
14983 ins_cost(6 * INSN_COST);
14984 ins_encode %{
14985 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
14986 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
14987 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
14988 __ movw(rscratch1, 0x80000000); // Develop 0 (EQ),
14989 __ cselw(rscratch1, rscratch1, zr, Assembler::NE); // or 0x80000000 (NE)
14990 __ cmpw(rscratch1, 1); // 0x80000000 - 1 => VS
14991 %}
14992
14993 ins_pipe(pipe_slow);
14994 %}
14995
14996 instruct overflowMulL_reg_branch(cmpOp cmp, iRegL op1, iRegL op2, label labl, rFlagsReg cr)
14997 %{
14998 match(If cmp (OverflowMulL op1 op2));
14999 predicate(n->in(1)->as_Bool()->_test._test == BoolTest::overflow
15000 || n->in(1)->as_Bool()->_test._test == BoolTest::no_overflow);
15001 effect(USE labl, KILL cr);
15002
15003 format %{ "mul rscratch1, $op1, $op2\t#overflow check long\n\t"
15004 "smulh rscratch2, $op1, $op2\n\t"
15005 "cmp rscratch2, rscratch1, ASR #63\n\t"
15006 "b$cmp $labl" %}
15007 ins_cost(4 * INSN_COST); // Branch is rare so treat as INSN_COST
15008 ins_encode %{
15009 Label* L = $labl$$label;
15010 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15011 __ mul(rscratch1, $op1$$Register, $op2$$Register); // Result bits 0..63
15012 __ smulh(rscratch2, $op1$$Register, $op2$$Register); // Result bits 64..127
15013 __ cmp(rscratch2, rscratch1, Assembler::ASR, 63); // Top is pure sign ext
15014 __ br(cond == Assembler::VS ? Assembler::NE : Assembler::EQ, *L);
15015 %}
15016
15017 ins_pipe(pipe_serial);
15018 %}
15019
15020 // ============================================================================
15021 // Compare Instructions
15022
15023 instruct compI_reg_reg(rFlagsReg cr, iRegI op1, iRegI op2)
15024 %{
15025 match(Set cr (CmpI op1 op2));
15026
15027 effect(DEF cr, USE op1, USE op2);
15028
15029 ins_cost(INSN_COST);
15030 format %{ "cmpw $op1, $op2" %}
15031
15032 ins_encode(aarch64_enc_cmpw(op1, op2));
15033
15034 ins_pipe(icmp_reg_reg);
15035 %}
15036
15037 instruct compI_reg_immI0(rFlagsReg cr, iRegI op1, immI0 zero)
15038 %{
15039 match(Set cr (CmpI op1 zero));
15040
15041 effect(DEF cr, USE op1);
15042
15043 ins_cost(INSN_COST);
15044 format %{ "cmpw $op1, 0" %}
15045
15046 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15047
15048 ins_pipe(icmp_reg_imm);
15049 %}
15050
15051 instruct compI_reg_immIAddSub(rFlagsReg cr, iRegI op1, immIAddSub op2)
15052 %{
15053 match(Set cr (CmpI op1 op2));
15054
15055 effect(DEF cr, USE op1);
15056
15057 ins_cost(INSN_COST);
15058 format %{ "cmpw $op1, $op2" %}
15059
15060 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15061
15062 ins_pipe(icmp_reg_imm);
15063 %}
15064
15065 instruct compI_reg_immI(rFlagsReg cr, iRegI op1, immI op2)
15066 %{
15067 match(Set cr (CmpI op1 op2));
15068
15069 effect(DEF cr, USE op1);
15070
15071 ins_cost(INSN_COST * 2);
15072 format %{ "cmpw $op1, $op2" %}
15073
15074 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15075
15076 ins_pipe(icmp_reg_imm);
15077 %}
15078
15079 // Unsigned compare Instructions; really, same as signed compare
15080 // except it should only be used to feed an If or a CMovI which takes a
15081 // cmpOpU.
15082
15083 instruct compU_reg_reg(rFlagsRegU cr, iRegI op1, iRegI op2)
15084 %{
15085 match(Set cr (CmpU op1 op2));
15086
15087 effect(DEF cr, USE op1, USE op2);
15088
15089 ins_cost(INSN_COST);
15090 format %{ "cmpw $op1, $op2\t# unsigned" %}
15091
15092 ins_encode(aarch64_enc_cmpw(op1, op2));
15093
15094 ins_pipe(icmp_reg_reg);
15095 %}
15096
15097 instruct compU_reg_immI0(rFlagsRegU cr, iRegI op1, immI0 zero)
15098 %{
15099 match(Set cr (CmpU op1 zero));
15100
15101 effect(DEF cr, USE op1);
15102
15103 ins_cost(INSN_COST);
15104 format %{ "cmpw $op1, #0\t# unsigned" %}
15105
15106 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, zero));
15107
15108 ins_pipe(icmp_reg_imm);
15109 %}
15110
15111 instruct compU_reg_immIAddSub(rFlagsRegU cr, iRegI op1, immIAddSub op2)
15112 %{
15113 match(Set cr (CmpU op1 op2));
15114
15115 effect(DEF cr, USE op1);
15116
15117 ins_cost(INSN_COST);
15118 format %{ "cmpw $op1, $op2\t# unsigned" %}
15119
15120 ins_encode(aarch64_enc_cmpw_imm_addsub(op1, op2));
15121
15122 ins_pipe(icmp_reg_imm);
15123 %}
15124
15125 instruct compU_reg_immI(rFlagsRegU cr, iRegI op1, immI op2)
15126 %{
15127 match(Set cr (CmpU op1 op2));
15128
15129 effect(DEF cr, USE op1);
15130
15131 ins_cost(INSN_COST * 2);
15132 format %{ "cmpw $op1, $op2\t# unsigned" %}
15133
15134 ins_encode(aarch64_enc_cmpw_imm(op1, op2));
15135
15136 ins_pipe(icmp_reg_imm);
15137 %}
15138
15139 instruct compL_reg_reg(rFlagsReg cr, iRegL op1, iRegL op2)
15140 %{
15141 match(Set cr (CmpL op1 op2));
15142
15143 effect(DEF cr, USE op1, USE op2);
15144
15145 ins_cost(INSN_COST);
15146 format %{ "cmp $op1, $op2" %}
15147
15148 ins_encode(aarch64_enc_cmp(op1, op2));
15149
15150 ins_pipe(icmp_reg_reg);
15151 %}
15152
15153 instruct compL_reg_immL0(rFlagsReg cr, iRegL op1, immL0 zero)
15154 %{
15155 match(Set cr (CmpL op1 zero));
15156
15157 effect(DEF cr, USE op1);
15158
15159 ins_cost(INSN_COST);
15160 format %{ "tst $op1" %}
15161
15162 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15163
15164 ins_pipe(icmp_reg_imm);
15165 %}
15166
15167 instruct compL_reg_immLAddSub(rFlagsReg cr, iRegL op1, immLAddSub op2)
15168 %{
15169 match(Set cr (CmpL op1 op2));
15170
15171 effect(DEF cr, USE op1);
15172
15173 ins_cost(INSN_COST);
15174 format %{ "cmp $op1, $op2" %}
15175
15176 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15177
15178 ins_pipe(icmp_reg_imm);
15179 %}
15180
15181 instruct compL_reg_immL(rFlagsReg cr, iRegL op1, immL op2)
15182 %{
15183 match(Set cr (CmpL op1 op2));
15184
15185 effect(DEF cr, USE op1);
15186
15187 ins_cost(INSN_COST * 2);
15188 format %{ "cmp $op1, $op2" %}
15189
15190 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15191
15192 ins_pipe(icmp_reg_imm);
15193 %}
15194
15195 instruct compUL_reg_reg(rFlagsRegU cr, iRegL op1, iRegL op2)
15196 %{
15197 match(Set cr (CmpUL op1 op2));
15198
15199 effect(DEF cr, USE op1, USE op2);
15200
15201 ins_cost(INSN_COST);
15202 format %{ "cmp $op1, $op2" %}
15203
15204 ins_encode(aarch64_enc_cmp(op1, op2));
15205
15206 ins_pipe(icmp_reg_reg);
15207 %}
15208
15209 instruct compUL_reg_immL0(rFlagsRegU cr, iRegL op1, immL0 zero)
15210 %{
15211 match(Set cr (CmpUL op1 zero));
15212
15213 effect(DEF cr, USE op1);
15214
15215 ins_cost(INSN_COST);
15216 format %{ "tst $op1" %}
15217
15218 ins_encode(aarch64_enc_cmp_imm_addsub(op1, zero));
15219
15220 ins_pipe(icmp_reg_imm);
15221 %}
15222
15223 instruct compUL_reg_immLAddSub(rFlagsRegU cr, iRegL op1, immLAddSub op2)
15224 %{
15225 match(Set cr (CmpUL op1 op2));
15226
15227 effect(DEF cr, USE op1);
15228
15229 ins_cost(INSN_COST);
15230 format %{ "cmp $op1, $op2" %}
15231
15232 ins_encode(aarch64_enc_cmp_imm_addsub(op1, op2));
15233
15234 ins_pipe(icmp_reg_imm);
15235 %}
15236
15237 instruct compUL_reg_immL(rFlagsRegU cr, iRegL op1, immL op2)
15238 %{
15239 match(Set cr (CmpUL op1 op2));
15240
15241 effect(DEF cr, USE op1);
15242
15243 ins_cost(INSN_COST * 2);
15244 format %{ "cmp $op1, $op2" %}
15245
15246 ins_encode(aarch64_enc_cmp_imm(op1, op2));
15247
15248 ins_pipe(icmp_reg_imm);
15249 %}
15250
15251 instruct compP_reg_reg(rFlagsRegU cr, iRegP op1, iRegP op2)
15252 %{
15253 match(Set cr (CmpP op1 op2));
15254
15255 effect(DEF cr, USE op1, USE op2);
15256
15257 ins_cost(INSN_COST);
15258 format %{ "cmp $op1, $op2\t // ptr" %}
15259
15260 ins_encode(aarch64_enc_cmpp(op1, op2));
15261
15262 ins_pipe(icmp_reg_reg);
15263 %}
15264
15265 instruct compN_reg_reg(rFlagsRegU cr, iRegN op1, iRegN op2)
15266 %{
15267 match(Set cr (CmpN op1 op2));
15268
15269 effect(DEF cr, USE op1, USE op2);
15270
15271 ins_cost(INSN_COST);
15272 format %{ "cmp $op1, $op2\t // compressed ptr" %}
15273
15274 ins_encode(aarch64_enc_cmpn(op1, op2));
15275
15276 ins_pipe(icmp_reg_reg);
15277 %}
15278
15279 instruct testP_reg(rFlagsRegU cr, iRegP op1, immP0 zero)
15280 %{
15281 match(Set cr (CmpP op1 zero));
15282
15283 effect(DEF cr, USE op1, USE zero);
15284
15285 ins_cost(INSN_COST);
15286 format %{ "cmp $op1, 0\t // ptr" %}
15287
15288 ins_encode(aarch64_enc_testp(op1));
15289
15290 ins_pipe(icmp_reg_imm);
15291 %}
15292
15293 instruct testN_reg(rFlagsRegU cr, iRegN op1, immN0 zero)
15294 %{
15295 match(Set cr (CmpN op1 zero));
15296
15297 effect(DEF cr, USE op1, USE zero);
15298
15299 ins_cost(INSN_COST);
15300 format %{ "cmp $op1, 0\t // compressed ptr" %}
15301
15302 ins_encode(aarch64_enc_testn(op1));
15303
15304 ins_pipe(icmp_reg_imm);
15305 %}
15306
15307 // FP comparisons
15308 //
15309 // n.b. CmpF/CmpD set a normal flags reg which then gets compared
15310 // using normal cmpOp. See declaration of rFlagsReg for details.
15311
15312 instruct compF_reg_reg(rFlagsReg cr, vRegF src1, vRegF src2)
15313 %{
15314 match(Set cr (CmpF src1 src2));
15315
15316 ins_cost(3 * INSN_COST);
15317 format %{ "fcmps $src1, $src2" %}
15318
15319 ins_encode %{
15320 __ fcmps(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15321 %}
15322
15323 ins_pipe(pipe_class_compare);
15324 %}
15325
15326 instruct compF_reg_zero(rFlagsReg cr, vRegF src1, immF0 src2)
15327 %{
15328 match(Set cr (CmpF src1 src2));
15329
15330 ins_cost(3 * INSN_COST);
15331 format %{ "fcmps $src1, 0.0" %}
15332
15333 ins_encode %{
15334 __ fcmps(as_FloatRegister($src1$$reg), 0.0);
15335 %}
15336
15337 ins_pipe(pipe_class_compare);
15338 %}
15339 // FROM HERE
15340
15341 instruct compD_reg_reg(rFlagsReg cr, vRegD src1, vRegD src2)
15342 %{
15343 match(Set cr (CmpD src1 src2));
15344
15345 ins_cost(3 * INSN_COST);
15346 format %{ "fcmpd $src1, $src2" %}
15347
15348 ins_encode %{
15349 __ fcmpd(as_FloatRegister($src1$$reg), as_FloatRegister($src2$$reg));
15350 %}
15351
15352 ins_pipe(pipe_class_compare);
15353 %}
15354
15355 instruct compD_reg_zero(rFlagsReg cr, vRegD src1, immD0 src2)
15356 %{
15357 match(Set cr (CmpD src1 src2));
15358
15359 ins_cost(3 * INSN_COST);
15360 format %{ "fcmpd $src1, 0.0" %}
15361
15362 ins_encode %{
15363 __ fcmpd(as_FloatRegister($src1$$reg), 0.0);
15364 %}
15365
15366 ins_pipe(pipe_class_compare);
15367 %}
15368
15369 instruct compF3_reg_reg(iRegINoSp dst, vRegF src1, vRegF src2, rFlagsReg cr)
15370 %{
15371 match(Set dst (CmpF3 src1 src2));
15372 effect(KILL cr);
15373
15374 ins_cost(5 * INSN_COST);
15375 format %{ "fcmps $src1, $src2\n\t"
15376 "csinvw($dst, zr, zr, eq\n\t"
15377 "csnegw($dst, $dst, $dst, lt)"
15378 %}
15379
15380 ins_encode %{
15381 Label done;
15382 FloatRegister s1 = as_FloatRegister($src1$$reg);
15383 FloatRegister s2 = as_FloatRegister($src2$$reg);
15384 Register d = as_Register($dst$$reg);
15385 __ fcmps(s1, s2);
15386 // installs 0 if EQ else -1
15387 __ csinvw(d, zr, zr, Assembler::EQ);
15388 // keeps -1 if less or unordered else installs 1
15389 __ csnegw(d, d, d, Assembler::LT);
15390 __ bind(done);
15391 %}
15392
15393 ins_pipe(pipe_class_default);
15394
15395 %}
15396
15397 instruct compD3_reg_reg(iRegINoSp dst, vRegD src1, vRegD src2, rFlagsReg cr)
15398 %{
15399 match(Set dst (CmpD3 src1 src2));
15400 effect(KILL cr);
15401
15402 ins_cost(5 * INSN_COST);
15403 format %{ "fcmpd $src1, $src2\n\t"
15404 "csinvw($dst, zr, zr, eq\n\t"
15405 "csnegw($dst, $dst, $dst, lt)"
15406 %}
15407
15408 ins_encode %{
15409 Label done;
15410 FloatRegister s1 = as_FloatRegister($src1$$reg);
15411 FloatRegister s2 = as_FloatRegister($src2$$reg);
15412 Register d = as_Register($dst$$reg);
15413 __ fcmpd(s1, s2);
15414 // installs 0 if EQ else -1
15415 __ csinvw(d, zr, zr, Assembler::EQ);
15416 // keeps -1 if less or unordered else installs 1
15417 __ csnegw(d, d, d, Assembler::LT);
15418 __ bind(done);
15419 %}
15420 ins_pipe(pipe_class_default);
15421
15422 %}
15423
15424 instruct compF3_reg_immF0(iRegINoSp dst, vRegF src1, immF0 zero, rFlagsReg cr)
15425 %{
15426 match(Set dst (CmpF3 src1 zero));
15427 effect(KILL cr);
15428
15429 ins_cost(5 * INSN_COST);
15430 format %{ "fcmps $src1, 0.0\n\t"
15431 "csinvw($dst, zr, zr, eq\n\t"
15432 "csnegw($dst, $dst, $dst, lt)"
15433 %}
15434
15435 ins_encode %{
15436 Label done;
15437 FloatRegister s1 = as_FloatRegister($src1$$reg);
15438 Register d = as_Register($dst$$reg);
15439 __ fcmps(s1, 0.0);
15440 // installs 0 if EQ else -1
15441 __ csinvw(d, zr, zr, Assembler::EQ);
15442 // keeps -1 if less or unordered else installs 1
15443 __ csnegw(d, d, d, Assembler::LT);
15444 __ bind(done);
15445 %}
15446
15447 ins_pipe(pipe_class_default);
15448
15449 %}
15450
15451 instruct compD3_reg_immD0(iRegINoSp dst, vRegD src1, immD0 zero, rFlagsReg cr)
15452 %{
15453 match(Set dst (CmpD3 src1 zero));
15454 effect(KILL cr);
15455
15456 ins_cost(5 * INSN_COST);
15457 format %{ "fcmpd $src1, 0.0\n\t"
15458 "csinvw($dst, zr, zr, eq\n\t"
15459 "csnegw($dst, $dst, $dst, lt)"
15460 %}
15461
15462 ins_encode %{
15463 Label done;
15464 FloatRegister s1 = as_FloatRegister($src1$$reg);
15465 Register d = as_Register($dst$$reg);
15466 __ fcmpd(s1, 0.0);
15467 // installs 0 if EQ else -1
15468 __ csinvw(d, zr, zr, Assembler::EQ);
15469 // keeps -1 if less or unordered else installs 1
15470 __ csnegw(d, d, d, Assembler::LT);
15471 __ bind(done);
15472 %}
15473 ins_pipe(pipe_class_default);
15474
15475 %}
15476
15477 instruct cmpLTMask_reg_reg(iRegINoSp dst, iRegIorL2I p, iRegIorL2I q, rFlagsReg cr)
15478 %{
15479 match(Set dst (CmpLTMask p q));
15480 effect(KILL cr);
15481
15482 ins_cost(3 * INSN_COST);
15483
15484 format %{ "cmpw $p, $q\t# cmpLTMask\n\t"
15485 "csetw $dst, lt\n\t"
15486 "subw $dst, zr, $dst"
15487 %}
15488
15489 ins_encode %{
15490 __ cmpw(as_Register($p$$reg), as_Register($q$$reg));
15491 __ csetw(as_Register($dst$$reg), Assembler::LT);
15492 __ subw(as_Register($dst$$reg), zr, as_Register($dst$$reg));
15493 %}
15494
15495 ins_pipe(ialu_reg_reg);
15496 %}
15497
15498 instruct cmpLTMask_reg_zero(iRegINoSp dst, iRegIorL2I src, immI0 zero, rFlagsReg cr)
15499 %{
15500 match(Set dst (CmpLTMask src zero));
15501 effect(KILL cr);
15502
15503 ins_cost(INSN_COST);
15504
15505 format %{ "asrw $dst, $src, #31\t# cmpLTMask0" %}
15506
15507 ins_encode %{
15508 __ asrw(as_Register($dst$$reg), as_Register($src$$reg), 31);
15509 %}
15510
15511 ins_pipe(ialu_reg_shift);
15512 %}
15513
15514 // ============================================================================
15515 // Max and Min
15516
15517 // Like compI_reg_reg or compI_reg_immI0 but without match rule and second zero parameter.
15518
15519 instruct compI_reg_imm0(rFlagsReg cr, iRegI src)
15520 %{
15521 effect(DEF cr, USE src);
15522 ins_cost(INSN_COST);
15523 format %{ "cmpw $src, 0" %}
15524
15525 ins_encode %{
15526 __ cmpw($src$$Register, 0);
15527 %}
15528 ins_pipe(icmp_reg_imm);
15529 %}
15530
15531 instruct minI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15532 %{
15533 match(Set dst (MinI src1 src2));
15534 ins_cost(INSN_COST * 3);
15535
15536 expand %{
15537 rFlagsReg cr;
15538 compI_reg_reg(cr, src1, src2);
15539 cmovI_reg_reg_lt(dst, src1, src2, cr);
15540 %}
15541 %}
15542
15543 instruct maxI_reg_reg(iRegINoSp dst, iRegIorL2I src1, iRegIorL2I src2)
15544 %{
15545 match(Set dst (MaxI src1 src2));
15546 ins_cost(INSN_COST * 3);
15547
15548 expand %{
15549 rFlagsReg cr;
15550 compI_reg_reg(cr, src1, src2);
15551 cmovI_reg_reg_gt(dst, src1, src2, cr);
15552 %}
15553 %}
15554
15555
15556 // ============================================================================
15557 // Branch Instructions
15558
15559 // Direct Branch.
15560 instruct branch(label lbl)
15561 %{
15562 match(Goto);
15563
15564 effect(USE lbl);
15565
15566 ins_cost(BRANCH_COST);
15567 format %{ "b $lbl" %}
15568
15569 ins_encode(aarch64_enc_b(lbl));
15570
15571 ins_pipe(pipe_branch);
15572 %}
15573
15574 // Conditional Near Branch
15575 instruct branchCon(cmpOp cmp, rFlagsReg cr, label lbl)
15576 %{
15577 // Same match rule as `branchConFar'.
15578 match(If cmp cr);
15579
15580 effect(USE lbl);
15581
15582 ins_cost(BRANCH_COST);
15583 // If set to 1 this indicates that the current instruction is a
15584 // short variant of a long branch. This avoids using this
15585 // instruction in first-pass matching. It will then only be used in
15586 // the `Shorten_branches' pass.
15587 // ins_short_branch(1);
15588 format %{ "b$cmp $lbl" %}
15589
15590 ins_encode(aarch64_enc_br_con(cmp, lbl));
15591
15592 ins_pipe(pipe_branch_cond);
15593 %}
15594
15595 // Conditional Near Branch Unsigned
15596 instruct branchConU(cmpOpU cmp, rFlagsRegU cr, label lbl)
15597 %{
15598 // Same match rule as `branchConFar'.
15599 match(If cmp cr);
15600
15601 effect(USE lbl);
15602
15603 ins_cost(BRANCH_COST);
15604 // If set to 1 this indicates that the current instruction is a
15605 // short variant of a long branch. This avoids using this
15606 // instruction in first-pass matching. It will then only be used in
15607 // the `Shorten_branches' pass.
15608 // ins_short_branch(1);
15609 format %{ "b$cmp $lbl\t# unsigned" %}
15610
15611 ins_encode(aarch64_enc_br_conU(cmp, lbl));
15612
15613 ins_pipe(pipe_branch_cond);
15614 %}
15615
15616 // Make use of CBZ and CBNZ. These instructions, as well as being
15617 // shorter than (cmp; branch), have the additional benefit of not
15618 // killing the flags.
15619
15620 instruct cmpI_imm0_branch(cmpOpEqNe cmp, iRegIorL2I op1, immI0 op2, label labl, rFlagsReg cr) %{
15621 match(If cmp (CmpI op1 op2));
15622 effect(USE labl);
15623
15624 ins_cost(BRANCH_COST);
15625 format %{ "cbw$cmp $op1, $labl" %}
15626 ins_encode %{
15627 Label* L = $labl$$label;
15628 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15629 if (cond == Assembler::EQ)
15630 __ cbzw($op1$$Register, *L);
15631 else
15632 __ cbnzw($op1$$Register, *L);
15633 %}
15634 ins_pipe(pipe_cmp_branch);
15635 %}
15636
15637 instruct cmpL_imm0_branch(cmpOpEqNe cmp, iRegL op1, immL0 op2, label labl, rFlagsReg cr) %{
15638 match(If cmp (CmpL op1 op2));
15639 effect(USE labl);
15640
15641 ins_cost(BRANCH_COST);
15642 format %{ "cb$cmp $op1, $labl" %}
15643 ins_encode %{
15644 Label* L = $labl$$label;
15645 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15646 if (cond == Assembler::EQ)
15647 __ cbz($op1$$Register, *L);
15648 else
15649 __ cbnz($op1$$Register, *L);
15650 %}
15651 ins_pipe(pipe_cmp_branch);
15652 %}
15653
15654 instruct cmpP_imm0_branch(cmpOpEqNe cmp, iRegP op1, immP0 op2, label labl, rFlagsReg cr) %{
15655 match(If cmp (CmpP op1 op2));
15656 effect(USE labl);
15657
15658 ins_cost(BRANCH_COST);
15659 format %{ "cb$cmp $op1, $labl" %}
15660 ins_encode %{
15661 Label* L = $labl$$label;
15662 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15663 if (cond == Assembler::EQ)
15664 __ cbz($op1$$Register, *L);
15665 else
15666 __ cbnz($op1$$Register, *L);
15667 %}
15668 ins_pipe(pipe_cmp_branch);
15669 %}
15670
15671 instruct cmpN_imm0_branch(cmpOpEqNe cmp, iRegN op1, immN0 op2, label labl, rFlagsReg cr) %{
15672 match(If cmp (CmpN op1 op2));
15673 effect(USE labl);
15674
15675 ins_cost(BRANCH_COST);
15676 format %{ "cbw$cmp $op1, $labl" %}
15677 ins_encode %{
15678 Label* L = $labl$$label;
15679 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15680 if (cond == Assembler::EQ)
15681 __ cbzw($op1$$Register, *L);
15682 else
15683 __ cbnzw($op1$$Register, *L);
15684 %}
15685 ins_pipe(pipe_cmp_branch);
15686 %}
15687
15688 instruct cmpP_narrowOop_imm0_branch(cmpOpEqNe cmp, iRegN oop, immP0 zero, label labl, rFlagsReg cr) %{
15689 match(If cmp (CmpP (DecodeN oop) zero));
15690 effect(USE labl);
15691
15692 ins_cost(BRANCH_COST);
15693 format %{ "cb$cmp $oop, $labl" %}
15694 ins_encode %{
15695 Label* L = $labl$$label;
15696 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15697 if (cond == Assembler::EQ)
15698 __ cbzw($oop$$Register, *L);
15699 else
15700 __ cbnzw($oop$$Register, *L);
15701 %}
15702 ins_pipe(pipe_cmp_branch);
15703 %}
15704
15705 instruct cmpUI_imm0_branch(cmpOpUEqNeLeGt cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15706 match(If cmp (CmpU op1 op2));
15707 effect(USE labl);
15708
15709 ins_cost(BRANCH_COST);
15710 format %{ "cbw$cmp $op1, $labl" %}
15711 ins_encode %{
15712 Label* L = $labl$$label;
15713 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15714 if (cond == Assembler::EQ || cond == Assembler::LS) {
15715 __ cbzw($op1$$Register, *L);
15716 } else {
15717 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15718 __ cbnzw($op1$$Register, *L);
15719 }
15720 %}
15721 ins_pipe(pipe_cmp_branch);
15722 %}
15723
15724 instruct cmpUL_imm0_branch(cmpOpUEqNeLeGt cmp, iRegL op1, immL0 op2, label labl) %{
15725 match(If cmp (CmpUL op1 op2));
15726 effect(USE labl);
15727
15728 ins_cost(BRANCH_COST);
15729 format %{ "cb$cmp $op1, $labl" %}
15730 ins_encode %{
15731 Label* L = $labl$$label;
15732 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15733 if (cond == Assembler::EQ || cond == Assembler::LS) {
15734 __ cbz($op1$$Register, *L);
15735 } else {
15736 assert(cond == Assembler::NE || cond == Assembler::HI, "unexpected condition");
15737 __ cbnz($op1$$Register, *L);
15738 }
15739 %}
15740 ins_pipe(pipe_cmp_branch);
15741 %}
15742
15743 // Test bit and Branch
15744
15745 // Patterns for short (< 32KiB) variants
15746 instruct cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15747 match(If cmp (CmpL op1 op2));
15748 effect(USE labl);
15749
15750 ins_cost(BRANCH_COST);
15751 format %{ "cb$cmp $op1, $labl # long" %}
15752 ins_encode %{
15753 Label* L = $labl$$label;
15754 Assembler::Condition cond =
15755 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15756 __ tbr(cond, $op1$$Register, 63, *L);
15757 %}
15758 ins_pipe(pipe_cmp_branch);
15759 ins_short_branch(1);
15760 %}
15761
15762 instruct cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15763 match(If cmp (CmpI op1 op2));
15764 effect(USE labl);
15765
15766 ins_cost(BRANCH_COST);
15767 format %{ "cb$cmp $op1, $labl # int" %}
15768 ins_encode %{
15769 Label* L = $labl$$label;
15770 Assembler::Condition cond =
15771 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15772 __ tbr(cond, $op1$$Register, 31, *L);
15773 %}
15774 ins_pipe(pipe_cmp_branch);
15775 ins_short_branch(1);
15776 %}
15777
15778 instruct cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15779 match(If cmp (CmpL (AndL op1 op2) op3));
15780 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15781 effect(USE labl);
15782
15783 ins_cost(BRANCH_COST);
15784 format %{ "tb$cmp $op1, $op2, $labl" %}
15785 ins_encode %{
15786 Label* L = $labl$$label;
15787 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15788 int bit = exact_log2_long($op2$$constant);
15789 __ tbr(cond, $op1$$Register, bit, *L);
15790 %}
15791 ins_pipe(pipe_cmp_branch);
15792 ins_short_branch(1);
15793 %}
15794
15795 instruct cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15796 match(If cmp (CmpI (AndI op1 op2) op3));
15797 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15798 effect(USE labl);
15799
15800 ins_cost(BRANCH_COST);
15801 format %{ "tb$cmp $op1, $op2, $labl" %}
15802 ins_encode %{
15803 Label* L = $labl$$label;
15804 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15805 int bit = exact_log2((juint)$op2$$constant);
15806 __ tbr(cond, $op1$$Register, bit, *L);
15807 %}
15808 ins_pipe(pipe_cmp_branch);
15809 ins_short_branch(1);
15810 %}
15811
15812 // And far variants
15813 instruct far_cmpL_branch_sign(cmpOpLtGe cmp, iRegL op1, immL0 op2, label labl) %{
15814 match(If cmp (CmpL op1 op2));
15815 effect(USE labl);
15816
15817 ins_cost(BRANCH_COST);
15818 format %{ "cb$cmp $op1, $labl # long" %}
15819 ins_encode %{
15820 Label* L = $labl$$label;
15821 Assembler::Condition cond =
15822 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15823 __ tbr(cond, $op1$$Register, 63, *L, /*far*/true);
15824 %}
15825 ins_pipe(pipe_cmp_branch);
15826 %}
15827
15828 instruct far_cmpI_branch_sign(cmpOpLtGe cmp, iRegIorL2I op1, immI0 op2, label labl) %{
15829 match(If cmp (CmpI op1 op2));
15830 effect(USE labl);
15831
15832 ins_cost(BRANCH_COST);
15833 format %{ "cb$cmp $op1, $labl # int" %}
15834 ins_encode %{
15835 Label* L = $labl$$label;
15836 Assembler::Condition cond =
15837 ((Assembler::Condition)$cmp$$cmpcode == Assembler::LT) ? Assembler::NE : Assembler::EQ;
15838 __ tbr(cond, $op1$$Register, 31, *L, /*far*/true);
15839 %}
15840 ins_pipe(pipe_cmp_branch);
15841 %}
15842
15843 instruct far_cmpL_branch_bit(cmpOpEqNe cmp, iRegL op1, immL op2, immL0 op3, label labl) %{
15844 match(If cmp (CmpL (AndL op1 op2) op3));
15845 predicate(is_power_of_2((julong)n->in(2)->in(1)->in(2)->get_long()));
15846 effect(USE labl);
15847
15848 ins_cost(BRANCH_COST);
15849 format %{ "tb$cmp $op1, $op2, $labl" %}
15850 ins_encode %{
15851 Label* L = $labl$$label;
15852 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15853 int bit = exact_log2_long($op2$$constant);
15854 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15855 %}
15856 ins_pipe(pipe_cmp_branch);
15857 %}
15858
15859 instruct far_cmpI_branch_bit(cmpOpEqNe cmp, iRegIorL2I op1, immI op2, immI0 op3, label labl) %{
15860 match(If cmp (CmpI (AndI op1 op2) op3));
15861 predicate(is_power_of_2((juint)n->in(2)->in(1)->in(2)->get_int()));
15862 effect(USE labl);
15863
15864 ins_cost(BRANCH_COST);
15865 format %{ "tb$cmp $op1, $op2, $labl" %}
15866 ins_encode %{
15867 Label* L = $labl$$label;
15868 Assembler::Condition cond = (Assembler::Condition)$cmp$$cmpcode;
15869 int bit = exact_log2((juint)$op2$$constant);
15870 __ tbr(cond, $op1$$Register, bit, *L, /*far*/true);
15871 %}
15872 ins_pipe(pipe_cmp_branch);
15873 %}
15874
15875 // Test bits
15876
15877 instruct cmpL_and(cmpOp cmp, iRegL op1, immL op2, immL0 op3, rFlagsReg cr) %{
15878 match(Set cr (CmpL (AndL op1 op2) op3));
15879 predicate(Assembler::operand_valid_for_logical_immediate
15880 (/*is_32*/false, n->in(1)->in(2)->get_long()));
15881
15882 ins_cost(INSN_COST);
15883 format %{ "tst $op1, $op2 # long" %}
15884 ins_encode %{
15885 __ tst($op1$$Register, $op2$$constant);
15886 %}
15887 ins_pipe(ialu_reg_reg);
15888 %}
15889
15890 instruct cmpI_and(cmpOp cmp, iRegIorL2I op1, immI op2, immI0 op3, rFlagsReg cr) %{
15891 match(Set cr (CmpI (AndI op1 op2) op3));
15892 predicate(Assembler::operand_valid_for_logical_immediate
15893 (/*is_32*/true, n->in(1)->in(2)->get_int()));
15894
15895 ins_cost(INSN_COST);
15896 format %{ "tst $op1, $op2 # int" %}
15897 ins_encode %{
15898 __ tstw($op1$$Register, $op2$$constant);
15899 %}
15900 ins_pipe(ialu_reg_reg);
15901 %}
15902
15903 instruct cmpL_and_reg(cmpOp cmp, iRegL op1, iRegL op2, immL0 op3, rFlagsReg cr) %{
15904 match(Set cr (CmpL (AndL op1 op2) op3));
15905
15906 ins_cost(INSN_COST);
15907 format %{ "tst $op1, $op2 # long" %}
15908 ins_encode %{
15909 __ tst($op1$$Register, $op2$$Register);
15910 %}
15911 ins_pipe(ialu_reg_reg);
15912 %}
15913
15914 instruct cmpI_and_reg(cmpOp cmp, iRegIorL2I op1, iRegIorL2I op2, immI0 op3, rFlagsReg cr) %{
15915 match(Set cr (CmpI (AndI op1 op2) op3));
15916
15917 ins_cost(INSN_COST);
15918 format %{ "tstw $op1, $op2 # int" %}
15919 ins_encode %{
15920 __ tstw($op1$$Register, $op2$$Register);
15921 %}
15922 ins_pipe(ialu_reg_reg);
15923 %}
15924
15925
15926 // Conditional Far Branch
15927 // Conditional Far Branch Unsigned
15928 // TODO: fixme
15929
15930 // counted loop end branch near
15931 instruct branchLoopEnd(cmpOp cmp, rFlagsReg cr, label lbl)
15932 %{
15933 match(CountedLoopEnd cmp cr);
15934
15935 effect(USE lbl);
15936
15937 ins_cost(BRANCH_COST);
15938 // short variant.
15939 // ins_short_branch(1);
15940 format %{ "b$cmp $lbl \t// counted loop end" %}
15941
15942 ins_encode(aarch64_enc_br_con(cmp, lbl));
15943
15944 ins_pipe(pipe_branch);
15945 %}
15946
15947 // counted loop end branch far
15948 // TODO: fixme
15949
15950 // ============================================================================
15951 // inlined locking and unlocking
15952
15953 instruct cmpFastLock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15954 %{
15955 predicate(LockingMode != LM_LIGHTWEIGHT);
15956 match(Set cr (FastLock object box));
15957 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15958
15959 ins_cost(5 * INSN_COST);
15960 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15961
15962 ins_encode %{
15963 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15964 %}
15965
15966 ins_pipe(pipe_serial);
15967 %}
15968
15969 instruct cmpFastUnlock(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2)
15970 %{
15971 predicate(LockingMode != LM_LIGHTWEIGHT);
15972 match(Set cr (FastUnlock object box));
15973 effect(TEMP tmp, TEMP tmp2);
15974
15975 ins_cost(5 * INSN_COST);
15976 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2" %}
15977
15978 ins_encode %{
15979 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register);
15980 %}
15981
15982 ins_pipe(pipe_serial);
15983 %}
15984
15985 instruct cmpFastLockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
15986 %{
15987 predicate(LockingMode == LM_LIGHTWEIGHT);
15988 match(Set cr (FastLock object box));
15989 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
15990
15991 ins_cost(5 * INSN_COST);
15992 format %{ "fastlock $object,$box\t! kills $tmp,$tmp2,$tmp3" %}
15993
15994 ins_encode %{
15995 __ fast_lock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
15996 %}
15997
15998 ins_pipe(pipe_serial);
15999 %}
16000
16001 instruct cmpFastUnlockLightweight(rFlagsReg cr, iRegP object, iRegP box, iRegPNoSp tmp, iRegPNoSp tmp2, iRegPNoSp tmp3)
16002 %{
16003 predicate(LockingMode == LM_LIGHTWEIGHT);
16004 match(Set cr (FastUnlock object box));
16005 effect(TEMP tmp, TEMP tmp2, TEMP tmp3);
16006
16007 ins_cost(5 * INSN_COST);
16008 format %{ "fastunlock $object,$box\t! kills $tmp, $tmp2, $tmp3" %}
16009
16010 ins_encode %{
16011 __ fast_unlock_lightweight($object$$Register, $box$$Register, $tmp$$Register, $tmp2$$Register, $tmp3$$Register);
16012 %}
16013
16014 ins_pipe(pipe_serial);
16015 %}
16016
16017 // ============================================================================
16018 // Safepoint Instructions
16019
16020 // TODO
16021 // provide a near and far version of this code
16022
16023 instruct safePoint(rFlagsReg cr, iRegP poll)
16024 %{
16025 match(SafePoint poll);
16026 effect(KILL cr);
16027
16028 format %{
16029 "ldrw zr, [$poll]\t# Safepoint: poll for GC"
16030 %}
16031 ins_encode %{
16032 __ read_polling_page(as_Register($poll$$reg), relocInfo::poll_type);
16033 %}
16034 ins_pipe(pipe_serial); // ins_pipe(iload_reg_mem);
16035 %}
16036
16037
16038 // ============================================================================
16039 // Procedure Call/Return Instructions
16040
16041 // Call Java Static Instruction
16042
16043 instruct CallStaticJavaDirect(method meth)
16044 %{
16045 match(CallStaticJava);
16046
16047 effect(USE meth);
16048
16049 ins_cost(CALL_COST);
16050
16051 format %{ "call,static $meth \t// ==> " %}
16052
16053 ins_encode(aarch64_enc_java_static_call(meth),
16054 aarch64_enc_call_epilog);
16055
16056 ins_pipe(pipe_class_call);
16057 %}
16058
16059 // TO HERE
16060
16061 // Call Java Dynamic Instruction
16062 instruct CallDynamicJavaDirect(method meth)
16063 %{
16064 match(CallDynamicJava);
16065
16066 effect(USE meth);
16067
16068 ins_cost(CALL_COST);
16069
16070 format %{ "CALL,dynamic $meth \t// ==> " %}
16071
16072 ins_encode(aarch64_enc_java_dynamic_call(meth),
16073 aarch64_enc_call_epilog);
16074
16075 ins_pipe(pipe_class_call);
16076 %}
16077
16078 // Call Runtime Instruction
16079
16080 instruct CallRuntimeDirect(method meth)
16081 %{
16082 match(CallRuntime);
16083
16084 effect(USE meth);
16085
16086 ins_cost(CALL_COST);
16087
16088 format %{ "CALL, runtime $meth" %}
16089
16090 ins_encode( aarch64_enc_java_to_runtime(meth) );
16091
16092 ins_pipe(pipe_class_call);
16093 %}
16094
16095 // Call Runtime Instruction
16096
16097 instruct CallLeafDirect(method meth)
16098 %{
16099 match(CallLeaf);
16100
16101 effect(USE meth);
16102
16103 ins_cost(CALL_COST);
16104
16105 format %{ "CALL, runtime leaf $meth" %}
16106
16107 ins_encode( aarch64_enc_java_to_runtime(meth) );
16108
16109 ins_pipe(pipe_class_call);
16110 %}
16111
16112 // Call Runtime Instruction
16113
16114 instruct CallLeafNoFPDirect(method meth)
16115 %{
16116 match(CallLeafNoFP);
16117
16118 effect(USE meth);
16119
16120 ins_cost(CALL_COST);
16121
16122 format %{ "CALL, runtime leaf nofp $meth" %}
16123
16124 ins_encode( aarch64_enc_java_to_runtime(meth) );
16125
16126 ins_pipe(pipe_class_call);
16127 %}
16128
16129 // Tail Call; Jump from runtime stub to Java code.
16130 // Also known as an 'interprocedural jump'.
16131 // Target of jump will eventually return to caller.
16132 // TailJump below removes the return address.
16133 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16134 // emitted just above the TailCall which has reset rfp to the caller state.
16135 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
16136 %{
16137 match(TailCall jump_target method_ptr);
16138
16139 ins_cost(CALL_COST);
16140
16141 format %{ "br $jump_target\t# $method_ptr holds method" %}
16142
16143 ins_encode(aarch64_enc_tail_call(jump_target));
16144
16145 ins_pipe(pipe_class_call);
16146 %}
16147
16148 instruct TailjmpInd(iRegPNoSpNoRfp jump_target, iRegP_R0 ex_oop)
16149 %{
16150 match(TailJump jump_target ex_oop);
16151
16152 ins_cost(CALL_COST);
16153
16154 format %{ "br $jump_target\t# $ex_oop holds exception oop" %}
16155
16156 ins_encode(aarch64_enc_tail_jmp(jump_target));
16157
16158 ins_pipe(pipe_class_call);
16159 %}
16160
16161 // Forward exception.
16162 instruct ForwardExceptionjmp()
16163 %{
16164 match(ForwardException);
16165 ins_cost(CALL_COST);
16166
16167 format %{ "b forward_exception_stub" %}
16168 ins_encode %{
16169 __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
16170 %}
16171 ins_pipe(pipe_class_call);
16172 %}
16173
16174 // Create exception oop: created by stack-crawling runtime code.
16175 // Created exception is now available to this handler, and is setup
16176 // just prior to jumping to this handler. No code emitted.
16177 // TODO check
16178 // should ex_oop be in r0? intel uses rax, ppc cannot use r0 so uses rarg1
16179 instruct CreateException(iRegP_R0 ex_oop)
16180 %{
16181 match(Set ex_oop (CreateEx));
16182
16183 format %{ " -- \t// exception oop; no code emitted" %}
16184
16185 size(0);
16186
16187 ins_encode( /*empty*/ );
16188
16189 ins_pipe(pipe_class_empty);
16190 %}
16191
16192 // Rethrow exception: The exception oop will come in the first
16193 // argument position. Then JUMP (not call) to the rethrow stub code.
16194 instruct RethrowException() %{
16195 match(Rethrow);
16196 ins_cost(CALL_COST);
16197
16198 format %{ "b rethrow_stub" %}
16199
16200 ins_encode( aarch64_enc_rethrow() );
16201
16202 ins_pipe(pipe_class_call);
16203 %}
16204
16205
16206 // Return Instruction
16207 // epilog node loads ret address into lr as part of frame pop
16208 instruct Ret()
16209 %{
16210 match(Return);
16211
16212 format %{ "ret\t// return register" %}
16213
16214 ins_encode( aarch64_enc_ret() );
16215
16216 ins_pipe(pipe_branch);
16217 %}
16218
16219 // Die now.
16220 instruct ShouldNotReachHere() %{
16221 match(Halt);
16222
16223 ins_cost(CALL_COST);
16224 format %{ "ShouldNotReachHere" %}
16225
16226 ins_encode %{
16227 if (is_reachable()) {
16228 __ stop(_halt_reason);
16229 }
16230 %}
16231
16232 ins_pipe(pipe_class_default);
16233 %}
16234
16235 // ============================================================================
16236 // Partial Subtype Check
16237 //
16238 // superklass array for an instance of the superklass. Set a hidden
16239 // internal cache on a hit (cache is checked with exposed code in
16240 // gen_subtype_check()). Return NZ for a miss or zero for a hit. The
16241 // encoding ALSO sets flags.
16242
16243 instruct partialSubtypeCheck(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, rFlagsReg cr)
16244 %{
16245 match(Set result (PartialSubtypeCheck sub super));
16246 effect(KILL cr, KILL temp);
16247
16248 ins_cost(1100); // slightly larger than the next version
16249 format %{ "partialSubtypeCheck $result, $sub, $super" %}
16250
16251 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16252
16253 opcode(0x1); // Force zero of result reg on hit
16254
16255 ins_pipe(pipe_class_memory);
16256 %}
16257
16258 instruct partialSubtypeCheckConstSuper(iRegP_R4 sub, iRegP_R0 super_reg, immP super_con, vRegD_V0 vtemp, iRegP_R5 result,
16259 iRegP_R1 tempR1, iRegP_R2 tempR2, iRegP_R3 tempR3,
16260 rFlagsReg cr)
16261 %{
16262 match(Set result (PartialSubtypeCheck sub (Binary super_reg super_con)));
16263 predicate(UseSecondarySupersTable);
16264 effect(KILL cr, TEMP tempR1, TEMP tempR2, TEMP tempR3, TEMP vtemp);
16265
16266 ins_cost(700); // smaller than the next version
16267 format %{ "partialSubtypeCheck $result, $sub, $super_reg, $super_con" %}
16268
16269 ins_encode %{
16270 bool success = false;
16271 u1 super_klass_slot = ((Klass*)$super_con$$constant)->hash_slot();
16272 if (InlineSecondarySupersTest) {
16273 success = __ lookup_secondary_supers_table($sub$$Register, $super_reg$$Register,
16274 $tempR1$$Register, $tempR2$$Register, $tempR3$$Register,
16275 $vtemp$$FloatRegister,
16276 $result$$Register,
16277 super_klass_slot);
16278 } else {
16279 address call = __ trampoline_call(RuntimeAddress(StubRoutines::lookup_secondary_supers_table_stub(super_klass_slot)));
16280 success = (call != nullptr);
16281 }
16282 if (!success) {
16283 ciEnv::current()->record_failure("CodeCache is full");
16284 return;
16285 }
16286 %}
16287
16288 ins_pipe(pipe_class_memory);
16289 %}
16290
16291 instruct partialSubtypeCheckVsZero(iRegP_R4 sub, iRegP_R0 super, iRegP_R2 temp, iRegP_R5 result, immP0 zero, rFlagsReg cr)
16292 %{
16293 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
16294 effect(KILL temp, KILL result);
16295
16296 ins_cost(1100); // slightly larger than the next version
16297 format %{ "partialSubtypeCheck $result, $sub, $super == 0" %}
16298
16299 ins_encode(aarch64_enc_partial_subtype_check(sub, super, temp, result));
16300
16301 opcode(0x0); // Don't zero result reg on hit
16302
16303 ins_pipe(pipe_class_memory);
16304 %}
16305
16306 // Intrisics for String.compareTo()
16307
16308 instruct string_compareU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16309 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16310 %{
16311 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16312 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16313 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16314
16315 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16316 ins_encode %{
16317 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16318 __ string_compare($str1$$Register, $str2$$Register,
16319 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16320 $tmp1$$Register, $tmp2$$Register,
16321 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::UU);
16322 %}
16323 ins_pipe(pipe_class_memory);
16324 %}
16325
16326 instruct string_compareL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16327 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2, rFlagsReg cr)
16328 %{
16329 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16330 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16331 effect(KILL tmp1, KILL tmp2, USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16332
16333 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1" %}
16334 ins_encode %{
16335 __ string_compare($str1$$Register, $str2$$Register,
16336 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16337 $tmp1$$Register, $tmp2$$Register,
16338 fnoreg, fnoreg, fnoreg, pnoreg, pnoreg, StrIntrinsicNode::LL);
16339 %}
16340 ins_pipe(pipe_class_memory);
16341 %}
16342
16343 instruct string_compareUL(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16344 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16345 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16346 %{
16347 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16348 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16349 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16350 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16351
16352 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16353 ins_encode %{
16354 __ string_compare($str1$$Register, $str2$$Register,
16355 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16356 $tmp1$$Register, $tmp2$$Register,
16357 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16358 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::UL);
16359 %}
16360 ins_pipe(pipe_class_memory);
16361 %}
16362
16363 instruct string_compareLU(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16364 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16365 vRegD_V0 vtmp1, vRegD_V1 vtmp2, vRegD_V2 vtmp3, rFlagsReg cr)
16366 %{
16367 predicate((UseSVE == 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16368 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16369 effect(KILL tmp1, KILL tmp2, KILL vtmp1, KILL vtmp2, KILL vtmp3,
16370 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16371
16372 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # KILL $tmp1, $tmp2, $vtmp1, $vtmp2, $vtmp3" %}
16373 ins_encode %{
16374 __ string_compare($str1$$Register, $str2$$Register,
16375 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16376 $tmp1$$Register, $tmp2$$Register,
16377 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister,
16378 $vtmp3$$FloatRegister, pnoreg, pnoreg, StrIntrinsicNode::LU);
16379 %}
16380 ins_pipe(pipe_class_memory);
16381 %}
16382
16383 // Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
16384 // these string_compare variants as NEON register type for convenience so that the prototype of
16385 // string_compare can be shared with all variants.
16386
16387 instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16388 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16389 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16390 pRegGov_P1 pgtmp2, rFlagsReg cr)
16391 %{
16392 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
16393 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16394 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16395 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16396
16397 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16398 ins_encode %{
16399 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16400 __ string_compare($str1$$Register, $str2$$Register,
16401 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16402 $tmp1$$Register, $tmp2$$Register,
16403 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16404 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16405 StrIntrinsicNode::LL);
16406 %}
16407 ins_pipe(pipe_class_memory);
16408 %}
16409
16410 instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16411 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16412 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16413 pRegGov_P1 pgtmp2, rFlagsReg cr)
16414 %{
16415 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
16416 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16417 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16418 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16419
16420 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16421 ins_encode %{
16422 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16423 __ string_compare($str1$$Register, $str2$$Register,
16424 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16425 $tmp1$$Register, $tmp2$$Register,
16426 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16427 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16428 StrIntrinsicNode::LU);
16429 %}
16430 ins_pipe(pipe_class_memory);
16431 %}
16432
16433 instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16434 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16435 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16436 pRegGov_P1 pgtmp2, rFlagsReg cr)
16437 %{
16438 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
16439 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16440 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16441 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16442
16443 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16444 ins_encode %{
16445 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16446 __ string_compare($str1$$Register, $str2$$Register,
16447 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16448 $tmp1$$Register, $tmp2$$Register,
16449 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16450 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16451 StrIntrinsicNode::UL);
16452 %}
16453 ins_pipe(pipe_class_memory);
16454 %}
16455
16456 instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
16457 iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
16458 vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
16459 pRegGov_P1 pgtmp2, rFlagsReg cr)
16460 %{
16461 predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
16462 match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
16463 effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
16464 USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);
16465
16466 format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result # USE sve" %}
16467 ins_encode %{
16468 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16469 __ string_compare($str1$$Register, $str2$$Register,
16470 $cnt1$$Register, $cnt2$$Register, $result$$Register,
16471 $tmp1$$Register, $tmp2$$Register,
16472 $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
16473 as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
16474 StrIntrinsicNode::UU);
16475 %}
16476 ins_pipe(pipe_class_memory);
16477 %}
16478
16479 instruct string_indexofUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16480 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16481 iRegINoSp tmp3, iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16482 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16483 %{
16484 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16485 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16486 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16487 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16488 TEMP vtmp0, TEMP vtmp1, KILL cr);
16489 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UU) "
16490 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16491
16492 ins_encode %{
16493 __ string_indexof($str1$$Register, $str2$$Register,
16494 $cnt1$$Register, $cnt2$$Register,
16495 $tmp1$$Register, $tmp2$$Register,
16496 $tmp3$$Register, $tmp4$$Register,
16497 $tmp5$$Register, $tmp6$$Register,
16498 -1, $result$$Register, StrIntrinsicNode::UU);
16499 %}
16500 ins_pipe(pipe_class_memory);
16501 %}
16502
16503 instruct string_indexofLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16504 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2, iRegINoSp tmp3,
16505 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16506 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16507 %{
16508 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16509 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16510 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16511 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5, TEMP tmp6,
16512 TEMP vtmp0, TEMP vtmp1, KILL cr);
16513 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (LL) "
16514 "# KILL $str1 $cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16515
16516 ins_encode %{
16517 __ string_indexof($str1$$Register, $str2$$Register,
16518 $cnt1$$Register, $cnt2$$Register,
16519 $tmp1$$Register, $tmp2$$Register,
16520 $tmp3$$Register, $tmp4$$Register,
16521 $tmp5$$Register, $tmp6$$Register,
16522 -1, $result$$Register, StrIntrinsicNode::LL);
16523 %}
16524 ins_pipe(pipe_class_memory);
16525 %}
16526
16527 instruct string_indexofUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2, iRegI_R2 cnt2,
16528 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,iRegINoSp tmp3,
16529 iRegINoSp tmp4, iRegINoSp tmp5, iRegINoSp tmp6,
16530 vRegD_V0 vtmp0, vRegD_V1 vtmp1, rFlagsReg cr)
16531 %{
16532 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16533 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 cnt2)));
16534 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2,
16535 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP tmp5,
16536 TEMP tmp6, TEMP vtmp0, TEMP vtmp1, KILL cr);
16537 format %{ "String IndexOf $str1,$cnt1,$str2,$cnt2 -> $result (UL) "
16538 "# KILL $str1 cnt1 $str2 $cnt2 $tmp1 $tmp2 $tmp3 $tmp4 $tmp5 $tmp6 V0-V1 cr" %}
16539
16540 ins_encode %{
16541 __ string_indexof($str1$$Register, $str2$$Register,
16542 $cnt1$$Register, $cnt2$$Register,
16543 $tmp1$$Register, $tmp2$$Register,
16544 $tmp3$$Register, $tmp4$$Register,
16545 $tmp5$$Register, $tmp6$$Register,
16546 -1, $result$$Register, StrIntrinsicNode::UL);
16547 %}
16548 ins_pipe(pipe_class_memory);
16549 %}
16550
16551 instruct string_indexof_conUU(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16552 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16553 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16554 %{
16555 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UU);
16556 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16557 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16558 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16559 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UU) "
16560 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16561
16562 ins_encode %{
16563 int icnt2 = (int)$int_cnt2$$constant;
16564 __ string_indexof($str1$$Register, $str2$$Register,
16565 $cnt1$$Register, zr,
16566 $tmp1$$Register, $tmp2$$Register,
16567 $tmp3$$Register, $tmp4$$Register, zr, zr,
16568 icnt2, $result$$Register, StrIntrinsicNode::UU);
16569 %}
16570 ins_pipe(pipe_class_memory);
16571 %}
16572
16573 instruct string_indexof_conLL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16574 immI_le_4 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16575 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16576 %{
16577 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::LL);
16578 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16579 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16580 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16581 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (LL) "
16582 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16583
16584 ins_encode %{
16585 int icnt2 = (int)$int_cnt2$$constant;
16586 __ string_indexof($str1$$Register, $str2$$Register,
16587 $cnt1$$Register, zr,
16588 $tmp1$$Register, $tmp2$$Register,
16589 $tmp3$$Register, $tmp4$$Register, zr, zr,
16590 icnt2, $result$$Register, StrIntrinsicNode::LL);
16591 %}
16592 ins_pipe(pipe_class_memory);
16593 %}
16594
16595 instruct string_indexof_conUL(iRegP_R1 str1, iRegI_R4 cnt1, iRegP_R3 str2,
16596 immI_1 int_cnt2, iRegI_R0 result, iRegINoSp tmp1,
16597 iRegINoSp tmp2, iRegINoSp tmp3, iRegINoSp tmp4, rFlagsReg cr)
16598 %{
16599 predicate(((StrIndexOfNode*)n)->encoding() == StrIntrinsicNode::UL);
16600 match(Set result (StrIndexOf (Binary str1 cnt1) (Binary str2 int_cnt2)));
16601 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt1,
16602 TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, KILL cr);
16603 format %{ "String IndexOf $str1,$cnt1,$str2,$int_cnt2 -> $result (UL) "
16604 "# KILL $str1 $cnt1 $str2 $tmp1 $tmp2 $tmp3 $tmp4 cr" %}
16605
16606 ins_encode %{
16607 int icnt2 = (int)$int_cnt2$$constant;
16608 __ string_indexof($str1$$Register, $str2$$Register,
16609 $cnt1$$Register, zr,
16610 $tmp1$$Register, $tmp2$$Register,
16611 $tmp3$$Register, $tmp4$$Register, zr, zr,
16612 icnt2, $result$$Register, StrIntrinsicNode::UL);
16613 %}
16614 ins_pipe(pipe_class_memory);
16615 %}
16616
16617 instruct string_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16618 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16619 iRegINoSp tmp3, rFlagsReg cr)
16620 %{
16621 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16622 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
16623 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16624 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16625
16626 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16627
16628 ins_encode %{
16629 __ string_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16630 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16631 $tmp3$$Register);
16632 %}
16633 ins_pipe(pipe_class_memory);
16634 %}
16635
16636 instruct stringL_indexof_char(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16637 iRegI_R0 result, iRegINoSp tmp1, iRegINoSp tmp2,
16638 iRegINoSp tmp3, rFlagsReg cr)
16639 %{
16640 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16641 predicate((UseSVE == 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
16642 effect(USE_KILL str1, USE_KILL cnt1, USE_KILL ch,
16643 TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
16644
16645 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result" %}
16646
16647 ins_encode %{
16648 __ stringL_indexof_char($str1$$Register, $cnt1$$Register, $ch$$Register,
16649 $result$$Register, $tmp1$$Register, $tmp2$$Register,
16650 $tmp3$$Register);
16651 %}
16652 ins_pipe(pipe_class_memory);
16653 %}
16654
16655 instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16656 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16657 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16658 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L);
16659 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16660 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16661 format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16662 ins_encode %{
16663 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16664 $result$$Register, $ztmp1$$FloatRegister,
16665 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16666 $ptmp$$PRegister, true /* isL */);
16667 %}
16668 ins_pipe(pipe_class_memory);
16669 %}
16670
16671 instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
16672 iRegI_R0 result, vecA ztmp1, vecA ztmp2,
16673 pRegGov pgtmp, pReg ptmp, rFlagsReg cr) %{
16674 predicate(UseSVE > 0 && ((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U);
16675 match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
16676 effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);
16677 format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}
16678 ins_encode %{
16679 __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register,
16680 $result$$Register, $ztmp1$$FloatRegister,
16681 $ztmp2$$FloatRegister, $pgtmp$$PRegister,
16682 $ptmp$$PRegister, false /* isL */);
16683 %}
16684 ins_pipe(pipe_class_memory);
16685 %}
16686
16687 instruct string_equalsL(iRegP_R1 str1, iRegP_R3 str2, iRegI_R4 cnt,
16688 iRegI_R0 result, rFlagsReg cr)
16689 %{
16690 predicate(((StrEqualsNode*)n)->encoding() == StrIntrinsicNode::LL);
16691 match(Set result (StrEquals (Binary str1 str2) cnt));
16692 effect(USE_KILL str1, USE_KILL str2, USE_KILL cnt, KILL cr);
16693
16694 format %{ "String Equals $str1,$str2,$cnt -> $result" %}
16695 ins_encode %{
16696 // Count is in 8-bit bytes; non-Compact chars are 16 bits.
16697 __ string_equals($str1$$Register, $str2$$Register,
16698 $result$$Register, $cnt$$Register);
16699 %}
16700 ins_pipe(pipe_class_memory);
16701 %}
16702
16703 instruct array_equalsB(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16704 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16705 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16706 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16707 iRegP_R10 tmp, rFlagsReg cr)
16708 %{
16709 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::LL);
16710 match(Set result (AryEq ary1 ary2));
16711 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16712 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16713 TEMP vtmp6, TEMP vtmp7, KILL cr);
16714
16715 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16716 ins_encode %{
16717 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16718 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16719 $result$$Register, $tmp$$Register, 1);
16720 if (tpc == nullptr) {
16721 ciEnv::current()->record_failure("CodeCache is full");
16722 return;
16723 }
16724 %}
16725 ins_pipe(pipe_class_memory);
16726 %}
16727
16728 instruct array_equalsC(iRegP_R1 ary1, iRegP_R2 ary2, iRegI_R0 result,
16729 iRegP_R3 tmp1, iRegP_R4 tmp2, iRegP_R5 tmp3,
16730 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16731 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, vRegD_V7 vtmp7,
16732 iRegP_R10 tmp, rFlagsReg cr)
16733 %{
16734 predicate(((AryEqNode*)n)->encoding() == StrIntrinsicNode::UU);
16735 match(Set result (AryEq ary1 ary2));
16736 effect(KILL tmp, USE_KILL ary1, USE_KILL ary2, TEMP tmp1, TEMP tmp2, TEMP tmp3,
16737 TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16738 TEMP vtmp6, TEMP vtmp7, KILL cr);
16739
16740 format %{ "Array Equals $ary1,ary2 -> $result # KILL $ary1 $ary2 $tmp $tmp1 $tmp2 $tmp3 V0-V7 cr" %}
16741 ins_encode %{
16742 address tpc = __ arrays_equals($ary1$$Register, $ary2$$Register,
16743 $tmp1$$Register, $tmp2$$Register, $tmp3$$Register,
16744 $result$$Register, $tmp$$Register, 2);
16745 if (tpc == nullptr) {
16746 ciEnv::current()->record_failure("CodeCache is full");
16747 return;
16748 }
16749 %}
16750 ins_pipe(pipe_class_memory);
16751 %}
16752
16753 instruct count_positives(iRegP_R1 ary1, iRegI_R2 len, iRegI_R0 result, rFlagsReg cr)
16754 %{
16755 match(Set result (CountPositives ary1 len));
16756 effect(USE_KILL ary1, USE_KILL len, KILL cr);
16757 format %{ "count positives byte[] $ary1,$len -> $result" %}
16758 ins_encode %{
16759 address tpc = __ count_positives($ary1$$Register, $len$$Register, $result$$Register);
16760 if (tpc == nullptr) {
16761 ciEnv::current()->record_failure("CodeCache is full");
16762 return;
16763 }
16764 %}
16765 ins_pipe( pipe_slow );
16766 %}
16767
16768 // fast char[] to byte[] compression
16769 instruct string_compress(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16770 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16771 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16772 iRegI_R0 result, rFlagsReg cr)
16773 %{
16774 match(Set result (StrCompressedCopy src (Binary dst len)));
16775 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP vtmp5,
16776 USE_KILL src, USE_KILL dst, USE len, KILL cr);
16777
16778 format %{ "String Compress $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16779 ins_encode %{
16780 __ char_array_compress($src$$Register, $dst$$Register, $len$$Register,
16781 $result$$Register, $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16782 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16783 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16784 %}
16785 ins_pipe(pipe_slow);
16786 %}
16787
16788 // fast byte[] to char[] inflation
16789 instruct string_inflate(Universe dummy, iRegP_R0 src, iRegP_R1 dst, iRegI_R2 len, iRegP_R3 tmp,
16790 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2, vRegD_V3 vtmp3,
16791 vRegD_V4 vtmp4, vRegD_V5 vtmp5, vRegD_V6 vtmp6, rFlagsReg cr)
16792 %{
16793 match(Set dummy (StrInflatedCopy src (Binary dst len)));
16794 effect(TEMP vtmp0, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3,
16795 TEMP vtmp4, TEMP vtmp5, TEMP vtmp6, TEMP tmp,
16796 USE_KILL src, USE_KILL dst, USE_KILL len, KILL cr);
16797
16798 format %{ "String Inflate $src,$dst # KILL $tmp $src $dst $len V0-V6 cr" %}
16799 ins_encode %{
16800 address tpc = __ byte_array_inflate($src$$Register, $dst$$Register, $len$$Register,
16801 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16802 $vtmp2$$FloatRegister, $tmp$$Register);
16803 if (tpc == nullptr) {
16804 ciEnv::current()->record_failure("CodeCache is full");
16805 return;
16806 }
16807 %}
16808 ins_pipe(pipe_class_memory);
16809 %}
16810
16811 // encode char[] to byte[] in ISO_8859_1
16812 instruct encode_iso_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16813 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16814 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16815 iRegI_R0 result, rFlagsReg cr)
16816 %{
16817 predicate(!((EncodeISOArrayNode*)n)->is_ascii());
16818 match(Set result (EncodeISOArray src (Binary dst len)));
16819 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16820 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16821
16822 format %{ "Encode ISO array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16823 ins_encode %{
16824 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16825 $result$$Register, false,
16826 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16827 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16828 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16829 %}
16830 ins_pipe(pipe_class_memory);
16831 %}
16832
16833 instruct encode_ascii_array(iRegP_R2 src, iRegP_R1 dst, iRegI_R3 len,
16834 vRegD_V0 vtmp0, vRegD_V1 vtmp1, vRegD_V2 vtmp2,
16835 vRegD_V3 vtmp3, vRegD_V4 vtmp4, vRegD_V5 vtmp5,
16836 iRegI_R0 result, rFlagsReg cr)
16837 %{
16838 predicate(((EncodeISOArrayNode*)n)->is_ascii());
16839 match(Set result (EncodeISOArray src (Binary dst len)));
16840 effect(USE_KILL src, USE_KILL dst, USE len, KILL vtmp0, KILL vtmp1,
16841 KILL vtmp2, KILL vtmp3, KILL vtmp4, KILL vtmp5, KILL cr);
16842
16843 format %{ "Encode ASCII array $src,$dst,$len -> $result # KILL $src $dst V0-V5 cr" %}
16844 ins_encode %{
16845 __ encode_iso_array($src$$Register, $dst$$Register, $len$$Register,
16846 $result$$Register, true,
16847 $vtmp0$$FloatRegister, $vtmp1$$FloatRegister,
16848 $vtmp2$$FloatRegister, $vtmp3$$FloatRegister,
16849 $vtmp4$$FloatRegister, $vtmp5$$FloatRegister);
16850 %}
16851 ins_pipe(pipe_class_memory);
16852 %}
16853
16854 //----------------------------- CompressBits/ExpandBits ------------------------
16855
16856 instruct compressBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16857 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16858 match(Set dst (CompressBits src mask));
16859 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16860 format %{ "mov $tsrc, $src\n\t"
16861 "mov $tmask, $mask\n\t"
16862 "bext $tdst, $tsrc, $tmask\n\t"
16863 "mov $dst, $tdst"
16864 %}
16865 ins_encode %{
16866 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16867 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16868 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16869 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16870 %}
16871 ins_pipe(pipe_slow);
16872 %}
16873
16874 instruct compressBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16875 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16876 match(Set dst (CompressBits (LoadI mem) mask));
16877 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16878 format %{ "ldrs $tsrc, $mem\n\t"
16879 "ldrs $tmask, $mask\n\t"
16880 "bext $tdst, $tsrc, $tmask\n\t"
16881 "mov $dst, $tdst"
16882 %}
16883 ins_encode %{
16884 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16885 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16886 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16887 __ sve_bext($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16888 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16889 %}
16890 ins_pipe(pipe_slow);
16891 %}
16892
16893 instruct compressBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16894 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16895 match(Set dst (CompressBits src mask));
16896 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16897 format %{ "mov $tsrc, $src\n\t"
16898 "mov $tmask, $mask\n\t"
16899 "bext $tdst, $tsrc, $tmask\n\t"
16900 "mov $dst, $tdst"
16901 %}
16902 ins_encode %{
16903 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16904 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16905 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16906 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16907 %}
16908 ins_pipe(pipe_slow);
16909 %}
16910
16911 instruct compressBitsL_memcon(iRegLNoSp dst, memory8 mem, immL mask,
16912 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16913 match(Set dst (CompressBits (LoadL mem) mask));
16914 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16915 format %{ "ldrd $tsrc, $mem\n\t"
16916 "ldrd $tmask, $mask\n\t"
16917 "bext $tdst, $tsrc, $tmask\n\t"
16918 "mov $dst, $tdst"
16919 %}
16920 ins_encode %{
16921 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16922 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16923 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16924 __ sve_bext($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16925 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16926 %}
16927 ins_pipe(pipe_slow);
16928 %}
16929
16930 instruct expandBitsI_reg(iRegINoSp dst, iRegIorL2I src, iRegIorL2I mask,
16931 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16932 match(Set dst (ExpandBits src mask));
16933 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16934 format %{ "mov $tsrc, $src\n\t"
16935 "mov $tmask, $mask\n\t"
16936 "bdep $tdst, $tsrc, $tmask\n\t"
16937 "mov $dst, $tdst"
16938 %}
16939 ins_encode %{
16940 __ mov($tsrc$$FloatRegister, __ S, 0, $src$$Register);
16941 __ mov($tmask$$FloatRegister, __ S, 0, $mask$$Register);
16942 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16943 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16944 %}
16945 ins_pipe(pipe_slow);
16946 %}
16947
16948 instruct expandBitsI_memcon(iRegINoSp dst, memory4 mem, immI mask,
16949 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16950 match(Set dst (ExpandBits (LoadI mem) mask));
16951 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16952 format %{ "ldrs $tsrc, $mem\n\t"
16953 "ldrs $tmask, $mask\n\t"
16954 "bdep $tdst, $tsrc, $tmask\n\t"
16955 "mov $dst, $tdst"
16956 %}
16957 ins_encode %{
16958 loadStore(masm, &MacroAssembler::ldrs, $tsrc$$FloatRegister, $mem->opcode(),
16959 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 4);
16960 __ ldrs($tmask$$FloatRegister, $constantaddress($mask));
16961 __ sve_bdep($tdst$$FloatRegister, __ S, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16962 __ mov($dst$$Register, $tdst$$FloatRegister, __ S, 0);
16963 %}
16964 ins_pipe(pipe_slow);
16965 %}
16966
16967 instruct expandBitsL_reg(iRegLNoSp dst, iRegL src, iRegL mask,
16968 vRegD tdst, vRegD tsrc, vRegD tmask) %{
16969 match(Set dst (ExpandBits src mask));
16970 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16971 format %{ "mov $tsrc, $src\n\t"
16972 "mov $tmask, $mask\n\t"
16973 "bdep $tdst, $tsrc, $tmask\n\t"
16974 "mov $dst, $tdst"
16975 %}
16976 ins_encode %{
16977 __ mov($tsrc$$FloatRegister, __ D, 0, $src$$Register);
16978 __ mov($tmask$$FloatRegister, __ D, 0, $mask$$Register);
16979 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
16980 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
16981 %}
16982 ins_pipe(pipe_slow);
16983 %}
16984
16985
16986 instruct expandBitsL_memcon(iRegINoSp dst, memory8 mem, immL mask,
16987 vRegF tdst, vRegF tsrc, vRegF tmask) %{
16988 match(Set dst (ExpandBits (LoadL mem) mask));
16989 effect(TEMP tdst, TEMP tsrc, TEMP tmask);
16990 format %{ "ldrd $tsrc, $mem\n\t"
16991 "ldrd $tmask, $mask\n\t"
16992 "bdep $tdst, $tsrc, $tmask\n\t"
16993 "mov $dst, $tdst"
16994 %}
16995 ins_encode %{
16996 loadStore(masm, &MacroAssembler::ldrd, $tsrc$$FloatRegister, $mem->opcode(),
16997 as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp, 8);
16998 __ ldrd($tmask$$FloatRegister, $constantaddress($mask));
16999 __ sve_bdep($tdst$$FloatRegister, __ D, $tsrc$$FloatRegister, $tmask$$FloatRegister);
17000 __ mov($dst$$Register, $tdst$$FloatRegister, __ D, 0);
17001 %}
17002 ins_pipe(pipe_slow);
17003 %}
17004
17005 // ============================================================================
17006 // This name is KNOWN by the ADLC and cannot be changed.
17007 // The ADLC forces a 'TypeRawPtr::BOTTOM' output type
17008 // for this guy.
17009 instruct tlsLoadP(thread_RegP dst)
17010 %{
17011 match(Set dst (ThreadLocal));
17012
17013 ins_cost(0);
17014
17015 format %{ " -- \t// $dst=Thread::current(), empty" %}
17016
17017 size(0);
17018
17019 ins_encode( /*empty*/ );
17020
17021 ins_pipe(pipe_class_empty);
17022 %}
17023
17024 //----------PEEPHOLE RULES-----------------------------------------------------
17025 // These must follow all instruction definitions as they use the names
17026 // defined in the instructions definitions.
17027 //
17028 // peepmatch ( root_instr_name [preceding_instruction]* );
17029 //
17030 // peepconstraint %{
17031 // (instruction_number.operand_name relational_op instruction_number.operand_name
17032 // [, ...] );
17033 // // instruction numbers are zero-based using left to right order in peepmatch
17034 //
17035 // peepreplace ( instr_name ( [instruction_number.operand_name]* ) );
17036 // // provide an instruction_number.operand_name for each operand that appears
17037 // // in the replacement instruction's match rule
17038 //
17039 // ---------VM FLAGS---------------------------------------------------------
17040 //
17041 // All peephole optimizations can be turned off using -XX:-OptoPeephole
17042 //
17043 // Each peephole rule is given an identifying number starting with zero and
17044 // increasing by one in the order seen by the parser. An individual peephole
17045 // can be enabled, and all others disabled, by using -XX:OptoPeepholeAt=#
17046 // on the command-line.
17047 //
17048 // ---------CURRENT LIMITATIONS----------------------------------------------
17049 //
17050 // Only match adjacent instructions in same basic block
17051 // Only equality constraints
17052 // Only constraints between operands, not (0.dest_reg == RAX_enc)
17053 // Only one replacement instruction
17054 //
17055 // ---------EXAMPLE----------------------------------------------------------
17056 //
17057 // // pertinent parts of existing instructions in architecture description
17058 // instruct movI(iRegINoSp dst, iRegI src)
17059 // %{
17060 // match(Set dst (CopyI src));
17061 // %}
17062 //
17063 // instruct incI_iReg(iRegINoSp dst, immI1 src, rFlagsReg cr)
17064 // %{
17065 // match(Set dst (AddI dst src));
17066 // effect(KILL cr);
17067 // %}
17068 //
17069 // // Change (inc mov) to lea
17070 // peephole %{
17071 // // increment preceded by register-register move
17072 // peepmatch ( incI_iReg movI );
17073 // // require that the destination register of the increment
17074 // // match the destination register of the move
17075 // peepconstraint ( 0.dst == 1.dst );
17076 // // construct a replacement instruction that sets
17077 // // the destination to ( move's source register + one )
17078 // peepreplace ( leaI_iReg_immI( 0.dst 1.src 0.src ) );
17079 // %}
17080 //
17081
17082 // Implementation no longer uses movX instructions since
17083 // machine-independent system no longer uses CopyX nodes.
17084 //
17085 // peephole
17086 // %{
17087 // peepmatch (incI_iReg movI);
17088 // peepconstraint (0.dst == 1.dst);
17089 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17090 // %}
17091
17092 // peephole
17093 // %{
17094 // peepmatch (decI_iReg movI);
17095 // peepconstraint (0.dst == 1.dst);
17096 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17097 // %}
17098
17099 // peephole
17100 // %{
17101 // peepmatch (addI_iReg_imm movI);
17102 // peepconstraint (0.dst == 1.dst);
17103 // peepreplace (leaI_iReg_immI(0.dst 1.src 0.src));
17104 // %}
17105
17106 // peephole
17107 // %{
17108 // peepmatch (incL_iReg movL);
17109 // peepconstraint (0.dst == 1.dst);
17110 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17111 // %}
17112
17113 // peephole
17114 // %{
17115 // peepmatch (decL_iReg movL);
17116 // peepconstraint (0.dst == 1.dst);
17117 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17118 // %}
17119
17120 // peephole
17121 // %{
17122 // peepmatch (addL_iReg_imm movL);
17123 // peepconstraint (0.dst == 1.dst);
17124 // peepreplace (leaL_iReg_immL(0.dst 1.src 0.src));
17125 // %}
17126
17127 // peephole
17128 // %{
17129 // peepmatch (addP_iReg_imm movP);
17130 // peepconstraint (0.dst == 1.dst);
17131 // peepreplace (leaP_iReg_imm(0.dst 1.src 0.src));
17132 // %}
17133
17134 // // Change load of spilled value to only a spill
17135 // instruct storeI(memory mem, iRegI src)
17136 // %{
17137 // match(Set mem (StoreI mem src));
17138 // %}
17139 //
17140 // instruct loadI(iRegINoSp dst, memory mem)
17141 // %{
17142 // match(Set dst (LoadI mem));
17143 // %}
17144 //
17145
17146 //----------SMARTSPILL RULES---------------------------------------------------
17147 // These must follow all instruction definitions as they use the names
17148 // defined in the instructions definitions.
17149
17150 // Local Variables:
17151 // mode: c++
17152 // End: